diff --git a/.gitignore b/.gitignore index 693e89349f..16e14f29a4 100644 --- a/.gitignore +++ b/.gitignore @@ -137,3 +137,6 @@ jobs/sftp-*/archive # ruff **/.ruff_cache +# Delta restore run artifacts +data-tool/scripts/generated/delta_restore/ + diff --git a/colin-api/src/colin_api/models/__init__.py b/colin-api/src/colin_api/models/__init__.py index e2ec54740f..07fcac795b 100644 --- a/colin-api/src/colin_api/models/__init__.py +++ b/colin-api/src/colin_api/models/__init__.py @@ -1,6 +1,8 @@ """Model imports.""" + from .address import Address from .business import Business +from .business_snapshot import BusinessSnapshot from .cont_out import ContOut from .corp_involved import CorpInvolved from .corp_name import CorpName diff --git a/colin-api/src/colin_api/models/business.py b/colin-api/src/colin_api/models/business.py index 698cb2ee28..c26a550edb 100644 --- a/colin-api/src/colin_api/models/business.py +++ b/colin-api/src/colin_api/models/business.py @@ -81,6 +81,13 @@ class CorpStateTypes(Enum): TypeCodes.ULC_CONTINUE_IN.value, TypeCodes.CCC_CONTINUE_IN.value] + # Corp types that auth may sync auth info (including the corp password) for. + # These are the COLIN businesses that can be affiliated in the business registry + # dashboard while not loaded in LEAR. + AUTH_INFO_TYPES = [TypeCodes.BC_COMP.value, + TypeCodes.ULC_COMP.value, + TypeCodes.CCC_COMP.value] + NUMBERED_CORP_NAME_SUFFIX = { TypeCodes.BCOMP.value: 'B.C. LTD.', TypeCodes.BC_COMP.value: 'B.C. LTD.', @@ -116,6 +123,16 @@ class CorpStateTypes(Enum): def __init__(self): """Initialize with all values None.""" + @property + def lear_state(self) -> str: + """Return the LEAR-style business state: only op-state ACT is active, everything else is historical. + + Same rule as the tombstone migration (data-tool tombstone_utils) and the search + importers - keyed off CORP_OP_STATE.OP_STATE_TYP_CD (ACT/HIS), not the + fine-grained corp state code. + """ + return 'ACTIVE' if self.corp_state_class == 'ACT' else 'HISTORICAL' + def as_dict(self) -> Dict: """Return dict version of self.""" return { @@ -374,6 +391,50 @@ def find_by_identifier(cls, # pylint: disable=too-many-statements # pass through exception to caller raise err + @classmethod + def get_auth_info(cls, identifier: str) -> Dict: + """Return the info needed by auth to create/refresh an entity for a COLIN business. + + The response contains the corporation password, so callers must be service + authenticated and must never log the result or pass it on to a browser. + Restricted to the corp types that can be affiliated while not loaded in LEAR. + """ + # auth holds the prefixed identifier, COLIN stores the bare corp num. BC/ULC/CC all + # use the BC prefix, so it can be stripped without knowing the legal type first. + if identifier.startswith('BC'): + identifier = identifier[2:] + + # acquire a single session from the pool and reuse it for both lookups + con = DB.connection + business = cls.find_by_identifier(identifier, corp_types=cls.AUTH_INFO_TYPES, con=con) + + cursor = con.cursor() + cursor.execute( + """ + select corp_password + from CORPORATION + where corp_num=:corp_num + """, + corp_num=identifier + ) + result = cursor.fetchone() + pass_code = result[0] if result else None + + return { + 'identifier': business.corp_num, + 'legalName': business.corp_name, + 'legalType': business.corp_type, + # LEAR-style state, not COLIN's free-text status description - auth stores it + # verbatim and the dashboard filters on ACTIVE/HISTORICAL + 'status': business.lear_state, + 'goodStanding': business.good_standing, + 'businessNumber': business.business_number, + # find_by_identifier returns the COLIN 'True'/'False' string; normalize for callers + 'adminFreeze': business.admin_freeze == 'True', + 'email': business.email, + 'passCode': pass_code + } + @classmethod def create_corporation(cls, con, filing_info: Dict): """Insert a new business from an incorporation filing.""" diff --git a/colin-api/src/colin_api/models/business_snapshot.py b/colin-api/src/colin_api/models/business_snapshot.py new file mode 100644 index 0000000000..662de119f4 --- /dev/null +++ b/colin-api/src/colin_api/models/business_snapshot.py @@ -0,0 +1,199 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Snapshot of a COLIN business, normalized to LEAR structure.""" +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Dict, Optional + +import pycountry +from flask import current_app + +from colin_api.exceptions import PartiesNotFoundException +from colin_api.models.business import Business +from colin_api.models.corp_party import Party +from colin_api.models.office import Office +from colin_api.models.shares import ShareObject +from colin_api.resources.db import DB + + +class BusinessSnapshot: # pylint: disable=too-few-public-methods + """Builds the LEAR-structured snapshot dict for a COLIN business.""" + + # snapshot offices are limited to the two the amalgamation flow prepopulates + OFFICE_TYPES = ('registeredOffice', 'recordsOffice') + + @classmethod + def get_snapshot(cls, orig_identifier: str) -> Dict: + """Return the business/parties/offices/shareClasses/resolutions snapshot.""" + identifier = orig_identifier + if identifier.startswith('BC'): + identifier = identifier[2:] + + con = DB.connection + business = Business.find_by_identifier(identifier, con=con) + cursor = con.cursor() + + try: + parties = Party.get_current(cursor, identifier) + except PartiesNotFoundException: + # no current directors on file is bad data but not an error state here + parties = [] + + offices = Office.convert_obj_list(Office.get_current(cursor, identifier)) or {} + + # mirror the /sharestructure resource: current structure is the one with no end event + share_structs = ShareObject.get_all(cursor, identifier) or [] + share_struct = next((x for x in share_structs if not x.end_event_id), None) + share_classes = share_struct.to_dict()['shareClasses'] if share_struct else [] + + resolutions = Business.get_resolutions(cursor, identifier) + + return { + 'business': cls._business_dict(business, orig_identifier, cursor), + 'parties': [cls._normalize_party(party) for party in parties], + 'offices': { + office_type: cls._normalize_office(office) + for office_type, office in offices.items() if office_type in cls.OFFICE_TYPES + }, + 'shareClasses': [cls._normalize_share_class(share_class) for share_class in share_classes], + 'resolutions': [{'date': date} for date in resolutions], + } + + @classmethod + def _business_dict(cls, business: Business, orig_identifier: str, cursor) -> Dict: + """Return the business section in the shape of LEAR's slim business json.""" + return { + 'identifier': orig_identifier, + 'legalName': business.corp_name, + 'legalType': business.corp_type, + 'state': business.lear_state, + # tri-state: None when COLIN can't compute good standing for the corp + 'goodStanding': business.good_standing, + # find_by_identifier returns the COLIN 'True'/'False' string + 'adminFreeze': business.admin_freeze == 'True', + 'foundingDate': cls._to_iso_datetime(business.founding_date), + 'taxId': business.business_number, + 'hasFutureEffectiveFiling': cls._has_future_effective_filing(cursor, business.corp_num), + } + + @staticmethod + def _has_future_effective_filing(cursor, corp_num: str) -> bool: + """Return whether any filing has an effective date still in the future.""" + current_date = datetime.now(timezone.utc).strftime('%Y-%m-%d') + cursor.execute( + """ + select count(*) + from event join filing on event.event_id = filing.event_id + where event.corp_num=:corp_num + and filing.effective_dt > TO_DATE(:current_date, 'YYYY-mm-dd') + """, + corp_num=corp_num, + current_date=current_date + ) + return cursor.fetchone()[0] > 0 + + @classmethod + def _normalize_party(cls, party: Party) -> Dict: + """Return a party in the structure of LEAR's /parties items.""" + raw = party.as_dict() + return { + 'officer': {**raw['officer'], 'id': raw['id'], 'email': None}, + 'deliveryAddress': cls._normalize_address(raw['deliveryAddress']), + 'mailingAddress': cls._normalize_address(raw['mailingAddress']), + 'roles': raw['roles'] or [], + } + + @classmethod + def _normalize_office(cls, office: Dict) -> Dict: + """Return an office's addresses in LEAR structure.""" + return { + 'deliveryAddress': cls._normalize_address(office.get('deliveryAddress')), + 'mailingAddress': cls._normalize_address(office.get('mailingAddress')), + } + + @classmethod + def _normalize_address(cls, address: Optional[Dict]) -> Optional[Dict]: + """Return an Address.as_dict in the structure of LEAR's address json.""" + if not address: + return None + return { + 'id': address.get('addressId'), + 'streetAddress': address.get('streetAddress'), + 'streetAddressAdditional': address.get('streetAddressAdditional'), + 'addressCity': address.get('addressCity'), + 'addressRegion': address.get('addressRegion'), + 'addressCountry': cls._country_to_alpha2(address.get('addressCountry')), + 'postalCode': address.get('postalCode'), + 'deliveryInstructions': address.get('deliveryInstructions'), + } + + @classmethod + def _normalize_share_class(cls, share_class: Dict) -> Dict: + """Return a share class in the structure of LEAR's /share-classes items.""" + return { + 'id': share_class['id'], + 'name': share_class['name'], + # COLIN never stores a priority - the class id preserves creation order + 'priority': share_class['displayOrder'], + 'hasMaximumShares': share_class['hasMaximumShares'], + 'maxNumberOfShares': cls._to_int(share_class['maxNumberOfShares']), + 'hasParValue': share_class['hasParValue'], + 'parValue': float(share_class['parValue']) if share_class['parValue'] is not None else None, + 'currency': share_class['currency'], + 'currencyAdditional': share_class['currencyAdditional'], + 'hasRightsOrRestrictions': share_class['hasRightsOrRestrictions'], + 'series': [cls._normalize_share_series(series) for series in share_class['series']], + } + + @classmethod + def _normalize_share_series(cls, series: Dict) -> Dict: + """Return a share series in the structure of LEAR's series json.""" + return { + 'id': series['id'], + 'name': series['name'], + 'priority': series['displayOrder'], + 'hasMaximumShares': series['hasMaximumShares'], + 'maxNumberOfShares': cls._to_int(series['maxNumberOfShares']), + 'hasRightsOrRestrictions': series['hasRightsOrRestrictions'], + } + + _country_cache: Dict[str, str] = {} + + @classmethod + def _country_to_alpha2(cls, country: Optional[str]) -> Optional[str]: + """Map COLIN's country description to the alpha-2 code LEAR stores.""" + if not country: + return country + if len(country) == 2: # already a code + return country + if country not in cls._country_cache: + try: + cls._country_cache[country] = pycountry.countries.search_fuzzy(country)[0].alpha_2 + except LookupError: + current_app.logger.error('Could not map COLIN country %s to an alpha-2 code', country) + cls._country_cache[country] = country + return cls._country_cache[country] + + @staticmethod + def _to_int(value) -> Optional[int]: + """Coerce an Oracle NUMBER to int, preserving None.""" + return int(value) if value is not None else None + + @staticmethod + def _to_iso_datetime(value: Optional[str]) -> Optional[str]: + """Rewrite convert_to_json_datetime's '-00:00' suffix to the ISO '+00:00' LEAR uses.""" + if value and value.endswith('-00:00'): + return value[:-6] + '+00:00' + return value diff --git a/colin-api/src/colin_api/models/filing.py b/colin-api/src/colin_api/models/filing.py index d419929e2b..f55d51176a 100644 --- a/colin-api/src/colin_api/models/filing.py +++ b/colin-api/src/colin_api/models/filing.py @@ -1298,7 +1298,7 @@ def get_future_effective_filings(cls, business: Business) -> List: """Get the list of all future effective filings for a business.""" try: future_effective_filings = [] - current_date = datetime.datetime.utcnow().strftime('%Y-%m-%d') + current_date = datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d') cursor = DB.connection.cursor() cursor.execute( """ diff --git a/colin-api/src/colin_api/models/shares.py b/colin-api/src/colin_api/models/shares.py index 2124ea7894..3e9d832578 100644 --- a/colin-api/src/colin_api/models/shares.py +++ b/colin-api/src/colin_api/models/shares.py @@ -58,6 +58,7 @@ class ShareClass(Share): # pylint: disable=too-many-instance-attributes; # pylint: disable=too-few-public-methods currency_type = None + other_currency = None has_par_value = None par_value_amt = None series = None @@ -77,6 +78,7 @@ def to_dict(self): 'maxNumberOfShares': self.max_number_shares, 'parValue': self.par_value_amt, 'currency': self.currency_type, + 'currencyAdditional': self.other_currency, 'hasMaximumShares': self.has_max_shares == 'N' or False, 'hasParValue': self.has_par_value == 'Y' or False, 'hasRightsOrRestrictions': self.has_special_rights == 'Y' or False, @@ -140,6 +142,7 @@ def _get_share_classes(cls, cursor, event_id, corp_num): row = dict(zip([x[0].lower() for x in description], row)) share_class = ShareClass() share_class.currency_type = row['currency_typ_cd'] + share_class.other_currency = row['other_currency'] share_class.has_max_shares = row['max_share_ind'] share_class.has_special_rights = row['spec_rights_ind'] share_class.has_par_value = row['par_value_ind'] diff --git a/colin-api/src/colin_api/resources/business.py b/colin-api/src/colin_api/resources/business.py index 6276762853..ff1f859511 100644 --- a/colin-api/src/colin_api/resources/business.py +++ b/colin-api/src/colin_api/resources/business.py @@ -21,7 +21,7 @@ from flask_restx import Namespace, Resource, cors from colin_api.exceptions import GenericException -from colin_api.models import Business, CorpName +from colin_api.models import Business, BusinessSnapshot, CorpName from colin_api.resources.db import DB from colin_api.utils.auth import COLIN_SVC_ROLE, jwt from colin_api.utils.util import cors_preflight @@ -62,6 +62,60 @@ def get(identifier: str): ), HTTPStatus.INTERNAL_SERVER_ERROR +@cors_preflight('GET') +@API.route('//auth-info', methods=['GET', 'OPTIONS']) +class BusinessAuthInfo(Resource): + """Business info used by auth to create/refresh an entity for a COLIN business.""" + + @staticmethod + @cors.crossdomain(origin='*') + @jwt.requires_roles([COLIN_SVC_ROLE]) + def get(identifier: str): + """Return the auth info for a COLIN business. + + Service endpoint only - the response contains the corporation password so it must + never be proxied to a browser, and the payload must not be logged. + """ + try: + auth_info = Business.get_auth_info(identifier) + return jsonify(auth_info), HTTPStatus.OK + + except GenericException as err: # pylint: disable=duplicate-code + return jsonify({'message': err.error}), err.status_code + + except Exception as err: # pylint: disable=broad-except; want to catch all errors + # general catch-all exception. Only the identifier is logged - never the payload. + current_app.logger.error('Error retrieving auth info for %s: %s', identifier, type(err).__name__) + return jsonify( + {'message': 'Error when trying to retrieve business record from COLIN'} + ), HTTPStatus.INTERNAL_SERVER_ERROR + + +@cors_preflight('GET') +@API.route('//snapshot', methods=['GET', 'OPTIONS']) +class BusinessSnapshotInfo(Resource): + """LEAR-shaped snapshot of a COLIN business.""" + + @staticmethod + @cors.crossdomain(origin='*') + @jwt.requires_roles([COLIN_SVC_ROLE]) + def get(identifier: str): + """Return the business/parties/offices/shareClasses/resolutions snapshot.""" + try: + snapshot = BusinessSnapshot.get_snapshot(identifier) + return jsonify(snapshot), HTTPStatus.OK + + except GenericException as err: # pylint: disable=duplicate-code + return jsonify({'message': err.error}), err.status_code + + except Exception as err: # pylint: disable=broad-except; want to catch all errors + # general catch-all exception + current_app.logger.error(err.with_traceback(None)) + return jsonify( + {'message': 'Error when trying to retrieve business record from COLIN'} + ), HTTPStatus.INTERNAL_SERVER_ERROR + + @cors_preflight('GET, POST') @API.route('//', methods=['GET']) @API.route('/', methods=['POST']) diff --git a/colin-api/src/colin_api/version.py b/colin-api/src/colin_api/version.py index e7eaeba399..74e4fcb3db 100644 --- a/colin-api/src/colin_api/version.py +++ b/colin-api/src/colin_api/version.py @@ -22,4 +22,4 @@ Development release segment: .devN """ -__version__ = '2.171.7' # pylint: disable=invalid-name +__version__ = '2.172.0' # pylint: disable=invalid-name diff --git a/colin-api/tests/unit/__init__.py b/colin-api/tests/unit/__init__.py index d755f6a1d8..2201725833 100644 --- a/colin-api/tests/unit/__init__.py +++ b/colin-api/tests/unit/__init__.py @@ -15,4 +15,124 @@ """The Unit Test for the API. For our purposes this server and its Postgres Database are part of the Unit Test Suite. + +Also holds the shared builders for COLIN model objects used by the mocked-Oracle test +suites (auth-info, snapshot) - the fixtures composing them live in conftest.py. """ +from colin_api.models import Business, Office, Party, ShareObject +from colin_api.models.shares import Share, ShareClass +from colin_api.utils.auth import jwt as _jwt + + +# what Address.as_dict emits +RAW_ADDRESS = { + 'streetAddress': '123 FAKE ST', + 'streetAddressAdditional': '', + 'addressCity': 'VICTORIA', + 'addressRegion': 'BC', + 'addressCountry': 'CANADA', + 'postalCode': 'V8V 8V8', + 'deliveryInstructions': '', + 'addressId': 4444, + 'actions': [] +} + +# the LEAR shape of the same address +LEAR_ADDRESS = { + 'id': 4444, + 'streetAddress': '123 FAKE ST', + 'streetAddressAdditional': '', + 'addressCity': 'VICTORIA', + 'addressRegion': 'BC', + 'addressCountry': 'CA', + 'postalCode': 'V8V 8V8', + 'deliveryInstructions': '' +} + + +def bypass_auth(mocker, roles_valid=True): + """Stub out token validation on the jwt manager. + + The attribute names differ between flask-jwt-oidc releases (and validate_roles has taken + different arities), so patch whichever the installed version exposes. MagicMock accepts + any signature, so this holds across versions. + """ + for attr in ('_require_auth_validation', '_validate_token', 'validate_token'): + if hasattr(_jwt, attr): + mocker.patch.object(_jwt, attr, return_value=None) + mocker.patch.object(_jwt, 'validate_roles', return_value=roles_valid) + + +def build_business(**overrides): + """Return a Business object as find_by_identifier would build it.""" + business = Business() + business.corp_num = '0870226' + business.corp_name = 'COLIN TEST COMPANY LTD.' + business.corp_type = 'BC' + # CORP_OP_STATE.OP_STATE_TYP_CD - only ever ACT/HIS; drives the response's LEAR-style state + business.corp_state_class = 'ACT' + business.good_standing = True + business.business_number = '791861078BC0001' + # COLIN returns admin_freeze as a 'True'/'False' string + business.admin_freeze = 'False' + # convert_to_json_datetime emits a '-00:00' suffix + business.founding_date = '2000-01-01T08:00:00-00:00' + business.email = 'registered.office@test.com' + for key, value in overrides.items(): + setattr(business, key, value) + return business + + +def build_director(): + """Return a Party object as Party.get_current would build it.""" + party = Party() + party.officer = { + 'firstName': 'JANE', 'lastName': 'DOE', 'middleInitial': '', + 'organizationName': '', 'partyType': 'person' + } + party.delivery_address = dict(RAW_ADDRESS) + party.mailing_address = dict(RAW_ADDRESS) + party.title = '' + party.appointment_date = '2010-05-05' + party.cessation_date = None + party.start_event_id = 111 + party.end_event_id = '' + party.corp_party_id = 999 + party.roles = [{'roleType': 'Director', 'appointmentDate': '2010-05-05', 'cessationDate': None}] + return party + + +def build_office(office_type): + """Return an Office object as Office.get_current would build it.""" + office = Office() + office.office_type = office_type + office.delivery_address = dict(RAW_ADDRESS) + office.mailing_address = dict(RAW_ADDRESS) + return office + + +def build_share_structure(): + """Return the current ShareObject as ShareObject.get_all would build it.""" + series = Share() + series.share_id = 1 + series.share_name = 'SERIES 1' + series.has_max_shares = 'Y' # COLIN semantics: 'N' means has a maximum + series.has_special_rights = 'N' + series.max_number_shares = None + + share_class = ShareClass() + share_class.share_id = 0 + share_class.share_name = 'CLASS A' + share_class.currency_type = 'OTH' + share_class.other_currency = 'BITCOIN' + share_class.has_max_shares = 'N' + share_class.has_par_value = 'Y' + share_class.has_special_rights = 'Y' + share_class.par_value_amt = 1.5 + share_class.max_number_shares = 10000 + share_class.series = [series] + + share_struct = ShareObject() + share_struct.end_event_id = None + share_struct.share_classes = [share_class] + return share_struct diff --git a/colin-api/tests/unit/api/test_business_auth_info.py b/colin-api/tests/unit/api/test_business_auth_info.py new file mode 100644 index 0000000000..423892c1d9 --- /dev/null +++ b/colin-api/tests/unit/api/test_business_auth_info.py @@ -0,0 +1,166 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests to assure the business auth-info end-point. + +Test-Suite to ensure that the /businesses//auth-info endpoint used by auth +to create and refresh entities for COLIN businesses is working as expected. + +The Oracle connection is mocked out, so these run without access to COLIN. The wider +business lookup (find_by_identifier) is stubbed - what is exercised here is the endpoint's +own behaviour: the corp password query, the corp type restriction, response shape and +error mapping. +""" +from colin_api.exceptions import BusinessNotFoundException +from colin_api.models import Business +from tests.unit import build_business, bypass_auth + + +AUTH_INFO_URL = '/api/v1/businesses/BC0870226/auth-info' +PASS_CODE = '111111111' + + +def test_get_auth_info(client, mocker, authorized, mock_db): # pylint: disable=unused-argument + """Assert the auth info needed by auth is returned for a COLIN corp.""" + mocker.patch.object(Business, 'find_by_identifier', return_value=build_business()) + # the corp password row - conftest's mock_db defaults fetchone to a count-style (0,) + mock_db.cursor.fetchone.return_value = (PASS_CODE,) + + rv = client.get(AUTH_INFO_URL) + + assert rv.status_code == 200 + assert rv.json == { + 'identifier': '0870226', + 'legalName': 'COLIN TEST COMPANY LTD.', + 'legalType': 'BC', + 'status': 'ACTIVE', + 'goodStanding': True, + 'businessNumber': '791861078BC0001', + 'adminFreeze': False, + 'email': 'registered.office@test.com', + 'passCode': PASS_CODE + } + + +def test_get_auth_info_maps_historical_state(client, mocker, authorized, mock_db): # pylint: disable=unused-argument + """Assert a non-active corp (eg. amalgamated or dissolved) reports the LEAR-style HISTORICAL.""" + mocker.patch.object(Business, 'find_by_identifier', return_value=build_business(corp_state_class='HIS')) + + rv = client.get(AUTH_INFO_URL) + + assert rv.status_code == 200 + assert rv.json['status'] == 'HISTORICAL' + + +def test_get_auth_info_strips_bc_prefix(client, mocker, authorized, mock_db): # pylint: disable=unused-argument + """Assert the BC prefix is stripped, since COLIN stores the bare corp number.""" + find = mocker.patch.object(Business, 'find_by_identifier', return_value=build_business()) + + assert client.get(AUTH_INFO_URL).status_code == 200 + + assert find.call_args.args[0] == '0870226' + # the corp password lookup uses the bare corp num too + assert mock_db.cursor.execute.call_args.kwargs['corp_num'] == '0870226' + + +def test_get_auth_info_restricted_to_in_scope_corp_types(client, mocker, authorized, + mock_db): # pylint: disable=unused-argument + """Assert only BC/ULC/CC are looked up. + + The response carries a credential, so the surface is limited to the corp types that can + be affiliated from COLIN while not loaded in LEAR. + """ + find = mocker.patch.object(Business, 'find_by_identifier', return_value=build_business()) + + client.get(AUTH_INFO_URL) + + assert find.call_args.kwargs['corp_types'] == ['BC', 'ULC', 'CC'] + + +def test_get_auth_info_queries_corp_password(client, mocker, authorized, mock_db): # pylint: disable=unused-argument + """Assert the passcode is read from the corporation corp_password column.""" + mocker.patch.object(Business, 'find_by_identifier', return_value=build_business()) + + client.get(AUTH_INFO_URL) + + assert 'corp_password' in mock_db.cursor.execute.call_args.args[0].lower() + + +def test_get_auth_info_reuses_single_connection(client, mocker, authorized, + mock_db): # pylint: disable=unused-argument + """Assert one pooled session is used for both lookups. + + DB.connection acquires a session from a pool capped at 10, so the business lookup and + the passcode lookup must share one. + """ + find = mocker.patch.object(Business, 'find_by_identifier', return_value=build_business()) + + client.get(AUTH_INFO_URL) + + # both the business lookup and the passcode cursor come off the same connection + assert find.call_args.kwargs['con'] is mock_db.connection + assert mock_db.connection.cursor.call_count == 1 + + +def test_get_auth_info_normalizes_admin_freeze(client, mocker, authorized, + mock_db): # pylint: disable=unused-argument + """Assert COLIN's 'True'/'False' string is returned as a real boolean.""" + mocker.patch.object(Business, 'find_by_identifier', return_value=build_business(admin_freeze='True')) + + rv = client.get(AUTH_INFO_URL) + + assert rv.status_code == 200 + assert rv.json['adminFreeze'] is True + + +def test_get_auth_info_without_passcode(client, mocker, authorized, mock_db): # pylint: disable=unused-argument + """Assert a business with no corp password returns a null passcode rather than failing.""" + mocker.patch.object(Business, 'find_by_identifier', return_value=build_business()) + mock_db.cursor.fetchone.return_value = None + + rv = client.get(AUTH_INFO_URL) + + assert rv.status_code == 200 + assert rv.json['passCode'] is None + + +def test_get_auth_info_no_results(client, mocker, authorized, mock_db): # pylint: disable=unused-argument + """Assert a business that does not exist in COLIN returns a 404.""" + mocker.patch.object(Business, 'find_by_identifier', + side_effect=BusinessNotFoundException(identifier='BC0000000')) + + rv = client.get('/api/v1/businesses/BC0000000/auth-info') + + assert rv.status_code == 404 + assert None is not rv.json['message'] + + +def test_get_auth_info_handles_unexpected_error(client, mocker, authorized, + mock_db): # pylint: disable=unused-argument + """Assert an unexpected failure returns a 500 without leaking the payload.""" + mocker.patch.object(Business, 'find_by_identifier', side_effect=Exception('oracle exploded')) + + rv = client.get(AUTH_INFO_URL) + + assert rv.status_code == 500 + assert 'oracle exploded' not in str(rv.json) + + +def test_get_auth_info_requires_colin_service_role(client, mocker): + """Assert the endpoint is gated on the colin service role.""" + bypass_auth(mocker, roles_valid=False) + + rv = client.get(AUTH_INFO_URL) + + assert rv.status_code == 401 diff --git a/colin-api/tests/unit/api/test_business_snapshot.py b/colin-api/tests/unit/api/test_business_snapshot.py new file mode 100644 index 0000000000..23c3ea1eea --- /dev/null +++ b/colin-api/tests/unit/api/test_business_snapshot.py @@ -0,0 +1,184 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests to assure the business snapshot end-point.""" +from colin_api.exceptions import BusinessNotFoundException, PartiesNotFoundException +from colin_api.models import Business, Party, ShareObject +from tests.unit import LEAR_ADDRESS, build_business, bypass_auth + + +SNAPSHOT_URL = '/api/v1/businesses/BC0870226/snapshot' + + +def test_get_snapshot(client, mocker, authorized, mock_db, mock_lookups): # pylint: disable=unused-argument + """Assert the full LEAR-normalized snapshot is returned.""" + rv = client.get(SNAPSHOT_URL) + + assert rv.status_code == 200 + assert rv.json == { + 'business': { + 'identifier': 'BC0870226', + 'legalName': 'COLIN TEST COMPANY LTD.', + 'legalType': 'BC', + 'state': 'ACTIVE', + 'goodStanding': True, + 'adminFreeze': False, + 'foundingDate': '2000-01-01T08:00:00+00:00', + 'taxId': '791861078BC0001', + 'hasFutureEffectiveFiling': False + }, + 'parties': [{ + 'officer': { + 'id': 999, + 'firstName': 'JANE', + 'lastName': 'DOE', + 'middleInitial': '', + 'organizationName': '', + 'partyType': 'person', + 'email': None + }, + 'deliveryAddress': LEAR_ADDRESS, + 'mailingAddress': LEAR_ADDRESS, + 'roles': [{'roleType': 'Director', 'appointmentDate': '2010-05-05', 'cessationDate': None}] + }], + 'offices': { + 'registeredOffice': {'deliveryAddress': LEAR_ADDRESS, 'mailingAddress': LEAR_ADDRESS}, + 'recordsOffice': {'deliveryAddress': LEAR_ADDRESS, 'mailingAddress': LEAR_ADDRESS} + }, + 'shareClasses': [{ + 'id': 0, + 'name': 'CLASS A', + 'priority': 0, + 'hasMaximumShares': True, + 'maxNumberOfShares': 10000, + 'hasParValue': True, + 'parValue': 1.5, + 'currency': 'OTH', + 'currencyAdditional': 'BITCOIN', + 'hasRightsOrRestrictions': True, + 'series': [{ + 'id': 1, + 'name': 'SERIES 1', + 'priority': 1, + 'hasMaximumShares': False, + 'maxNumberOfShares': None, + 'hasRightsOrRestrictions': False + }] + }], + 'resolutions': [{'date': '2020-01-01'}, {'date': '2019-06-15'}] + } + + +def test_get_snapshot_maps_historical_state(client, mocker, authorized, mock_db, + mock_lookups): # pylint: disable=unused-argument + """Assert a non-active corp (eg. amalgamated or dissolved) reports the LEAR-style HISTORICAL.""" + mocker.patch.object(Business, 'find_by_identifier', return_value=build_business(corp_state_class='HIS')) + + rv = client.get(SNAPSHOT_URL) + + assert rv.status_code == 200 + assert rv.json['business']['state'] == 'HISTORICAL' + + +def test_get_snapshot_reports_future_effective_filing(client, mocker, authorized, mock_db, + mock_lookups): # pylint: disable=unused-argument + """Assert an outstanding future effective filing is reported.""" + mock_db.cursor.fetchone.return_value = (1,) + + rv = client.get(SNAPSHOT_URL) + + assert rv.status_code == 200 + assert rv.json['business']['hasFutureEffectiveFiling'] is True + # the count query runs against the bare corp num + assert mock_db.cursor.execute.call_args.kwargs['corp_num'] == '0870226' + assert 'effective_dt' in mock_db.cursor.execute.call_args.args[0] + + +def test_get_snapshot_without_parties(client, mocker, authorized, mock_db, + mock_lookups): # pylint: disable=unused-argument + """Assert a corp with no current directors on file still returns a snapshot.""" + mocker.patch.object(Party, 'get_current', side_effect=PartiesNotFoundException(identifier='0870226')) + + rv = client.get(SNAPSHOT_URL) + + assert rv.status_code == 200 + assert rv.json['parties'] == [] + + +def test_get_snapshot_without_share_structure(client, mocker, authorized, mock_db, + mock_lookups): # pylint: disable=unused-argument + """Assert a corp with no share structure returns an empty list rather than failing.""" + mocker.patch.object(ShareObject, 'get_all', return_value=None) + + rv = client.get(SNAPSHOT_URL) + + assert rv.status_code == 200 + assert rv.json['shareClasses'] == [] + + +def test_get_snapshot_null_good_standing(client, mocker, authorized, mock_db, + mock_lookups): # pylint: disable=unused-argument + """Assert COLIN's tri-state good standing passes through as null when unknown.""" + mocker.patch.object(Business, 'find_by_identifier', return_value=build_business(good_standing=None)) + + rv = client.get(SNAPSHOT_URL) + + assert rv.status_code == 200 + assert rv.json['business']['goodStanding'] is None + + +def test_get_snapshot_single_connection_and_scope(client, mocker, authorized, mock_db, + mock_lookups): # pylint: disable=unused-argument + """Assert one pooled session is used and the lookup is restricted to in-scope corp types.""" + find = mocker.patch.object(Business, 'find_by_identifier', return_value=build_business()) + + assert client.get(SNAPSHOT_URL).status_code == 200 + + # BC prefix stripped, same restriction as auth-info, and the pooled session is shared + assert find.call_args.args[0] == '0870226' + assert find.call_args.kwargs['con'] is mock_db.connection + assert 'corp_types' not in find.call_args.kwargs + assert mock_db.connection.cursor.call_count == 1 + + +def test_get_snapshot_no_results(client, mocker, authorized, mock_db, + mock_lookups): # pylint: disable=unused-argument + """Assert a business that does not exist in COLIN returns a 404.""" + mocker.patch.object(Business, 'find_by_identifier', + side_effect=BusinessNotFoundException(identifier='BC0000000')) + + rv = client.get('/api/v1/businesses/BC0000000/snapshot') + + assert rv.status_code == 404 + assert None is not rv.json['message'] + + +def test_get_snapshot_handles_unexpected_error(client, mocker, authorized, mock_db, + mock_lookups): # pylint: disable=unused-argument + """Assert an unexpected failure returns a 500 without leaking internals.""" + mocker.patch.object(Business, 'find_by_identifier', side_effect=Exception('oracle exploded')) + + rv = client.get(SNAPSHOT_URL) + + assert rv.status_code == 500 + assert 'oracle exploded' not in str(rv.json) + + +def test_get_snapshot_requires_colin_service_role(client, mocker): + """Assert the endpoint is gated on the colin service role.""" + bypass_auth(mocker, roles_valid=False) + + rv = client.get(SNAPSHOT_URL) + + assert rv.status_code == 401 diff --git a/colin-api/tests/unit/conftest.py b/colin-api/tests/unit/conftest.py index 0d196d4763..6eecd547ae 100644 --- a/colin-api/tests/unit/conftest.py +++ b/colin-api/tests/unit/conftest.py @@ -12,11 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. """Common setup and fixtures for the pytest suite used by this service.""" +from types import SimpleNamespace +from unittest.mock import MagicMock + import pytest from sqlalchemy import event, text from colin_api import create_app from colin_api import jwt as _jwt +from colin_api.models import Business, Office, Party, ShareObject + +from . import build_business, build_director, build_office, build_share_structure, bypass_auth @pytest.fixture(scope='session') @@ -54,6 +60,46 @@ def client_ctx(app): # pylint: disable=redefined-outer-name yield _client +@pytest.fixture +def authorized(mocker): + """Bypass the colin service role check - the gate itself is asserted separately.""" + bypass_auth(mocker) + + +@pytest.fixture +def mock_db(mocker): + """Mock the Oracle connection for the modules that acquire it directly. + + cursor.fetchone defaults to (0,) so count-style lookups (eg. the snapshot's + future-effective filing count) read zero; tests needing a specific row (eg. the + corp password) override the return value. + """ + cursor = MagicMock() + cursor.fetchone.return_value = (0,) + connection = MagicMock() + connection.cursor.return_value = cursor + db = MagicMock() # pylint: disable=invalid-name; mirrors the patched module attribute + db.connection = connection + mocker.patch('colin_api.models.business.DB', db) + mocker.patch('colin_api.models.business_snapshot.DB', db) + return SimpleNamespace(connection=connection, cursor=cursor) + + +@pytest.fixture +def mock_lookups(mocker): + """Stub the model lookups the snapshot composes, with realistic model objects.""" + mocker.patch.object(Business, 'find_by_identifier', return_value=build_business()) + mocker.patch.object(Party, 'get_current', return_value=[build_director()]) + mocker.patch.object( + Office, 'get_current', + # the liquidation office proves out-of-scope office types are dropped + return_value=[build_office('registeredOffice'), build_office('recordsOffice'), + build_office('liquidationOffice')] + ) + mocker.patch.object(ShareObject, 'get_all', return_value=[build_share_structure()]) + mocker.patch.object(Business, 'get_resolutions', return_value=['2020-01-01', '2019-06-15']) + + @pytest.fixture(scope='function') def session(app, db): # pylint: disable=redefined-outer-name, invalid-name """Return a function-scoped session.""" diff --git a/colin-api/tests/unit/models/__init__.py b/colin-api/tests/unit/models/__init__.py new file mode 100644 index 0000000000..2ae0b1ff88 --- /dev/null +++ b/colin-api/tests/unit/models/__init__.py @@ -0,0 +1 @@ +"""Unit tests for colin-api models.""" diff --git a/colin-api/tests/unit/models/test_shares.py b/colin-api/tests/unit/models/test_shares.py new file mode 100644 index 0000000000..bd85533fcf --- /dev/null +++ b/colin-api/tests/unit/models/test_shares.py @@ -0,0 +1,40 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the ShareObject model.""" +from unittest.mock import MagicMock + +from colin_api.models import ShareObject + + +def test_get_share_classes_carries_other_currency(): + """Assert the OTH free-text currency is read and serialized as currencyAdditional.""" + cursor = MagicMock() + cursor.description = [ + ('SHARE_CLASS_ID',), ('CURRENCY_TYP_CD',), ('MAX_SHARE_IND',), ('SHARE_QUANTITY',), + ('SPEC_RIGHTS_IND',), ('PAR_VALUE_IND',), ('PAR_VALUE_AMT',), ('CLASS_NME',), ('OTHER_CURRENCY',) + ] + # one class row, then no series rows for it + cursor.fetchall.side_effect = [ + [(0, 'OTH', 'N', 5000, 'Y', 'Y', 2.0, 'CLASS A', 'BITCOIN')], + [] + ] + + # pylint: disable-next=protected-access + share_classes = ShareObject._get_share_classes(cursor, event_id=1, corp_num='0870226') + + assert len(share_classes) == 1 + assert share_classes[0].other_currency == 'BITCOIN' + assert share_classes[0].to_dict()['currencyAdditional'] == 'BITCOIN' + assert share_classes[0].to_dict()['currency'] == 'OTH' diff --git a/data-tool/.corps.env.sample b/data-tool/.corps.env.sample index 9d18b1af2a..e35dbe5bdd 100644 --- a/data-tool/.corps.env.sample +++ b/data-tool/.corps.env.sample @@ -106,6 +106,11 @@ AUTH_CREATE_ENTITY=True AUTH_UPSERT_CONTACT=False AUTH_CREATE_AFFILIATIONS=False AUTH_SEND_UNAFFILIATED_EMAIL=False +# Both auth_create_flow and auth_invite_flow propagate this setting. +# True sends the optional JSON boolean isReminder: true in the POST body; when +# omitted or false, the body remains {}. No reminder query parameter is used. +# Leave this global flag false for initial-invite auth_create_flow runs. +AUTH_INVITE_IS_REMINDER=False AUTH_FAIL_IF_MISSING_EMAIL=False AUTH_DRY_RUN=False @@ -141,21 +146,86 @@ VERIFY_AUTH_CHECK_INVITE=False VERIFY_AUTH_CONSOLE_LIMIT=25 # Inspect Auth controls. -# Read-only factual Auth DB state inspector. -# Run with: make run-inspect-auth -# Uses the targeting block above. INSPECT_AUTH_FILTER is applied after Auth DB read-back; -# it does not change candidate selection. -# Default ALL writes every inspected candidate to CSV. Non-ALL filters write matched rows only. -# Accepted: ALL,HAS_ANY_AUTH,HAS_ENTITY,MISSING_ENTITY,HAS_CONTACT,ENTITY_WITHOUT_CONTACT, -# HAS_AFFILIATION,ENTITY_WITHOUT_AFFILIATION,HAS_INVITE,ENTITY_WITHOUT_INVITE -# HAS_CONTACT / ENTITY_WITHOUT_CONTACT mean usable contact email, not merely any raw contact row. +# Read-only factual Auth DB state inspector. INSPECT_AUTH_FILTER is applied after +# Auth DB read-back; it does not change candidate selection. HAS_CONTACT means a +# usable contact email, not merely any raw contact row. +# +# Recipe 1 — Initial-state orientation: who never got / never claimed an invite? +# Use ENTITY_WITHOUT_INVITE to find entities with no invite, then HAS_INVITE to +# orient against businesses with invite history. +# INSPECT_AUTH_FILTER=ENTITY_WITHOUT_INVITE +# # Or, for businesses with invite history: INSPECT_AUTH_FILTER=HAS_INVITE +# INSPECT_AUTH_INVITE_EXPIRY_DAYS=7 +# INSPECT_AUTH_INVITE_CRITERIA= +# Run: make run-inspect-auth +# Review the complete AUTH_CORP_NUMS artifact in inspect-auth-summary.txt. For a +# reminder cohort, use a criteria recipe below, paste its handoff block into .env, +# set a new AUTH_REPEATABLE_CYCLE_KEY, then run: make run-auth-invite +# +# Recipe 2 — Reminder 1: exactly one qualifying invite, and it is expired. +# INSPECT_AUTH_FILTER=NO_AFFILIATION_INVITE_CRITERIA +# INSPECT_AUTH_INVITE_EXPIRY_DAYS=30 +# INSPECT_AUTH_INVITE_CRITERIA="count==1, all_expired" +# Equivalent age form at timestamp precision: count==1, age[1]>30. +# After aging by 30 days + 1 second, >30 is true while >31 is false. +# Run: make run-inspect-auth +# Review and paste the handoff block from inspect-auth-summary.txt into .env, set a +# new AUTH_REPEATABLE_CYCLE_KEY, then run: make run-auth-invite +# +# Recipe 3 — Reminder 2 cadence: two invites, oldest >=60d and newest >=30d. +# INSPECT_AUTH_FILTER=NO_AFFILIATION_INVITE_CRITERIA +# INSPECT_AUTH_INVITE_EXPIRY_DAYS=30 +# INSPECT_AUTH_INVITE_CRITERIA="count==2, age[1]>=60, newest_age>=30" +# Run: make run-inspect-auth +# Review and paste the handoff block from inspect-auth-summary.txt into .env, set a +# new AUTH_REPEATABLE_CYCLE_KEY, then run: make run-auth-invite +# +# Reference — filters and criteria. +# Default ALL writes every inspected candidate to CSV; non-ALL filters write only +# matched rows. Accepted filters: ALL,HAS_ANY_AUTH,HAS_ENTITY,MISSING_ENTITY, +# HAS_CONTACT,ENTITY_WITHOUT_CONTACT,HAS_AFFILIATION,ENTITY_WITHOUT_AFFILIATION, +# HAS_INVITE,ENTITY_WITHOUT_INVITE,NO_AFFILIATION_INVITE_CRITERIA. +# The criteria filter also requires an Auth entity, no Auth affiliation, and >=1 +# non-deleted, unclaimed UNAFFILIATED_EMAIL invite across the business. +# Grammar (case-insensitive; whitespace ignored; comma means AND): +# criteria := clause ("," clause)* +# clause := count OP N | all_expired | AGE OP N +# OP := == | >= | <= | > | < +# AGE := age[POSITION] | oldest_age | newest_age | all_ages | any_age +# Age clauses do not support ==. N/POSITION are positive integers. +# Aliases/canonical endpoints: +# oldest_age -> age[1] +# all_ages>=N/>N -> newest_age>=N/>N; all_ages<=N/ age[1]<=N/=N/>N -> age[1]>=N/>N; any_age<=N/ newest_age<=N/=/> age thresholds. +# Unsupported: OR/NOT/parentheses, type/status selectors, bare age[*], and reminder +# ordinals. Auth stores no reminder ordinal; chronological position is only a proxy. +# INSPECT_AUTH_INVITE_CRITERIA is required only for the criteria filter. Expiry days +# controls expiry diagnostics/all_expired, not age clauses; blank is invalid. +# +# Console and artifacts. +# matched = all rows passing the active filter; shown = rows printed after the +# console limit. Confirm the title's "showing X of Y matched" before handoff. +# INSPECT_AUTH_CONSOLE_LIMIT accepts 0 (hide optional console sections), a positive +# integer N (show at most N), or ALL. Console output is a preview; artifacts always +# remain complete: inspect-auth-inspection.csv contains rows and +# inspect-auth-summary.txt contains complete AUTH_CORP_NUMS lists/handoff blocks. +# Legend: uncl=unclaimed (all types); q=qualifying UNAFFILIATED_EMAIL unclaimed; +# exp=sent=cutoff; unk=NULL sent_date; clm=claimed; del=deleted; +# ages=qualifying days oldest→newest; s:=all-type unclaimed sent range; +# a:=latest accepted across all non-deleted invite types. Full detail is in the CSV. INSPECT_AUTH_FILTER=ALL -# Console-only row/identifier preview limit. Accepted: 0, positive integer, ALL. -# 0 suppresses optional console sections; positive values cap matched inspection rows and identifier groups printed to console. -# Complete inspection rows are written to inspect-auth-inspection.csv. -# Complete AUTH_CORP_NUMS copy lists are written to inspect-auth-summary.txt. -# With filter ALL, inspect prints non-empty concrete identifier groups, no ALL mega-list. +INSPECT_AUTH_INVITE_EXPIRY_DAYS=7 +INSPECT_AUTH_INVITE_CRITERIA= INSPECT_AUTH_CONSOLE_LIMIT=25 +# Console-only business_names width: ALL/blank is uncapped; a positive integer truncates with an ellipsis. +INSPECT_AUTH_BUSINESS_NAMES_MAX_LENGTH=ALL # Delete/reset controls. AUTH_DELETE_AFFILIATIONS=False AUTH_DELETE_ENTITY=False diff --git a/data-tool/Makefile b/data-tool/Makefile index a646a1d30a..f63c112b7d 100644 --- a/data-tool/Makefile +++ b/data-tool/Makefile @@ -1,6 +1,7 @@ .PHONY: license .PHONY: setup .PHONY: run +.PHONY: test-delta-restore # Get the absolute path of the Makefile's directory MKFILE_PATH:=$(abspath $(lastword $(MAKEFILE_LIST))) @@ -89,15 +90,6 @@ run-prefect-server: ## Start Prefect server run-prefect-reset-db: ## clears all data and reapplies the schema. Handy for clearing flow run history. . $(PREFECT_VENV_DIR)/bin/activate && prefect server database reset -y -run-corps-migration: ## Run migration flow - @echo "Current directory: $(CURRENT_ABS_DIR)" - @echo "SQLAlchemy path: $(CURRENT_ABS_DIR)/$(SQLALCHEMY_VENV_DIR)/lib/$(PYTHON_VERSION)/site-packages" - . $(VENV_DIR)/bin/activate && \ - export PYTHONPATH="$(CURRENT_ABS_DIR):$$PYTHONPATH" && \ - export SQLALCHEMY_PATH="$(CURRENT_ABS_DIR)/$(SQLALCHEMY_VENV_DIR)/lib/$(PYTHON_VERSION)/site-packages" && \ - FLASK_ENV=development && \ - python flows/migrate_corps_flow.py - run-corps-delete: ## Run delete flow . $(VENV_DIR)/bin/activate && \ python flows/batch_delete_flow.py @@ -164,6 +156,9 @@ run-auth-delete: ## Run auth delete flow . $(VENV_DIR)/bin/activate && \ $(AUTH_MODULE_RUNNER) auth.auth_delete_flow +test-delta-restore: ## Run delta restore static/sql smoke harness + ./tests/delta_restore/run_tests.sh + ################################################################################# # Self Documenting Commands # diff --git a/data-tool/backup_extract_tables.sh b/data-tool/backup_extract_tables.sh index df61bd2aef..dabd3399bc 100755 --- a/data-tool/backup_extract_tables.sh +++ b/data-tool/backup_extract_tables.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# restore_extract.sh +# backup_extract_tables.sh # # Backs up a list of Postgres tables that we want to restore whenever a new COLIN extract is created @@ -14,16 +14,17 @@ PGHOST="${PGHOST:-localhost}" # or ‑‑host PGPORT="${PGPORT:-5432}" # or ‑‑port PGUSER="${PGUSER:-postgres}" # or ‑‑user PGDATABASE="${PGDATABASE:-colin-mig-corps-test}" # or ‑‑dbname +# Optional PostgreSQL client binary directory. Leave empty to use PATH. +# Example: PG_BIN=/path/to/postgresql/bin ./backup_extract_tables.sh +PG_BIN="${PG_BIN:-}" # Supply the password *either* via a .pgpass file *or* one‑shot: # PGPASSWORD=secret ./backup_extract_tables.sh ############################################################################## -# -- Tables to keep ------------------------------------------------------------ -KEEP=(corp_processing auth_processing auth_component_operation colin_tracking mig_group mig_batch mig_corp_batch mig_corp_account corps_with_third_party email_domain_groups bar_corps bad_emails exclude_corps excluded_emails excluded_email_domains excluded_email_domain_patterns) - # -- Runtime options ----------------------------------------------------------- BACKUP_DIR="${BACKUP_DIR:-/backups}" -DUMP="$BACKUP_DIR/keep_$(date +%F).dump" +DUMP="${DUMP:-$BACKUP_DIR/keep_$(date +%F).dump}" +MANIFEST="$DUMP.manifest.json" ############################################################################## # INTERNAL HELPERS @@ -31,6 +32,20 @@ DUMP="$BACKUP_DIR/keep_$(date +%F).dump" die() { printf >&2 "error: %s\n" "$*"; exit 1; } +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PRESERVED_TABLES_CONF="$SCRIPT_DIR/scripts/restore/preserved_tables.conf" + +pg_tool() { + local tool="$1" + if [[ -n "$PG_BIN" ]]; then + printf '%s/%s' "${PG_BIN%/}" "$tool" + else + printf '%s' "$tool" + fi +} + +PG_DUMP_BIN="$(pg_tool pg_dump)" + # Build a single “‑h … ‑p … ‑U …” string so every call is consistent pg_conn_opts() { printf -- "-h %s -p %s -d %s -U %s" "$PGHOST" "$PGPORT" "$PGDATABASE" "$PGUSER" @@ -39,6 +54,75 @@ pg_conn_opts() { # Pass arrays (e.g., KEEP) as repeated --table switches as_table_opts() { local t; for t in "$@"; do printf -- '--table=%s ' "$t"; done; } +load_preserved_tables() { + local table _rest + + [[ -f "$PRESERVED_TABLES_CONF" ]] || die "missing preserved table config: $PRESERVED_TABLES_CONF" + + KEEP=() + while read -r table _rest; do + case "${table:-}" in + ''|'#'*) continue ;; + *) KEEP+=("$table") ;; + esac + done < "$PRESERVED_TABLES_CONF" + + [[ "${#KEEP[@]}" -gt 0 ]] || die "no preserved tables found in $PRESERVED_TABLES_CONF" +} + +sha256_file() { + local path="$1" + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$path" | awk '{print $1}' + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$path" | awk '{print $1}' + else + die "neither sha256sum nor shasum is available to create the manifest" + fi +} + +json_escape() { + local value="$1" + value=${value//\\/\\\\} + value=${value//\"/\\\"} + value=${value//$'\n'/\\n} + value=${value//$'\r'/\\r} + printf '%s' "$value" +} + +write_manifest() { + local sha256="$1" + local created_at pg_dump_version i + created_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + pg_dump_version="$("$PG_DUMP_BIN" --version)" + + { + printf '{\n' + printf ' "created_at": "%s",\n' "$(json_escape "$created_at")" + printf ' "created_by_script": "data-tool/backup_extract_tables.sh",\n' + printf ' "dump_path": "%s",\n' "$(json_escape "$DUMP")" + printf ' "source": {\n' + printf ' "host": "%s",\n' "$(json_escape "$PGHOST")" + printf ' "port": "%s",\n' "$(json_escape "$PGPORT")" + printf ' "database": "%s",\n' "$(json_escape "$PGDATABASE")" + printf ' "user": "%s"\n' "$(json_escape "$PGUSER")" + printf ' },\n' + printf ' "tables": [\n' + for i in "${!KEEP[@]}"; do + if [[ "$i" -gt 0 ]]; then + printf ',\n' + fi + printf ' "%s"' "$(json_escape "${KEEP[$i]}")" + done + printf '\n ],\n' + printf ' "dump_sha256": "%s",\n' "$(json_escape "$sha256")" + printf ' "pg_dump_version": "%s"\n' "$(json_escape "$pg_dump_version")" + printf '}\n' + } > "$MANIFEST" +} + +load_preserved_tables + ############################################################################## # BACK UP THE TABLES # ############################################################################## @@ -46,8 +130,11 @@ as_table_opts() { local t; for t in "$@"; do printf -- '--table=%s ' "$t"; done; printf "📦 Dumping preserved tables …\n" mkdir -p "$BACKUP_DIR" -pg_dump $(pg_conn_opts) -Fc \ +"$PG_DUMP_BIN" $(pg_conn_opts) -Fc \ $(as_table_opts "${KEEP[@]}") \ --no-owner --no-acl \ -f "$DUMP" +printf "🧾 Writing manifest …\n" +write_manifest "$(sha256_file "$DUMP")" +printf "✅ Wrote %s and %s\n" "$DUMP" "$MANIFEST" diff --git a/data-tool/delta_restore_extract.sh b/data-tool/delta_restore_extract.sh new file mode 100755 index 0000000000..04bb05a2a3 --- /dev/null +++ b/data-tool/delta_restore_extract.sh @@ -0,0 +1,1650 @@ +#!/usr/bin/env bash +# delta_restore_extract.sh +# +# Delta restore pipeline for preserved-table preview/apply runs. +# Stages a preserved-table dump, classifies rows, writes preview artifacts, and +# optionally applies selected NEW/CHANGED rows. + +set -euo pipefail +umask 077 + +############################################################################## +# CONNECTION DEFAULTS +############################################################################## + +PGHOST="${PGHOST:-localhost}" +PGPORT="${PGPORT:-5432}" +PGUSER="${PGUSER:-postgres}" +PGDATABASE="${PGDATABASE:-colin-mig-corps-test}" +# Optional PostgreSQL client binary directory. Leave empty to use PATH. +# Example: PG_BIN=/path/to/postgresql/bin ./delta_restore_extract.sh +PG_BIN="${PG_BIN:-}" + +############################################################################## +# PATHS / DEFAULTS +############################################################################## + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PRESERVED_TABLES_CONF="$SCRIPT_DIR/scripts/restore/preserved_tables.conf" +DELTA_DIR="$SCRIPT_DIR/scripts/restore/delta" +INSTALL_SQL="$DELTA_DIR/00_install.sql" +FUNCTIONS_SQL="$DELTA_DIR/10_functions.sql" +SESSION_BEGIN_SQL="$DELTA_DIR/20_session_begin.sql" +SESSION_END_SQL="$DELTA_DIR/21_session_end.sql" +REWRITE_AWK="$DELTA_DIR/rewrite_copy_targets.awk" +ALIGN_DETAILS_AWK="$DELTA_DIR/align_details.awk" +REPAIR_SEQUENCES_SQL="$SCRIPT_DIR/scripts/restore/repair_sequences.sql" +ACQUIRE_LOCK_SQL="$SCRIPT_DIR/scripts/subset/subset_pg_acquire_advisory_lock.sql" +RELEASE_LOCK_SQL="$SCRIPT_DIR/scripts/subset/subset_pg_release_advisory_lock.sql" +DEFAULT_REPORT_BASE="$SCRIPT_DIR/scripts/generated/delta_restore" + +DUMP="${DUMP:-}" +MODE="preview" +TABLES_ARG="all" +INCLUDE_CLASSES="" +EXCLUDE_CLASSES="" +SELECTION_FILE="" +ONLY_CORPS_FILE="" +SAMPLE_SIZE="20" +DETAILS_LIMIT="10000" +SELECTOR_SUGGESTION_LIMIT="50" +SELECTOR_SUGGESTION_LIMIT_MAX="100000" +ALIGN_DETAILS="true" +ALIGN_WIDTH="40" +MANIFEST_DEFAULT="new,changed" +DUMP_SHA256="" +REPORT_BASE="$DEFAULT_REPORT_BASE" +KEEP_ARTIFACTS="false" +YES="false" +CLEANUP="false" +LOCK_TIMEOUT_SECONDS="${LOCK_TIMEOUT_SECONDS:-30}" + +RUN_DIR="" +LOCK_FIFO="" +LOCK_PID="" +LOCK_ACQUIRED="false" + +############################################################################## +# HELPERS +############################################################################## + +die_with_code() { + local code="$1" + shift + printf >&2 "error: %s\n" "$*" + exit "$code" +} + +die() { die_with_code 1 "$@"; } +die2() { die_with_code 2 "$@"; } +die3() { die_with_code 3 "$@"; } +die4() { die_with_code 4 "$@"; } +die5() { die_with_code 5 "$@"; } + +usage() { + cat <<'USAGE' +Usage: + delta_restore_extract.sh --dump [--mode preview|apply|validate] [options] + delta_restore_extract.sh --cleanup + +Options parsed now: + --dump pg_dump -Fc archive to stage (or DUMP env var) + --mode preview|apply|validate default: preview; apply requires --yes or confirmation + --tables all| hard selection scope; staging/classification still use all preserved tables + --include-classes when supplied, limit selection to these classes + --exclude-classes remove classes after selection-file/include processing + --selection-file generated/edited candidates, bounded by explicit table/class scope + --only-corps restrict selected corp-bearing rows to listed corp identifiers + --sample-size preview sample limit; default: 20 + --details-limit detail rows per table/class; default: 10000 + --selector-suggestion-limit + exact manifest suggestions per table/class; default: 50 + 0 disables exact suggestions; maximum: 100000 + --no-aligned-details do not create fixed-width .txt detail companions + --align-width aligned detail cell width; default: 40, minimum: 6 + --manifest-default + generated selection default; default: new,changed + --report-dir base directory for run artifacts + --keep-artifacts retain delta schemas after successful apply + --yes skip interactive apply confirmation + --cleanup drop delta_stage/delta_map/delta_diff/delta_ctl and exit + -h, --help show this help + +Exit codes reserved by the plan: + 0 ok · 2 preflight/drift/corrupt dump · 3 advisory lock busy · 4 selection invalid · 5 apply verification failed + +Environment: + PGHOST, PGPORT, PGUSER, PGDATABASE, PGPASSWORD/.pgpass libpq connection settings + PG_BIN=/path/to/postgresql/bin optional PostgreSQL client-tools directory; unset or empty uses PATH + DUMP= alternative to --dump + LOCK_TIMEOUT_SECONDS=30 advisory-lock wait before exit 3 + +After run-directory initialization, each invocation prints its artifact directory on exit: + Artifacts: (success) + Artifacts retained for inspection: (failure) +USAGE +} + +pg_tool() { + local tool="$1" + if [[ -n "$PG_BIN" ]]; then + printf '%s/%s' "${PG_BIN%/}" "$tool" + else + printf '%s' "$tool" + fi +} + +PSQL_BIN="$(pg_tool psql)" +PG_RESTORE_BIN="$(pg_tool pg_restore)" + +pg_conn_opts() { + printf -- "-h %s -p %s -d %s -U %s" "$PGHOST" "$PGPORT" "$PGDATABASE" "$PGUSER" +} + +is_identifier() { + local val="$1" + case "$val" in + ''|*[!a-z0-9_]*|[0-9]*) return 1 ;; + *) return 0 ;; + esac +} + +table_in_file() { + local needle="$1" file="$2" + grep -qx -- "$needle" "$file" +} + +sql_literal() { + local value="$1" + printf "'%s'" "$(printf '%s' "$value" | sed "s/'/''/g")" +} + +sha256_file() { + local path="$1" + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$path" | awk '{print $1}' + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$path" | awk '{print $1}' + else + die2 "neither sha256sum nor shasum is available to verify the dump" + fi +} + +psql_file() { + local path="$1" + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 -f "$path" +} + +psql_cmd() { + local sql="$1" + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 -c "$sql" +} + +psql_delta_session_file() { + local payload="$1" stdout_path="$2" stderr_path="$3" + shift 3 + + if [[ "$stderr_path" = "-" ]]; then + "$PSQL_BIN" -X -q $(pg_conn_opts) -v ON_ERROR_STOP=1 "$@" \ + -f "$SESSION_BEGIN_SQL" -f "$payload" -f "$SESSION_END_SQL" > "$stdout_path" + else + local status + if "$PSQL_BIN" -X -q $(pg_conn_opts) -v ON_ERROR_STOP=1 "$@" \ + -f "$SESSION_BEGIN_SQL" -f "$payload" -f "$SESSION_END_SQL" \ + > "$stdout_path" 2> "$stderr_path"; then + status=0 + else + status=$? + fi + cat "$stderr_path" >&2 + return "$status" + fi +} + +record_temp_stats_snapshot() { + local snapshot="$1" + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 -tA -F $'\t' >> "$RUN_DIR/temp_stats.tsv" < SELECTOR_SUGGESTION_LIMIT_MAX )); then + die "--selector-suggestion-limit must be a decimal integer from 0 to $SELECTOR_SUGGESTION_LIMIT_MAX" + fi + ;; + esac + case "$ALIGN_WIDTH" in + ''|*[!0-9]*) die "--align-width must be a positive integer" ;; + *) + while [[ "$ALIGN_WIDTH" = 0* && "$ALIGN_WIDTH" != "0" ]]; do + ALIGN_WIDTH="${ALIGN_WIDTH#0}" + done + case "$ALIGN_WIDTH" in + 0) die "--align-width must be a positive integer" ;; + [1-5]) ALIGN_WIDTH="6" ;; + esac + ;; + esac +} + +validate_static_files() { + [[ -f "$PRESERVED_TABLES_CONF" ]] || die2 "missing preserved table config: $PRESERVED_TABLES_CONF" + [[ -f "$INSTALL_SQL" ]] || die2 "missing install SQL: $INSTALL_SQL" + [[ -f "$FUNCTIONS_SQL" ]] || die2 "missing classification functions SQL: $FUNCTIONS_SQL" + [[ -f "$SESSION_BEGIN_SQL" ]] || die2 "missing session begin SQL: $SESSION_BEGIN_SQL" + [[ -f "$SESSION_END_SQL" ]] || die2 "missing session end SQL: $SESSION_END_SQL" + [[ -f "$REWRITE_AWK" ]] || die2 "missing COPY rewrite awk: $REWRITE_AWK" + [[ -f "$ALIGN_DETAILS_AWK" ]] || die2 "missing detail alignment awk: $ALIGN_DETAILS_AWK" + [[ -f "$REPAIR_SEQUENCES_SQL" ]] || die2 "missing repair sequences SQL: $REPAIR_SEQUENCES_SQL" + [[ -f "$ACQUIRE_LOCK_SQL" ]] || die2 "missing advisory lock acquire SQL: $ACQUIRE_LOCK_SQL" + [[ -f "$RELEASE_LOCK_SQL" ]] || die2 "missing advisory lock release SQL: $RELEASE_LOCK_SQL" +} + +init_run_dir() { + local ts + ts="$(date -u +%Y%m%d_%H%M%S)" + RUN_DIR="$REPORT_BASE/$ts" + if [[ -e "$RUN_DIR" ]]; then + RUN_DIR="$REPORT_BASE/${ts}_$$" + fi + mkdir -p "$RUN_DIR" +} + +release_lock() { + if [[ "$LOCK_ACQUIRED" = "true" ]]; then + printf "\\i %s\n\\q\n" "$RELEASE_LOCK_SQL" >&3 2>/dev/null || true + exec 3>&- 2>/dev/null || true + wait "$LOCK_PID" >/dev/null 2>&1 || true + LOCK_ACQUIRED="false" + elif [[ -n "${LOCK_PID:-}" ]] && kill -0 "$LOCK_PID" >/dev/null 2>&1; then + kill "$LOCK_PID" >/dev/null 2>&1 || true + wait "$LOCK_PID" >/dev/null 2>&1 || true + fi +} + +on_exit() { + local status=$? + release_lock + if [[ -n "$RUN_DIR" ]]; then + if [[ "$status" -eq 0 ]]; then + printf "Artifacts: %s\n" "$RUN_DIR" + else + printf >&2 "Artifacts retained for inspection: %s\n" "$RUN_DIR" + fi + fi +} + +acquire_lock() { + LOCK_FIFO="$RUN_DIR/advisory_lock.fifo" + rm -f "$LOCK_FIFO" + mkfifo "$LOCK_FIFO" + + "$PSQL_BIN" -X $(pg_conn_opts) -q -v ON_ERROR_STOP=1 \ + > "$RUN_DIR/advisory_lock.out" \ + 2> "$RUN_DIR/advisory_lock.err" \ + < "$LOCK_FIFO" & + LOCK_PID=$! + + exec 3>"$LOCK_FIFO" + rm -f "$RUN_DIR/advisory_lock.acquired" + printf "\\i %s\n\\! touch %s\n" "$ACQUIRE_LOCK_SQL" "$RUN_DIR/advisory_lock.acquired" >&3 + + local waited=0 + while [[ "$waited" -lt "$LOCK_TIMEOUT_SECONDS" ]]; do + if [[ -f "$RUN_DIR/advisory_lock.acquired" ]]; then + LOCK_ACQUIRED="true" + printf "🔒 Advisory lock acquired.\n" + return 0 + fi + if ! kill -0 "$LOCK_PID" >/dev/null 2>&1; then + cat "$RUN_DIR/advisory_lock.err" >&2 2>/dev/null || true + die3 "failed to acquire advisory lock" + fi + sleep 1 + waited=$((waited + 1)) + done + + cat "$RUN_DIR/advisory_lock.err" >&2 2>/dev/null || true + die3 "timed out waiting for the subset/full-refresh advisory lock after ${LOCK_TIMEOUT_SECONDS}s" +} + +load_preserved_config() { + local table phase _rest + : > "$RUN_DIR/preserved_tables.tsv" + : > "$RUN_DIR/all_conf_tables.txt" + + while read -r table phase _rest; do + case "${table:-}" in + ''|'#'*) continue ;; + *) ;; + esac + is_identifier "$table" || die2 "invalid table name in $PRESERVED_TABLES_CONF: $table" + case "${phase:-}" in + ''|*[!0-9]*) die2 "invalid load phase for $table in $PRESERVED_TABLES_CONF: ${phase:-}" ;; + *) ;; + esac + printf "%s\t%s\n" "$table" "$phase" >> "$RUN_DIR/preserved_tables.tsv" + printf "%s\n" "$table" >> "$RUN_DIR/all_conf_tables.txt" + done < "$PRESERVED_TABLES_CONF" + + [[ -s "$RUN_DIR/preserved_tables.tsv" ]] || die2 "no preserved tables found in $PRESERVED_TABLES_CONF" +} + +select_tables() { + # Preview classification stages the full preserved set so parent ID maps exist + # even when a future selection narrows to --tables. Keep the requested list as + # an artifact for later selection/reporting work, but do not starve staging. + cp "$RUN_DIR/all_conf_tables.txt" "$RUN_DIR/selected_tables.txt" + : > "$RUN_DIR/requested_tables.txt" + + if [[ "$TABLES_ARG" = "all" ]]; then + cp "$RUN_DIR/all_conf_tables.txt" "$RUN_DIR/requested_tables.txt" + return 0 + fi + + local old_ifs item + old_ifs="$IFS" + IFS=',' + # shellcheck disable=SC2206 + set -- $TABLES_ARG + IFS="$old_ifs" + + [[ "$#" -gt 0 ]] || die2 "--tables did not include any table names" + for item in "$@"; do + item="$(printf '%s' "$item" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//')" + is_identifier "$item" || die2 "invalid --tables entry: $item" + table_in_file "$item" "$RUN_DIR/all_conf_tables.txt" || die2 "--tables entry is not in preserved_tables.conf: $item" + if ! table_in_file "$item" "$RUN_DIR/requested_tables.txt"; then + printf "%s\n" "$item" >> "$RUN_DIR/requested_tables.txt" + fi + done +} + +seed_table_config() { + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 \ + -c "COPY delta_ctl.table_config(table_name, load_phase) FROM STDIN" \ + < "$RUN_DIR/preserved_tables.tsv" + + cat > "$RUN_DIR/seed_table_config_metadata.sql" <<'SQL' +UPDATE delta_ctl.table_config SET pk_col = 'id' WHERE table_name IN ( + 'demigrated_filings', 'bad_emails', 'excluded_email_domain_patterns', 'mig_group', 'mig_batch', + 'mig_corp_batch', 'mig_corp_account', 'corp_processing', 'colin_tracking', + 'auth_processing', 'auth_component_operation' +); +UPDATE delta_ctl.table_config SET has_last_modified = true WHERE table_name IN ( + 'corp_processing', 'colin_tracking', 'auth_processing' +); +UPDATE delta_ctl.table_config SET match_mode = 'hash_parent' WHERE table_name = 'auth_component_operation'; +UPDATE delta_ctl.table_config SET nk_enforced = true WHERE table_name IN ( + 'demigrated_filings', 'bad_emails', 'excluded_emails', 'excluded_email_domains', + 'excluded_email_domain_patterns', 'corp_processing', 'colin_tracking', + 'auth_processing' +); +UPDATE delta_ctl.table_config SET fk_map = '{"mig_group_id":"mig_group"}'::jsonb WHERE table_name = 'mig_batch'; +UPDATE delta_ctl.table_config SET fk_map = '{"mig_batch_id":"mig_batch"}'::jsonb WHERE table_name IN ( + 'mig_corp_batch', 'mig_corp_account' +); +UPDATE delta_ctl.table_config SET fk_map = '{"mig_batch_id":"mig_batch","corp_num":"external:corporation.corp_num","last_processed_event_id":"external:event.event_id","failed_event_id":"external:event.event_id"}'::jsonb WHERE table_name = 'corp_processing'; +UPDATE delta_ctl.table_config SET fk_map = '{"mig_batch_id":"mig_batch","corp_num":"external:corporation.corp_num"}'::jsonb WHERE table_name IN ( + 'colin_tracking', 'auth_processing' +); +UPDATE delta_ctl.table_config SET fk_map = '{"auth_processing_id":"auth_processing"}'::jsonb WHERE table_name = 'auth_component_operation'; +SQL + psql_file "$RUN_DIR/seed_table_config_metadata.sql" +} + +record_metadata() { + cat > "$RUN_DIR/metadata.sql" < "$RUN_DIR/dump.sha256" +} + +verify_dump_and_manifest() { + [[ -n "$DUMP" ]] || die2 "--dump is required for preview/apply/validate mode" + [[ -r "$DUMP" ]] || die2 "dump is not readable: $DUMP" + + printf "🔎 Reading dump TOC …\n" + "$PG_RESTORE_BIN" -l "$DUMP" > "$RUN_DIR/toc.list" || die2 "pg_restore could not read dump: $DUMP" + + local manifest expected_sha actual_sha + compute_dump_sha + manifest="$DUMP.manifest.json" + if [[ -f "$manifest" ]]; then + cp "$manifest" "$RUN_DIR/dump.manifest.json" + expected_sha="$(sed -n 's/.*"dump_sha256"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$manifest" | head -n 1)" + if [[ -n "$expected_sha" ]]; then + actual_sha="$DUMP_SHA256" + if [[ "$expected_sha" != "$actual_sha" ]]; then + die2 "dump sha256 does not match manifest: expected $expected_sha, got $actual_sha" + fi + printf "🧾 Manifest sha256 verified.\n" + else + printf "⚠️ Manifest found but dump_sha256 was not present: %s\n" "$manifest" | tee -a "$RUN_DIR/warnings.log" + fi + fi +} + +filter_toc() { + : > "$RUN_DIR/toc_data.list" + : > "$RUN_DIR/present_tables.txt" + : > "$RUN_DIR/absent_tables.txt" + : > "$RUN_DIR/unexpected_dump_tables.txt" + + awk -v selected="$RUN_DIR/selected_tables.txt" \ + -v all_conf="$RUN_DIR/all_conf_tables.txt" \ + -v present="$RUN_DIR/present_tables.txt" \ + -v unexpected="$RUN_DIR/unexpected_dump_tables.txt" ' + BEGIN { + while ((getline t < selected) > 0) selected_table[t] = 1 + close(selected) + while ((getline t < all_conf) > 0) conf_table[t] = 1 + close(all_conf) + } + $0 ~ / TABLE DATA public / { + t = $0 + sub(/^.* TABLE DATA public /, "", t) + sub(/ .*/, "", t) + if (selected_table[t]) { + print $0 + if (!seen[t]++) print t >> present + } else if (!conf_table[t]) { + if (!warned[t]++) print t >> unexpected + } + } + ' "$RUN_DIR/toc.list" > "$RUN_DIR/toc_data.list" + + while read -r table; do + [[ -n "$table" ]] || continue + if ! table_in_file "$table" "$RUN_DIR/present_tables.txt"; then + printf "%s\n" "$table" >> "$RUN_DIR/absent_tables.txt" + fi + done < "$RUN_DIR/selected_tables.txt" + + if [[ -s "$RUN_DIR/unexpected_dump_tables.txt" ]]; then + printf "⚠️ Dump contains table data outside preserved_tables.conf; excluding:\n" | tee -a "$RUN_DIR/warnings.log" + sed 's/^/ - /' "$RUN_DIR/unexpected_dump_tables.txt" | tee -a "$RUN_DIR/warnings.log" + fi + if [[ -s "$RUN_DIR/absent_tables.txt" ]]; then + printf "ℹ️ Preserved table(s) absent from dump; marking SKIPPED_ABSENT:\n" | tee -a "$RUN_DIR/warnings.log" + sed 's/^/ - /' "$RUN_DIR/absent_tables.txt" | tee -a "$RUN_DIR/warnings.log" + fi +} + +install_control_schemas() { + printf "🏗 Installing delta control schemas …\n" + psql_file "$INSTALL_SQL" + psql_file "$FUNCTIONS_SQL" + seed_table_config + record_metadata +} + +create_stage_shells() { + : > "$RUN_DIR/create_stage_shells.sql" + + if [[ ! -s "$RUN_DIR/present_tables.txt" ]]; then + printf "ℹ️ No selected preserved table data was present in the dump; skipping stage shell creation.\n" + return 0 + fi + + cat > "$RUN_DIR/create_stage_shells.sql" <<'SQL' +DO $$ +DECLARE + missing text[]; +BEGIN + SELECT array_agg(v.table_name ORDER BY v.table_name) + INTO missing + FROM (VALUES +SQL + + local first="true" table + while read -r table; do + [[ -n "$table" ]] || continue + if [[ "$first" = "true" ]]; then + first="false" + else + printf ",\n" >> "$RUN_DIR/create_stage_shells.sql" + fi + printf " (%s)" "$(sql_literal "$table")" >> "$RUN_DIR/create_stage_shells.sql" + done < "$RUN_DIR/present_tables.txt" + + cat >> "$RUN_DIR/create_stage_shells.sql" <<'SQL' + ) AS v(table_name) + WHERE to_regclass(format('public.%I', v.table_name)) IS NULL; + + IF missing IS NOT NULL THEN + RAISE EXCEPTION 'local preserved table(s) missing: %. Apply the latest colin_corps_extract_postgres_ddl first.', array_to_string(missing, ', '); + END IF; +END; +$$; +SQL + + while read -r table; do + [[ -n "$table" ]] || continue + cat >> "$RUN_DIR/create_stage_shells.sql" < "$RUN_DIR/dump_columns_rows.tsv" + if [[ -f "$RUN_DIR/dump_columns.tsv" ]]; then + awk -F '\t' ' + NF >= 2 { + n = split($2, cols, ",") + for (i = 1; i <= n; i++) { + gsub(/^[[:space:]]+|[[:space:]]+$/, "", cols[i]) + if (cols[i] != "") print $1 "\t" cols[i] "\t" i + } + } + ' "$RUN_DIR/dump_columns.tsv" > "$RUN_DIR/dump_columns_rows.tsv" + fi + + psql_cmd "TRUNCATE delta_ctl.dump_columns" + if [[ -s "$RUN_DIR/dump_columns_rows.tsv" ]]; then + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 \ + -c "COPY delta_ctl.dump_columns(table_name, column_name, ordinal) FROM STDIN" \ + < "$RUN_DIR/dump_columns_rows.tsv" + fi +} + +run_schema_drift_preflight() { + load_dump_columns_sidecar + + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 -tA -F $'\t' > "$RUN_DIR/drift_dump_only.tsv" <<'SQL' +SELECT d.table_name, d.column_name +FROM delta_ctl.dump_columns d +LEFT JOIN information_schema.columns c + ON c.table_schema = 'public' + AND c.table_name = d.table_name + AND c.column_name = d.column_name +WHERE c.column_name IS NULL +ORDER BY d.table_name, d.ordinal; +SQL + + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 -tA -F $'\t' > "$RUN_DIR/drift_local_required.tsv" <<'SQL' +SELECT c.table_name, c.column_name +FROM information_schema.columns c +JOIN (SELECT DISTINCT table_name FROM delta_ctl.dump_columns) d0 ON d0.table_name = c.table_name +LEFT JOIN delta_ctl.dump_columns d + ON d.table_name = c.table_name + AND d.column_name = c.column_name +WHERE c.table_schema = 'public' + AND d.column_name IS NULL + AND c.is_nullable = 'NO' + AND c.column_default IS NULL + AND c.identity_generation IS NULL +ORDER BY c.table_name, c.ordinal_position; +SQL + + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 -tA -F $'\t' > "$RUN_DIR/drift_local_copy_permitted.tsv" <<'SQL' +SELECT c.table_name, c.column_name, + CASE + WHEN c.column_default IS NOT NULL THEN 'local-only column has a default; staged rows used local default' + WHEN c.identity_generation IS NOT NULL THEN 'local-only identity column; staged rows used local identity/default behavior' + ELSE 'local-only nullable column; staged rows used NULL' + END AS warning +FROM information_schema.columns c +JOIN (SELECT DISTINCT table_name FROM delta_ctl.dump_columns) d0 ON d0.table_name = c.table_name +LEFT JOIN delta_ctl.dump_columns d + ON d.table_name = c.table_name + AND d.column_name = c.column_name +WHERE c.table_schema = 'public' + AND d.column_name IS NULL + AND NOT (c.is_nullable = 'NO' AND c.column_default IS NULL AND c.identity_generation IS NULL) +ORDER BY c.table_name, c.ordinal_position; +SQL + + if [[ -s "$RUN_DIR/drift_dump_only.tsv" ]]; then + printf >&2 "Dump contains column(s) not present in local DDL. Apply latest colin_corps_extract_postgres_ddl first:\n" + sed 's/^/ /' "$RUN_DIR/drift_dump_only.tsv" >&2 + return 2 + fi + + if [[ -s "$RUN_DIR/drift_local_required.tsv" ]]; then + printf >&2 "Local DDL has required column(s) missing from the dump. Regenerate the preserved-table dump:\n" + sed 's/^/ /' "$RUN_DIR/drift_local_required.tsv" >&2 + return 2 + fi + + if [[ -s "$RUN_DIR/drift_local_copy_permitted.tsv" ]]; then + printf "⚠️ Local-only copy-permitted column(s) detected; adding to classify_ignore_cols for this run:\n" | tee -a "$RUN_DIR/warnings.log" + sed 's/^/ /' "$RUN_DIR/drift_local_copy_permitted.tsv" | tee -a "$RUN_DIR/warnings.log" + + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 <<'SQL' +WITH local_only AS ( + SELECT c.table_name, c.column_name, + CASE + WHEN c.column_default IS NOT NULL THEN 'local-only column has a default; staged rows used local default' + WHEN c.identity_generation IS NOT NULL THEN 'local-only identity column; staged rows used local identity/default behavior' + ELSE 'local-only nullable column; staged rows used NULL' + END AS warning + FROM information_schema.columns c + JOIN (SELECT DISTINCT table_name FROM delta_ctl.dump_columns) d0 ON d0.table_name = c.table_name + LEFT JOIN delta_ctl.dump_columns d + ON d.table_name = c.table_name + AND d.column_name = c.column_name + WHERE c.table_schema = 'public' + AND d.column_name IS NULL + AND NOT (c.is_nullable = 'NO' AND c.column_default IS NULL AND c.identity_generation IS NULL) +), grouped AS ( + SELECT table_name, array_agg(column_name ORDER BY column_name) AS cols + FROM local_only + GROUP BY table_name +) +INSERT INTO delta_ctl.drift_warnings(table_name, column_name, warning) +SELECT table_name, column_name, warning FROM local_only; + +UPDATE delta_ctl.table_config t +SET classify_ignore_cols = ( + SELECT ARRAY( + SELECT DISTINCT unnest(t.classify_ignore_cols || g.cols) + ORDER BY 1 + ) +) +FROM grouped g +WHERE g.table_name = t.table_name; +SQL + fi + + return 0 +} + +add_stage_index() { + local table="$1" index_name="$2" columns="$3" + if table_in_file "$table" "$RUN_DIR/present_tables.txt"; then + printf 'CREATE INDEX IF NOT EXISTS %s ON delta_stage.%s (%s);\n' "$index_name" "$table" "$columns" >> "$RUN_DIR/create_stage_indexes.sql" + fi +} + +create_stage_indexes() { + : > "$RUN_DIR/create_stage_indexes.sql" + + local table + while read -r table; do + [[ -n "$table" ]] || continue + add_stage_index "$table" "idx_delta_stage_${table}_drid" "_delta_row_id" + done < "$RUN_DIR/present_tables.txt" + + add_stage_index demigrated_filings idx_delta_stage_demigrated_filings_nk "filing_id" + add_stage_index email_domain_groups idx_delta_stage_email_domain_groups_nk "email_domain" + add_stage_index bad_emails idx_delta_stage_bad_emails_nk "lower(btrim(email))" + add_stage_index excluded_emails idx_delta_stage_excluded_emails_nk "lower(btrim(email))" + add_stage_index excluded_email_domains idx_delta_stage_excluded_email_domains_nk "lower(btrim(email_domain))" + add_stage_index excluded_email_domain_patterns idx_delta_stage_excluded_email_domain_patterns_nk "lower(btrim(email_domain)), lower(btrim(local_part_pattern))" + add_stage_index exclude_corps idx_delta_stage_exclude_corps_nk "corp_num" + add_stage_index corps_with_third_party idx_delta_stage_corps_with_third_party_nk "corp_num, vendor" + add_stage_index bar_corps idx_delta_stage_bar_corps_nk "identifier" + add_stage_index mig_group idx_delta_stage_mig_group_nk "name, target_environment, source_db" + add_stage_index mig_batch idx_delta_stage_mig_batch_nk "mig_group_id, name, target_environment" + add_stage_index mig_corp_batch idx_delta_stage_mig_corp_batch_nk "mig_batch_id, corp_num" + add_stage_index mig_corp_account idx_delta_stage_mig_corp_account_nk "corp_num, target_environment, account_id, mig_batch_id" + add_stage_index corp_processing idx_delta_stage_corp_processing_nk "corp_num, flow_name, environment" + add_stage_index colin_tracking idx_delta_stage_colin_tracking_nk "corp_num, flow_name, environment" + add_stage_index auth_processing idx_delta_stage_auth_processing_nk "corp_num, flow_name, environment, operation, operation_scope, attempt_key" + add_stage_index auth_component_operation idx_delta_stage_auth_component_operation_parent "auth_processing_id" + + if [[ -s "$RUN_DIR/create_stage_indexes.sql" ]]; then + printf "⚙️ Creating delta_stage classification indexes …\n" + psql_file "$RUN_DIR/create_stage_indexes.sql" + fi +} + +stream_stage_data() { + if [[ ! -s "$RUN_DIR/toc_data.list" ]]; then + printf "ℹ️ No TABLE DATA entries selected for staging.\n" + return 0 + fi + + cp "$RUN_DIR/present_tables.txt" "$RUN_DIR/expected_tables.txt" + printf "🚚 Streaming dump data into delta_stage …\n" + + if ! { + set +e + "$PG_RESTORE_BIN" --data-only -L "$RUN_DIR/toc_data.list" -f - "$DUMP" \ + | awk -f "$REWRITE_AWK" -v sidecar="$RUN_DIR/dump_columns.tsv" -v expected="$RUN_DIR/expected_tables.txt" + pipeline_status=("${PIPESTATUS[@]}") + pg_restore_status=${pipeline_status[0]} + awk_status=${pipeline_status[1]} + set -e + + if [[ "$pg_restore_status" -eq 0 && "$awk_status" -eq 0 ]]; then + printf "SELECT 1 AS delta_stream_complete;\n" + exit 0 + fi + + printf >&2 "staging producer failed: pg_restore=%s awk=%s\n" "$pg_restore_status" "$awk_status" + printf "SELECT 1/0 AS delta_stream_failed;\n" + exit 2 + } | "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 --single-transaction -q \ + -f "$SESSION_BEGIN_SQL" -f - -f "$SESSION_END_SQL"; then + printf >&2 "Staging stream failed; running sidecar drift diagnostics where possible.\n" + run_schema_drift_preflight || true + die2 "failed to stream dump data into delta_stage" + fi + + run_schema_drift_preflight || die2 "schema drift preflight failed" +} + +record_stage_counts() { + : > "$RUN_DIR/stage_counts.sql" + + while read -r table; do + [[ -n "$table" ]] || continue + cat >> "$RUN_DIR/stage_counts.sql" <> "$RUN_DIR/stage_counts.sql" < "$RUN_DIR/stage_counts.tsv" <<'SQL' +SELECT table_name, count_name, row_count +FROM delta_ctl.run_counts +WHERE count_name IN ('STAGED', 'SKIPPED_ABSENT') +ORDER BY table_name, count_name; +SQL +} + +run_preview_classification() { + printf "🧮 Classifying staged rows …\n" + cat > "$RUN_DIR/preview_classification.sql" <<'SQL' +SELECT delta_ctl.run_preview_classification(); +SQL + printf "captured_at\tsnapshot\tdatname\ttemp_files\ttemp_bytes\ttemp_bytes_pretty\n" > "$RUN_DIR/temp_stats.tsv" + record_temp_stats_snapshot "before_classification" + + set +e + psql_delta_session_file "$RUN_DIR/preview_classification.sql" \ + "$RUN_DIR/classification.out" "$RUN_DIR/classification.err" + local classification_status=$? + set -e + + record_temp_stats_snapshot "after_classification" || true + [[ "$classification_status" -eq 0 ]] || return "$classification_status" + + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 -tA -F $'\t' > "$RUN_DIR/class_counts.tsv" <<'SQL' +SELECT table_name, count_name, row_count, load_phase +FROM delta_ctl.fn_counts(); +SQL +} + +write_selection_manifest() { + printf "📝 Writing default selection manifest …\n" + cat > "$RUN_DIR/render_selection_manifest.sql" <<'SQL' +SELECT * FROM delta_ctl.render_selection_manifest(); +SQL + psql_delta_session_file "$RUN_DIR/render_selection_manifest.sql" \ + "$RUN_DIR/selection.conf" "$RUN_DIR/selection_manifest.err" -tA +} + +write_selection_cookbook() { + printf "📖 Writing selection cookbook …\n" + cat > "$RUN_DIR/render_selection_cookbook.sql" <<'SQL' +SELECT * FROM delta_ctl.render_selection_cookbook(); +SQL + psql_delta_session_file "$RUN_DIR/render_selection_cookbook.sql" \ + "$RUN_DIR/selection_cookbook.txt" "$RUN_DIR/selection_cookbook.err" -tA +} + +write_aligned_detail() { + local tsv_file="$1" total="$2" row_class="$3" + local txt_file="${tsv_file%.tsv}.txt" + local tmp_file="$txt_file.tmp" + local err_file="$txt_file.err" + local rendered truncated="" + + [[ "$ALIGN_DETAILS" = "true" && -s "$tsv_file" ]] || return 0 + + if ! rendered="$(awk 'NR > 1 && $0 !~ /^#/ { count++ } END { print count + 0 }' "$tsv_file")"; then + printf "⚠️ Could not count detail rows for aligned companion: %s\n" "$tsv_file" \ + | tee -a "$RUN_DIR/warnings.log" >&2 || true + return 0 + fi + if grep -q '^# TRUNCATED at ' "$tsv_file" || [[ "$total" -gt "$DETAILS_LIMIT" ]]; then + truncated=" — TRUNCATED" + fi + + if awk -v max_width="$ALIGN_WIDTH" -f "$ALIGN_DETAILS_AWK" "$tsv_file" "$tsv_file" \ + > "$tmp_file" 2> "$err_file" \ + && printf '# rendered %s of %s %s rows%s\n' \ + "$rendered" "$total" "$row_class" "$truncated" >> "$tmp_file" \ + && mv "$tmp_file" "$txt_file"; then + rm -f "$err_file" + else + [[ ! -s "$err_file" ]] || cat "$err_file" >&2 + printf "⚠️ Could not create aligned detail companion for %s; canonical TSV retained.\n" "$tsv_file" \ + | tee -a "$RUN_DIR/warnings.log" >&2 || true + rm -f "$tmp_file" "$err_file" "$txt_file" + fi + return 0 +} + +write_row_details() { + local table new_n changed_n blocked_n sql_file out_file rel + mkdir -p "$RUN_DIR/details" + : > "$RUN_DIR/detail_artifacts.txt" + while read -r table; do + [[ -n "$table" ]] || continue + new_n="$(awk -F '\t' -v t="$table" '$1 == t && $2 == "NEW" { print $3; exit }' "$RUN_DIR/class_counts.tsv")" + changed_n="$(awk -F '\t' -v t="$table" '$1 == t && ($2 == "CHANGED" || $2 == "CHANGED_LOCAL_NEWER") { n += $3 } END { print n + 0 }' "$RUN_DIR/class_counts.tsv")" + blocked_n="$(awk -F '\t' -v t="$table" '$1 == t && ($2 == "BLOCKED_FK" || $2 == "AMBIGUOUS_NK") { n += $3 } END { print n + 0 }' "$RUN_DIR/class_counts.tsv")" + + if [[ "${new_n:-0}" -gt 0 ]]; then + rel="details/$table.new.tsv"; out_file="$RUN_DIR/$rel"; sql_file="$RUN_DIR/render_$table.new.sql" + printf "SELECT * FROM delta_ctl.render_new_rows_tsv(%s, %s);\n" "$(sql_literal "$table")" "$DETAILS_LIMIT" > "$sql_file" + psql_delta_session_file "$sql_file" "$out_file" "$RUN_DIR/render_$table.new.err" -tA + write_aligned_detail "$out_file" "$new_n" "NEW" + printf '%s\n' "$rel" >> "$RUN_DIR/detail_artifacts.txt" + fi + if [[ "$changed_n" -gt 0 ]]; then + rel="details/$table.changed.tsv"; out_file="$RUN_DIR/$rel"; sql_file="$RUN_DIR/render_$table.changed.sql" + printf "SELECT * FROM delta_ctl.render_changed_rows_tsv(%s, %s);\n" "$(sql_literal "$table")" "$DETAILS_LIMIT" > "$sql_file" + psql_delta_session_file "$sql_file" "$out_file" "$RUN_DIR/render_$table.changed.err" -tA + write_aligned_detail "$out_file" "$changed_n" "CHANGED" + printf '%s\n' "$rel" >> "$RUN_DIR/detail_artifacts.txt" + fi + if [[ "$blocked_n" -gt 0 ]]; then + rel="details/$table.blocked.tsv"; out_file="$RUN_DIR/$rel"; sql_file="$RUN_DIR/render_$table.blocked.sql" + printf "SELECT * FROM delta_ctl.render_blocked_rows_tsv(%s, %s);\n" "$(sql_literal "$table")" "$DETAILS_LIMIT" > "$sql_file" + psql_delta_session_file "$sql_file" "$out_file" "$RUN_DIR/render_$table.blocked.err" -tA + write_aligned_detail "$out_file" "$blocked_n" "BLOCKED" + printf '%s\n' "$rel" >> "$RUN_DIR/detail_artifacts.txt" + fi + done < "$RUN_DIR/present_tables.txt" +} + +normalize_class_list() { + local value="$1" out="$2" + : > "$out" + [[ -n "$value" ]] || return 0 + printf '%s\n' "$value" | awk -F',' ' + function trim(s) { gsub(/^[[:space:]]+|[[:space:]]+$/, "", s); return s } + function cls(s) { + s = tolower(trim(s)) + if (s == "new") return "NEW" + if (s == "changed") return "CHANGED" + if (s == "changed_local_newer" || s == "local_newer") return "CHANGED_LOCAL_NEWER" + if (s == "") return "" + printf("invalid apply class: %s\n", s) > "/dev/stderr"; exit 4 + } + { for (i = 1; i <= NF; i++) { c = cls($i); if (c != "" && !seen[c]++) print c } } + ' > "$out" || return 4 +} + +scope_selection_file_input() { + [[ "$TABLES_ARG" != "all" || -n "$INCLUDE_CLASSES" || -n "$EXCLUDE_CLASSES" ]] || return 0 + + local requested="$RUN_DIR/requested_tables.txt" + local include_classes="$RUN_DIR/scope_include_classes.txt" + local exclude_classes="$RUN_DIR/scope_exclude_classes.txt" + local scoped="$RUN_DIR/selection_input.scoped.tsv" + local restrict_include="false" + + [[ -r "$requested" ]] || die4 "requested table scope is unavailable; re-run preview" + if [[ -n "$INCLUDE_CLASSES" ]]; then + restrict_include="true" + fi + normalize_class_list "$INCLUDE_CLASSES" "$include_classes" || die4 "invalid --include-classes" + normalize_class_list "$EXCLUDE_CLASSES" "$exclude_classes" || die4 "invalid --exclude-classes" + + awk -F '\t' \ + -v requested="$requested" \ + -v included="$include_classes" \ + -v excluded="$exclude_classes" \ + -v restrict_include="$restrict_include" ' + BEGIN { + while ((getline value < requested) > 0) allowed_table[value] = 1 + close(requested) + while ((getline value < included) > 0) allowed_class[value] = 1 + close(included) + while ((getline value < excluded) > 0) denied_class[value] = 1 + close(excluded) + } + ($1 in allowed_table) && + (restrict_include != "true" || ($2 in allowed_class)) && + !($2 in denied_class) { print $0 } + ' "$RUN_DIR/selection_input.tsv" > "$scoped" || die4 "could not enforce selection table/class scope" + mv "$scoped" "$RUN_DIR/selection_input.tsv" +} + +build_selection_input() { + local base_classes="$RUN_DIR/base_classes.txt" + local exclude_classes="$RUN_DIR/exclude_classes.txt" + local requested="$RUN_DIR/requested_tables.txt" + : > "$RUN_DIR/selection_input.tsv" + : > "$RUN_DIR/row_selection_input.tsv" + : > "$RUN_DIR/selection_header.tsv" + + if [[ -n "$SELECTION_FILE" ]]; then + [[ -r "$SELECTION_FILE" ]] || die4 "selection file is not readable: $SELECTION_FILE" + awk -v all_tables="$RUN_DIR/all_conf_tables.txt" \ + -v rowout="$RUN_DIR/row_selection_input.tsv" \ + -v headerout="$RUN_DIR/selection_header.tsv" ' + function trim(s) { gsub(/^[[:space:]]+|[[:space:]]+$/, "", s); return s } + function cls(s) { + s = tolower(trim(s)) + if (s == "new") return "NEW" + if (s == "changed") return "CHANGED" + if (s == "changed_local_newer" || s == "local_newer") return "CHANGED_LOCAL_NEWER" + if (s == "") return "" + fail("invalid apply class: " s) + } + function fail(message) { + printf("selection line %s: %s\n", NR, message) > "/dev/stderr" + failed = 1 + exit 4 + } + function decimal_le(a, b, aa, bb, k, ad, bd) { + aa = a; bb = b + sub(/^0+/, "", aa); sub(/^0+/, "", bb) + if (aa == "") aa = "0"; if (bb == "") bb = "0" + if (length(aa) != length(bb)) return length(aa) < length(bb) + for (k = 1; k <= length(aa); k++) { + ad = substr(aa, k, 1) + 0; bd = substr(bb, k, 1) + 0 + if (ad != bd) return ad < bd + } + return 1 + } + function copy_escape(s) { + gsub(/\\/, "\\\\", s) + gsub(/\t/, "\\t", s) + gsub(/\r/, "\\r", s) + return s + } + BEGIN { + while ((getline t < all_tables) > 0) { valid[t] = 1; order[++n] = t } + close(all_tables) + wildcard = "NEW,CHANGED" + } + { + raw = $0 + sub(/\r$/, "", raw) + if (raw ~ /^[[:space:]]*#[[:space:]]*dump_sha256=/) { + if (dump_header_seen++) fail("duplicate dump_sha256 header") + value = raw + sub(/^[[:space:]]*#[[:space:]]*dump_sha256=/, "", value) + print "dump_sha256\t" trim(value) >> headerout + } else if (raw ~ /^[[:space:]]*#[[:space:]]*staged[[:space:]]+/) { + value = raw + sub(/^[[:space:]]*#[[:space:]]*staged[[:space:]]+/, "", value) + count = split(value, entries, /[[:space:]]+/) + for (i = 1; i <= count; i++) { + if (entries[i] == "") continue + eq = index(entries[i], "=") + if (eq <= 1 || eq == length(entries[i])) { + fail("staged binding must use table=: " entries[i]) + } + staged_table = substr(entries[i], 1, eq - 1) + staged_count = substr(entries[i], eq + 1) + if (staged_count !~ /^[0-9]+$/) { + fail("staged count must be a non-negative decimal integer for " staged_table) + } + if (staged_header_seen[staged_table]++) { + fail("duplicate staged-count header for " staged_table) + } + print "staged\t" staged_table "\t" staged_count >> headerout + } + } + + line = raw + sub(/[[:space:]]*#.*/, "", line) + line = trim(line) + if (line == "") next + if (line !~ /^\[[^]]+\]/) fail("invalid syntax") + + target = line + sub(/^\[/, "", target); sub(/\].*/, "", target); target = trim(target) + rest = line; sub(/^\[[^]]+\]/, "", rest); rest = trim(rest) + + if (rest ~ /^[a-z_]+[.]rows[[:space:]]+/) { + if (target == "*") fail("row selectors do not support [*]") + if (!valid[target]) fail("selection table is not preserved: " target) + rowclass = rest; sub(/[.]rows.*/, "", rowclass); rowclass = cls(rowclass) + tail = rest; sub(/^[a-z_]+[.]rows[[:space:]]+/, "", tail) + mode = tail; sub(/[[:space:]]*=.*/, "", mode); mode = tolower(trim(mode)) + if (mode != "include" && mode != "exclude") fail("row selector mode must be include or exclude") + if (tail !~ /=/) fail("row selector is missing =") + payload = tail; sub(/^[^=]*=/, "", payload); payload = trim(payload) + colon = index(payload, ":") + if (colon < 2) fail("row selector must use id:, row:, or corp:") + kind = tolower(trim(substr(payload, 1, colon - 1))) + values = substr(payload, colon + 1) + if (kind != "id" && kind != "row" && kind != "corp") fail("unsupported row selector kind: " kind) + if (trim(values) == "") fail("row selector value list is empty") + value_count = split(values, vals, ",") + for (j = 1; j <= value_count; j++) { + value = trim(vals[j]) + if (value == "") fail("row selector contains an empty value") + from = "\\N"; to = "\\N"; corp = "\\N"; is_range = "f" + if (kind == "corp") { + corp = copy_escape(value) + } else { + if (value !~ /^[0-9]+(-[0-9]+)?$/) fail(kind " selector must be a non-negative integer or range: " value) + dash = index(value, "-") + if (dash > 0) { + from = substr(value, 1, dash - 1); to = substr(value, dash + 1); is_range = "t" + if (!decimal_le(from, to)) fail("row selector range is inverted: " value) + } else { + from = value; to = value + } + if (!decimal_le(from, "9223372036854775807") || !decimal_le(to, "9223372036854775807")) { + fail(kind " selector exceeds bigint maximum: " value) + } + } + key = target SUBSEP rowclass SUBSEP mode SUBSEP kind SUBSEP value + if (selector_seen[key]++) { + printf("warning: selection line %s duplicates selector %s:%s\n", NR, kind, value) > "/dev/stderr" + continue + } + print target "\t" rowclass "\t" mode "\t" kind "\t" from "\t" to "\t" corp "\t" is_range "\t" NR >> rowout + } + next + } + + if (rest !~ /^include[[:space:]]*=/) fail("expected include=classes or .rows include|exclude=:") + includes = rest; sub(/^include[[:space:]]*=/, "", includes); includes = trim(includes) + if (target == "*") wildcard = includes + else if (!valid[target]) fail("selection table is not preserved: " target) + else override[target] = includes + } + END { + if (failed) exit 4 + for (i = 1; i <= n; i++) { + t = order[i] + includes = (t in override) ? override[t] : wildcard + part_count = split(includes, parts, ",") + for (j = 1; j <= part_count; j++) { + c = cls(parts[j]) + if (c != "" && !seen[t, c]++) print t "\t" c + } + } + } + ' "$SELECTION_FILE" > "$RUN_DIR/selection_input.tsv" || die4 "invalid selection file: $SELECTION_FILE" + scope_selection_file_input + return 0 + fi + + normalize_class_list "${INCLUDE_CLASSES:-new,changed}" "$base_classes" || die4 "invalid --include-classes" + normalize_class_list "$EXCLUDE_CLASSES" "$exclude_classes" || die4 "invalid --exclude-classes" + + while read -r table; do + [[ -n "$table" ]] || continue + while read -r class; do + [[ -n "$class" ]] || continue + if ! grep -qx -- "$class" "$exclude_classes"; then + printf "%s\t%s\n" "$table" "$class" >> "$RUN_DIR/selection_input.tsv" + fi + done < "$base_classes" + done < "$requested" +} + +verify_selection_binding() { + [[ -s "$RUN_DIR/row_selection_input.tsv" ]] || return 0 + local bound_sha table expected actual + bound_sha="$(awk -F '\t' '$1 == "dump_sha256" { print $2; exit }' "$RUN_DIR/selection_header.tsv")" + [[ -n "$bound_sha" ]] || die4 "row-selector selection file is missing # dump_sha256=; re-run preview" + [[ "$bound_sha" = "$DUMP_SHA256" ]] || die4 "selection file was generated from a different dump (expected $DUMP_SHA256, found $bound_sha); re-run preview" + + while read -r table; do + [[ -n "$table" ]] || continue + expected="$(awk -F '\t' -v t="$table" '$1 == "staged" && $2 == t { print $3; exit }' "$RUN_DIR/selection_header.tsv")" + actual="$(awk -F '\t' -v t="$table" '$1 == t && $2 == "STAGED" { print $3; exit }' "$RUN_DIR/stage_counts.tsv")" + [[ -n "$expected" ]] || die4 "row: selector for $table is missing its # staged $table= binding" + [[ "$expected" = "${actual:-0}" ]] || die4 "staged row count for $table changed (selection=$expected current=${actual:-0}); re-run preview" + done < <(awk -F '\t' '$4 == "row" { print $1 }' "$RUN_DIR/row_selection_input.tsv" | sort -u) + + psql_cmd "INSERT INTO delta_ctl.run_metadata(key, value) VALUES + ('selection_file', $(sql_literal "$SELECTION_FILE")), + ('selection_bound_sha256', $(sql_literal "$bound_sha")) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, created_at = now();" >/dev/null +} + +load_selection_tables() { + build_selection_input + verify_selection_binding + psql_cmd "TRUNCATE delta_ctl.selection; TRUNCATE delta_ctl.row_selection; TRUNCATE delta_ctl.selection_diagnostics; TRUNCATE delta_ctl.only_corps;" + if [[ -s "$RUN_DIR/selection_input.tsv" ]]; then + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 \ + -c "COPY delta_ctl.selection(table_name, class) FROM STDIN" \ + < "$RUN_DIR/selection_input.tsv" + fi + if [[ -s "$RUN_DIR/row_selection_input.tsv" ]]; then + if ! "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 \ + -c "COPY delta_ctl.row_selection(table_name, class, mode, kind, value_from, value_to, corp_num, is_range, source_line) FROM STDIN" \ + < "$RUN_DIR/row_selection_input.tsv"; then + die4 "failed to load row selectors; check selection values and line diagnostics" + fi + fi + + if [[ -n "$ONLY_CORPS_FILE" ]]; then + [[ -r "$ONLY_CORPS_FILE" ]] || die4 "--only-corps file is not readable: $ONLY_CORPS_FILE" + awk 'NF && $1 !~ /^#/ { print $1 }' "$ONLY_CORPS_FILE" > "$RUN_DIR/only_corps.tsv" + if [[ -s "$RUN_DIR/only_corps.tsv" ]]; then + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 \ + -c "COPY delta_ctl.only_corps(corp_num) FROM STDIN" \ + < "$RUN_DIR/only_corps.tsv" + fi + fi +} + +prepare_apply_selection() { + printf "🎯 Loading and stamping selection …\n" + load_selection_tables + psql_cmd "SELECT delta_ctl.verify_row_selection();" > "$RUN_DIR/row_selection_validation.out" + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 -tA -F $'\t' > "$RUN_DIR/selection_diagnostics.tsv" <<'SQL' +SELECT table_name, class, mode, kind, selector, source_line, COALESCE(matched::text, ''), problem +FROM delta_ctl.selection_diagnostics +ORDER BY source_line, table_name, class; +SQL + if awk -F '\t' 'NF >= 8 && $8 != "" { found=1 } END { exit !found }' "$RUN_DIR/selection_diagnostics.tsv"; then + cat "$RUN_DIR/selection_diagnostics.tsv" >&2 + die4 "row selection validation failed; see selection_diagnostics.tsv" + fi + psql_cmd "SELECT delta_ctl.stamp_selection();" + if ! psql_cmd "SELECT delta_ctl.validate_dependencies();" > "$RUN_DIR/selection_validation.out" 2> "$RUN_DIR/selection_validation.err"; then + cat "$RUN_DIR/selection_validation.err" >&2 + die4 "selection/dependency validation failed" + fi + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 -tA -F $'\t' > "$RUN_DIR/dependency_violations.tsv" <<'SQL' +SELECT child_table, parent_table, reason, row_count, COALESCE(sample_ids, '') +FROM delta_ctl.dependency_violations +ORDER BY child_table, parent_table; +SQL + if [[ -s "$RUN_DIR/dependency_violations.tsv" ]]; then + cat "$RUN_DIR/dependency_violations.tsv" >&2 + die4 "selection/dependency validation failed; see dependency_violations.tsv" + fi + + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 -tA -F $'\t' > "$RUN_DIR/selected_counts.tsv" <<'SQL' +SELECT table_name, class, row_count, load_phase +FROM delta_ctl.selected_counts() +ORDER BY load_phase, table_name, class; +SQL + + printf "Selected rows:\n" + if [[ -s "$RUN_DIR/selected_counts.tsv" ]]; then + awk -F '\t' '{ printf " %-36s %-20s %s\n", $1, $2, $3 }' "$RUN_DIR/selected_counts.tsv" + else + printf " (none)\n" + fi +} + +confirm_apply() { + if [[ "$YES" = "true" ]]; then + return 0 + fi + printf "Proceed with applying selected rows to %s:%s/%s? Type 'yes' to continue: " "$PGHOST" "$PGPORT" "$PGDATABASE" + local answer + read -r answer + [[ "$answer" = "yes" ]] || die4 "apply cancelled" +} + +run_apply_transaction() { + printf "🚀 Applying selected rows in one transaction …\n" + set +e + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 \ + > "$RUN_DIR/apply_transaction.out" \ + 2> "$RUN_DIR/apply_transaction.err" <&2 + if grep -q "SELECTION_INVALID" "$RUN_DIR/apply_transaction.err"; then + die4 "selection/dependency validation failed during apply" + fi + if grep -q "APPLY_VERIFICATION_FAILED" "$RUN_DIR/apply_transaction.err"; then + die5 "apply verification failed; transaction was rolled back" + fi + die "apply transaction failed; transaction was rolled back" + fi +} + +post_apply_maintenance() { + printf "🛠 Repairing sequences …\n" + psql_file "$REPAIR_SEQUENCES_SQL" + + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 -tA > "$RUN_DIR/touched_tables.txt" <<'SQL' +SELECT table_name FROM delta_ctl.touched_tables ORDER BY table_name; +SQL + : > "$RUN_DIR/analyze_touched.sql" + while read -r table; do + [[ -n "$table" ]] || continue + printf 'ANALYZE public.%s;\n' "$table" >> "$RUN_DIR/analyze_touched.sql" + done < "$RUN_DIR/touched_tables.txt" + if [[ -s "$RUN_DIR/analyze_touched.sql" ]]; then + printf "📊 Analyzing touched tables …\n" + psql_file "$RUN_DIR/analyze_touched.sql" + fi +} + +write_apply_summary() { + local summary="$RUN_DIR/apply_summary.txt" + { + printf "Delta restore apply summary\n" + printf "===========================\n\n" + printf "Dump: %s\n" "$DUMP" + printf "Target: %s:%s/%s as %s\n" "$PGHOST" "$PGPORT" "$PGDATABASE" "$PGUSER" + printf "Run dir: %s\n" "$RUN_DIR" + printf "Selection source: %s\n" "${SELECTION_FILE:-CLI/default}" + printf "\n" + } > "$summary" + "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 -tA \ + -c "SELECT * FROM delta_ctl.render_apply_summary_lines();" \ + >> "$summary" + cat "$summary" +} + +write_preview_report() { + local report="$RUN_DIR/preview.txt" + local detail aligned_detail table_name detail_class rendered total truncated aligned + { + printf "Delta restore preview\n" + printf "=====================\n\n" + printf "Dump: %s\n" "$DUMP" + printf "Target: %s:%s/%s as %s\n" "$PGHOST" "$PGPORT" "$PGDATABASE" "$PGUSER" + printf "Run dir: %s\n" "$RUN_DIR" + printf "Mode: %s\n" "$MODE" + printf "Tables: %s\n" "$TABLES_ARG" + printf "Sample size: %s\n" "$SAMPLE_SIZE" + printf "Details limit: %s\n" "$DETAILS_LIMIT" + printf "Selection manifest: %s\n" "$RUN_DIR/selection.conf" + if [[ -f "$RUN_DIR/dump.manifest.json" ]]; then + printf "Manifest: %s\n" "$RUN_DIR/dump.manifest.json" + fi + printf "\nDetail artifacts\n" + printf '%s\n' '----------------' + if [[ -s "$RUN_DIR/detail_artifacts.txt" ]]; then + while read -r detail; do + table_name="${detail#details/}" + table_name="${table_name%.*.tsv}" + rendered="$(awk 'NR > 1 && $0 !~ /^#/ { count++ } END { print count + 0 }' "$RUN_DIR/$detail")" + case "$detail" in + *.new.tsv) + detail_class="NEW" + total="$(awk -F '\t' -v t="$table_name" '$1 == t && $2 == "NEW" { n += $3 } END { print n + 0 }' "$RUN_DIR/class_counts.tsv")" + ;; + *.changed.tsv) + detail_class="CHANGED" + total="$(awk -F '\t' -v t="$table_name" '$1 == t && ($2 == "CHANGED" || $2 == "CHANGED_LOCAL_NEWER") { n += $3 } END { print n + 0 }' "$RUN_DIR/class_counts.tsv")" + ;; + *.blocked.tsv) + detail_class="BLOCKED" + total="$(awk -F '\t' -v t="$table_name" '$1 == t && ($2 == "BLOCKED_FK" || $2 == "AMBIGUOUS_NK") { n += $3 } END { print n + 0 }' "$RUN_DIR/class_counts.tsv")" + ;; + esac + + truncated="" + if grep -q '^# TRUNCATED at ' "$RUN_DIR/$detail" || [[ "$total" -gt "$DETAILS_LIMIT" ]]; then + truncated=" (TRUNCATED)" + fi + aligned="" + aligned_detail="${detail%.tsv}.txt" + if [[ -f "$RUN_DIR/$aligned_detail" ]]; then + aligned=" · aligned: $aligned_detail" + fi + printf -- "- %s — %s of %s %s rows%s%s\n" \ + "$detail" "$rendered" "$total" "$detail_class" "$truncated" "$aligned" + done < "$RUN_DIR/detail_artifacts.txt" + else + printf "(none)\n" + fi + cat <<'TIPS' + +Viewing tips +------------ +Aligned text: open details/..txt in any editor (turn off word wrap). +Terminal: column -s $'\t' -t < details/
..tsv | less -S +SQL: the delta_stage/delta_diff schemas remain installed after preview; + see "Querying the staged run directly" in README_delta_restore.md. +TIPS + if [[ -s "$RUN_DIR/drift_local_copy_permitted.tsv" ]]; then + printf "\nSchema drift warnings\n" + printf "---------------------\n" + awk -F '\t' '{ printf "- %s.%s: %s\n", $1, $2, $3 }' "$RUN_DIR/drift_local_copy_permitted.tsv" + fi + printf "\n" + } > "$report" + + cat > "$RUN_DIR/render_preview_lines.sql" <> "$report" + + cat "$report" +} + +run_preview_staging() { + verify_dump_and_manifest + load_preserved_config + select_tables + filter_toc + install_control_schemas + create_stage_shells + stream_stage_data + create_stage_indexes + record_stage_counts + run_preview_classification + write_row_details + write_selection_manifest + write_selection_cookbook + write_preview_report + + printf '\nNext steps:\n' + printf ' 1. Review %q\n' "$RUN_DIR/preview.txt" + printf ' 2. cp %q my_selection.conf and edit it\n' "$RUN_DIR/selection.conf" + printf ' 3. To validate: %q --dump %q --mode validate' "$0" "$DUMP" + [[ "$TABLES_ARG" = "all" ]] || printf ' --tables %q' "$TABLES_ARG" + [[ -z "$INCLUDE_CLASSES" ]] || printf ' --include-classes %q' "$INCLUDE_CLASSES" + [[ -z "$EXCLUDE_CLASSES" ]] || printf ' --exclude-classes %q' "$EXCLUDE_CLASSES" + printf ' --selection-file my_selection.conf' + [[ -z "$ONLY_CORPS_FILE" ]] || printf ' --only-corps %q' "$ONLY_CORPS_FILE" + printf '\n' +} + +verify_resident_run() { + local resident_state resident_sha + + if ! resident_state="$("$PSQL_BIN" -X $(pg_conn_opts) -qAt -v ON_ERROR_STOP=1 <<'SQL' +SELECT CASE + WHEN to_regclass('delta_ctl.run_metadata') IS NULL THEN 'NO_RUN' + WHEN to_regclass('delta_ctl.run_counts') IS NULL + OR to_regclass('delta_ctl.table_config') IS NULL + OR to_regclass('delta_ctl.selection') IS NULL + OR to_regclass('delta_ctl.row_selection') IS NULL + OR to_regclass('delta_ctl.selection_diagnostics') IS NULL + OR to_regclass('delta_ctl.only_corps') IS NULL + OR to_regclass('delta_ctl.dependency_violations') IS NULL + OR NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'delta_stage') + OR NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'delta_map') + OR NOT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = 'delta_diff') + OR to_regprocedure('delta_ctl.verify_row_selection()') IS NULL + OR to_regprocedure('delta_ctl.stamp_selection()') IS NULL + OR to_regprocedure('delta_ctl.validate_dependencies()') IS NULL + OR to_regprocedure('delta_ctl.selected_counts()') IS NULL + THEN 'INCOMPLETE' + ELSE 'READY' +END; +SQL + )"; then + die2 "could not inspect resident delta run" + fi + + case "$resident_state" in + NO_RUN|'') die2 "no resident delta run found; run --mode preview first" ;; + READY) ;; + *) die2 "resident delta run is incomplete; re-run preview" ;; + esac + + compute_dump_sha + if ! resident_sha="$("$PSQL_BIN" -X $(pg_conn_opts) -qAt -v ON_ERROR_STOP=1 \ + -c "SELECT value FROM delta_ctl.run_metadata WHERE key = 'dump_sha256'")"; then + die2 "could not read resident delta run metadata" + fi + [[ -n "$resident_sha" ]] || die2 "resident delta run has no dump sha256; re-run preview" + [[ "$resident_sha" = "$DUMP_SHA256" ]] \ + || die2 "resident run was staged from a different dump; re-run preview" + + if ! "$PSQL_BIN" -X $(pg_conn_opts) -v ON_ERROR_STOP=1 -tA -F $'\t' \ + > "$RUN_DIR/stage_counts.tsv" <<'SQL' +SELECT table_name, count_name, row_count +FROM delta_ctl.run_counts +WHERE count_name IN ('STAGED', 'SKIPPED_ABSENT') +ORDER BY table_name, count_name; +SQL + then + die2 "could not regenerate staged-count binding from resident run" + fi + [[ -s "$RUN_DIR/stage_counts.tsv" ]] \ + || die2 "resident delta run has no staged-count metadata; re-run preview" +} + +run_validate_mode() { + verify_resident_run + load_preserved_config + select_tables + prepare_apply_selection + + printf "✅ Selection is valid.\n" + printf 'To apply: %q --dump %q --mode apply' "$0" "$DUMP" + [[ "$TABLES_ARG" = "all" ]] || printf ' --tables %q' "$TABLES_ARG" + [[ -z "$INCLUDE_CLASSES" ]] || printf ' --include-classes %q' "$INCLUDE_CLASSES" + [[ -z "$EXCLUDE_CLASSES" ]] || printf ' --exclude-classes %q' "$EXCLUDE_CLASSES" + [[ -z "$SELECTION_FILE" ]] || printf ' --selection-file %q' "$SELECTION_FILE" + [[ -z "$ONLY_CORPS_FILE" ]] || printf ' --only-corps %q' "$ONLY_CORPS_FILE" + printf ' --yes\n' +} + +run_apply_mode() { + verify_dump_and_manifest + load_preserved_config + select_tables + filter_toc + install_control_schemas + create_stage_shells + stream_stage_data + create_stage_indexes + record_stage_counts + run_preview_classification + write_row_details + write_selection_manifest + write_selection_cookbook + prepare_apply_selection + confirm_apply + run_apply_transaction + post_apply_maintenance + write_apply_summary + if [[ "$KEEP_ARTIFACTS" != "true" ]]; then + cleanup_schemas + fi +} + +############################################################################## +# MAIN +############################################################################## + +if [[ "${DELTA_RESTORE_SOURCE_ONLY:-false}" = "true" ]]; then + return 0 2>/dev/null || exit 0 +fi + +parse_args "$@" +validate_static_files +init_run_dir +trap on_exit EXIT + +acquire_lock + +if [[ "$CLEANUP" = "true" ]]; then + cleanup_schemas + exit 0 +fi + +case "$MODE" in + apply) run_apply_mode ;; + validate) run_validate_mode ;; + preview) run_preview_staging ;; +esac diff --git a/data-tool/dump_colin_extract_without_views.sh b/data-tool/dump_colin_extract_without_views.sh index 42988dd177..b170a7d62b 100755 --- a/data-tool/dump_colin_extract_without_views.sh +++ b/data-tool/dump_colin_extract_without_views.sh @@ -19,6 +19,7 @@ PGPORT="${PGPORT:-5432}" # or --port PGUSER="${PGUSER:-postgres}" # or --user PGDATABASE="${PGDATABASE:-colin-mig-corps-test}" # or --dbname PGSCHEMA="${PGSCHEMA:-public}" +TARGET_PGSCHEMA="${TARGET_PGSCHEMA:-colin_extract}" # Supply the password *either* via a .pgpass file *or* one-shot: # PGPASSWORD=secret MODE=dump ./dump_colin_extract_without_views.sh ############################################################################## @@ -47,6 +48,11 @@ print_command() { printf '\n' } +psql_cmd(){ + "psql" -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -v ON_ERROR_STOP=1 "$@" +} + + ############################################################################## # DERIVED VIEW/MV EXCLUSIONS ############################################################################## @@ -79,6 +85,11 @@ case "$MODE" in *) die "MODE must be 'print' or 'dump'" ;; esac +psql_cmd -c "ALTER SCHEMA ${PGSCHEMA} RENAME TO \"${TARGET_PGSCHEMA}\";" +printf "✅ schema renamed to ${TARGET_PGSCHEMA}...... %s\n" "$DUMP" + +PGSCHEMA="$TARGET_PGSCHEMA" + CMD=( "$PG_DUMP_BIN" -F t @@ -90,6 +101,7 @@ CMD=( -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" + -n "$PGSCHEMA" ) for object in "${EXCLUDED_OBJECTS[@]}"; do @@ -101,7 +113,7 @@ CMD+=(-f "$DUMP") printf "📦 COLIN extract dump without derived views/materialized views\n" printf "Mode: %s\n" "$MODE" printf "Database: %s@%s:%s/%s\n" "$PGUSER" "$PGHOST" "$PGPORT" "$PGDATABASE" -printf "Schema: %s\n" "$PGSCHEMA" +printf "Schema: %s\n" "$PGSCHEMA" "$TARGET_PGSCHEMA" printf "Output: %s\n" "$DUMP" printf "Excluded objects:\n" for object in "${EXCLUDED_OBJECTS[@]}"; do @@ -119,3 +131,5 @@ mkdir -p "$(dirname "$DUMP")" printf "\nRunning pg_dump...\n" "${CMD[@]}" printf "✅ Dump written to %s\n" "$DUMP" +psql_cmd -c "ALTER SCHEMA \"${TARGET_PGSCHEMA}\" RENAME TO public;" +printf "✅ schema naming back to public...... %s\n" "$DUMP" diff --git a/data-tool/flows/auth/auth_create_flow.py b/data-tool/flows/auth/auth_create_flow.py index 284d1464cb..d8f97c4344 100644 --- a/data-tool/flows/auth/auth_create_flow.py +++ b/data-tool/flows/auth/auth_create_flow.py @@ -33,6 +33,18 @@ from .auth_tasks import parse_accounts_csv, perform_auth_create_for_corp +def _build_auth_create_plan(config) -> AuthCreatePlan: + return AuthCreatePlan( + create_entity=bool(getattr(config, 'AUTH_CREATE_ENTITY', True)), + upsert_contact=bool(getattr(config, 'AUTH_UPSERT_CONTACT', False)), + create_affiliations=bool(getattr(config, 'AUTH_CREATE_AFFILIATIONS', False)), + send_unaffiliated_invite=bool(getattr(config, 'AUTH_SEND_UNAFFILIATED_EMAIL', False)), + invite_is_reminder=bool(getattr(config, 'AUTH_INVITE_IS_REMINDER', False)), + fail_if_missing_email=bool(getattr(config, 'AUTH_FAIL_IF_MISSING_EMAIL', False)), + dry_run=bool(getattr(config, 'AUTH_DRY_RUN', False)), + ) + + @flow( name='Auth-Create-Flow', log_prints=True, @@ -48,14 +60,7 @@ def auth_create_flow(): """ config = get_config() - plan = AuthCreatePlan( - create_entity=bool(getattr(config, 'AUTH_CREATE_ENTITY', True)), - upsert_contact=bool(getattr(config, 'AUTH_UPSERT_CONTACT', False)), - create_affiliations=bool(getattr(config, 'AUTH_CREATE_AFFILIATIONS', False)), - send_unaffiliated_invite=bool(getattr(config, 'AUTH_SEND_UNAFFILIATED_EMAIL', False)), - fail_if_missing_email=bool(getattr(config, 'AUTH_FAIL_IF_MISSING_EMAIL', False)), - dry_run=bool(getattr(config, 'AUTH_DRY_RUN', False)), - ) + plan = _build_auth_create_plan(config) validate_auth_create_flow_plan(plan) diff --git a/data-tool/flows/auth/auth_invite_flow.py b/data-tool/flows/auth/auth_invite_flow.py index 64a8622078..458967c2a5 100644 --- a/data-tool/flows/auth/auth_invite_flow.py +++ b/data-tool/flows/auth/auth_invite_flow.py @@ -32,6 +32,18 @@ from .auth_tasks import perform_auth_create_for_corp +def _build_auth_invite_plan(config) -> AuthCreatePlan: + return AuthCreatePlan( + create_entity=False, + upsert_contact=False, + create_affiliations=False, + send_unaffiliated_invite=True, + invite_is_reminder=bool(getattr(config, 'AUTH_INVITE_IS_REMINDER', False)), + fail_if_missing_email=bool(getattr(config, 'AUTH_FAIL_IF_MISSING_EMAIL', False)), + dry_run=bool(getattr(config, 'AUTH_DRY_RUN', False)), + ) + + @flow( name='Auth-Invite-Flow', log_prints=True, @@ -50,14 +62,7 @@ def auth_invite_flow(): config = get_config() selection_mode = parse_auth_selection_mode(config) - plan = AuthCreatePlan( - create_entity=False, - upsert_contact=False, - create_affiliations=False, - send_unaffiliated_invite=True, - fail_if_missing_email=bool(getattr(config, 'AUTH_FAIL_IF_MISSING_EMAIL', False)), - dry_run=bool(getattr(config, 'AUTH_DRY_RUN', False)), - ) + plan = _build_auth_invite_plan(config) log_auth_config_preflight( config, diff --git a/data-tool/flows/auth/auth_models.py b/data-tool/flows/auth/auth_models.py index 2d5b0a31df..0fb8582b2c 100644 --- a/data-tool/flows/auth/auth_models.py +++ b/data-tool/flows/auth/auth_models.py @@ -97,6 +97,7 @@ class AuthCreatePlan: fail_if_missing_email: bool = False dry_run: bool = False allow_entity_creation_for_affiliations: bool = True + invite_is_reminder: bool = False @dataclass(frozen=True) diff --git a/data-tool/flows/auth/auth_queries.py b/data-tool/flows/auth/auth_queries.py index 59a0d29e84..6f1ec72b0d 100644 --- a/data-tool/flows/auth/auth_queries.py +++ b/data-tool/flows/auth/auth_queries.py @@ -15,7 +15,7 @@ ) # Match the existing tombstone corp type cohort by default. -CORP_TYPE_FILTER = "('BC', 'C', 'ULC', 'CUL', 'CC', 'CCC', 'QA', 'QB', 'QC', 'QD', 'QE')" +CORP_TYPE_FILTER = "('BC', 'C', 'ULC', 'CUL', 'CC', 'CCC', 'QA', 'QB', 'QC', 'QD', 'QE', 'LIB')" _UNSET = object() diff --git a/data-tool/flows/auth/auth_report_helpers.py b/data-tool/flows/auth/auth_report_helpers.py index bd63f20e9f..bf6daab6cb 100644 --- a/data-tool/flows/auth/auth_report_helpers.py +++ b/data-tool/flows/auth/auth_report_helpers.py @@ -18,9 +18,14 @@ CHECK_CONTACT, CHECK_ENTITY, CHECK_INVITE, + INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA, NO_CACHE, AuthBatchResult, + AuthInviteDiagnostics, ConsoleLimit, + InviteClause, + InviteClauseKind, + InviteCriteria, blank_to_none, build_candidate_queries, build_inspection_identifier_rows, @@ -28,15 +33,21 @@ build_inspection_summary, calculate_batch_count, dispose_engine, + evaluate_invite_clauses, filter_inspection_states, flow, + format_clock_minutes_z, + format_clock_z, format_console_limit, + format_invite_criteria, get_candidate_count, get_candidate_page, is_csv_like_output_root, parse_auth_selection_mode, parse_console_limit, parse_inspect_filter, + parse_invite_criteria, + parse_invite_expiry_days, parse_manual_account_ids, parse_positive_int_tuple, print_inspection_identifier_rows, diff --git a/data-tool/flows/auth/auth_tasks.py b/data-tool/flows/auth/auth_tasks.py index 714b2c6d02..a39357d055 100644 --- a/data-tool/flows/auth/auth_tasks.py +++ b/data-tool/flows/auth/auth_tasks.py @@ -568,11 +568,14 @@ def perform_auth_create_for_corp( config=config, identifier=identifier, email=email, - token=token + token=token, + is_reminder=plan.invite_is_reminder, ) current_operation = None code = _status_code(status) detail_parts.append(f'invite:{code}') + if plan.invite_is_reminder: + detail_parts.append('invite:reminder') if code == int(HTTPStatus.OK): invite_action = AuthComponentStatus.SUCCESS else: @@ -592,6 +595,7 @@ def perform_auth_create_for_corp( status=invite_action, status_code=code, error=None if invite_action == AuthComponentStatus.SUCCESS else f'send_invite:{code}', + detail='invite:reminder' if plan.invite_is_reminder else None, ) except Exception as e: diff --git a/data-tool/flows/auth/inspect_auth_flow.py b/data-tool/flows/auth/inspect_auth_flow.py index 74530cf1f3..be8318adcf 100644 --- a/data-tool/flows/auth/inspect_auth_flow.py +++ b/data-tool/flows/auth/inspect_auth_flow.py @@ -4,13 +4,15 @@ verify-auth candidate selection, expected-account lookup, Auth DB read-back, inspection row builders, CSV writer, Prefect fallback decorators, and console limit helpers while using shared AUTH_REPORT_* throughput plus inspect-specific -INSPECT_AUTH_* configuration. +INSPECT_AUTH_* configuration, including expression-based INSPECT_AUTH_INVITE_CRITERIA +and defaulted diagnostics-only INSPECT_AUTH_INVITE_EXPIRY_DAYS. """ from __future__ import annotations import logging from dataclasses import dataclass +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Iterable @@ -24,11 +26,16 @@ CHECK_ENTITY, CHECK_INVITE, AUTH_OUTPUT_PATH_ATTR, + INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA, AuthBatchResult, ConsoleLimit, + InviteCriteria, blank_to_none, dispose_engine, + format_clock_minutes_z, + format_clock_z, format_console_limit, + format_invite_criteria, is_csv_like_output_root, parse_auth_selection_mode, parse_manual_account_ids, @@ -45,6 +52,8 @@ get_candidate_page, parse_console_limit, parse_inspect_filter, + parse_invite_criteria, + parse_invite_expiry_days, print_inspection_identifier_rows, print_inspection_rows, print_inspection_summary, @@ -68,6 +77,7 @@ INSPECTION_FILENAME = "inspect-auth-inspection.csv" INSPECTION_SUMMARY_FILENAME = "inspect-auth-summary.txt" INSPECT_AUTH_CONSOLE_LIMIT_ATTR = "INSPECT_AUTH_CONSOLE_LIMIT" +INSPECT_AUTH_BUSINESS_NAMES_MAX_LENGTH_ATTR = "INSPECT_AUTH_BUSINESS_NAMES_MAX_LENGTH" INSPECT_AUTH_READ_CHECKS = (CHECK_ENTITY, CHECK_CONTACT, CHECK_AFFILIATION, CHECK_INVITE) @@ -79,6 +89,7 @@ class InspectAuthSettings: batch_size: int inspect_filter: str console_limit: ConsoleLimit + business_names_max_length: int | None inspection_path: str summary_path: str selection_mode: AuthSelectionMode @@ -87,6 +98,10 @@ class InspectAuthSettings: auth_mig_batch_ids: tuple[int, ...] manual_account_ids: tuple[str, ...] selected_checks: tuple[str, ...] = () + run_clock: datetime | None = None + invite_expiry_days: int = 7 + invite_expiry_cutoff: datetime | None = None + invite_criteria: InviteCriteria | None = None @property def run_verify(self) -> bool: @@ -103,6 +118,11 @@ def read_expected_accounts(self) -> bool: """Return True when expected account IDs should be included in inspect rows.""" return self.selection_mode != AuthSelectionMode.MANUAL or bool(self.manual_account_ids) + @property + def invite_cutoff(self) -> datetime | None: + """Return the once-per-run expiry cutoff used by every Auth batch.""" + return self.invite_expiry_cutoff + def _coerce_int_setting(config: Any, attr_name: str) -> int: raw_value = getattr(config, attr_name, 0) @@ -112,6 +132,23 @@ def _coerce_int_setting(config: Any, attr_name: str) -> int: raise ValueError(f"{attr_name} must be a valid integer") from exc +def parse_business_names_max_length( + raw_value: Any, + config_key: str = INSPECT_AUTH_BUSINESS_NAMES_MAX_LENGTH_ATTR, +) -> int | None: + """Parse the optional inspect-auth console cap for the business-names cell.""" + raw = blank_to_none(raw_value) + if raw is None or raw.upper() == "ALL": + return None + try: + parsed = int(raw) + except (TypeError, ValueError) as exc: + raise ValueError(f"{config_key} must be a positive integer or ALL") from exc + if parsed <= 0: + raise ValueError(f"{config_key} must be a positive integer or ALL") + return parsed + + def _derive_inspection_path(output_root: str) -> str: return str(Path(output_root).expanduser() / INSPECTION_FILENAME) @@ -129,6 +166,16 @@ def _resolve_inspection_paths(config: Any) -> tuple[str, str]: return _derive_inspection_path(output_root), _derive_summary_path(output_root) +def _standalone_invite_cutoff(now_utc: datetime, expiry_days: int) -> datetime: + """Compute a diagnostics-only cutoff with a fail-fast configuration error.""" + try: + return now_utc - timedelta(days=expiry_days) + except OverflowError as exc: + raise ValueError( + "INSPECT_AUTH_INVITE_EXPIRY_DAYS is outside the supported range for the current reference clock" + ) from exc + + def validate_inspect_config(config: Any) -> InspectAuthSettings: """Validate inspect-auth config before database connections are opened.""" batches = _coerce_int_setting(config, "AUTH_REPORT_BATCHES") @@ -140,11 +187,48 @@ def validate_inspect_config(config: Any) -> InspectAuthSettings: raise ValueError("AUTH_REPORT_BATCH_SIZE must be greater than 0") inspect_filter = parse_inspect_filter(getattr(config, "INSPECT_AUTH_FILTER", None), "INSPECT_AUTH_FILTER") + invite_expiry_days = parse_invite_expiry_days( + getattr(config, "INSPECT_AUTH_INVITE_EXPIRY_DAYS", None) + ) + raw_invite_criteria = blank_to_none(getattr(config, "INSPECT_AUTH_INVITE_CRITERIA", None)) + if ( + raw_invite_criteria is not None + and inspect_filter != INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA + ): + raise ValueError( + "INSPECT_AUTH_INVITE_CRITERIA only applies to " + "NO_AFFILIATION_INVITE_CRITERIA" + ) + + now_utc = datetime.now(timezone.utc).replace(tzinfo=None, microsecond=0) + invite_criteria = parse_invite_criteria( + raw_invite_criteria, + now=now_utc, + expiry_days=invite_expiry_days, + ) + if ( + inspect_filter == INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA + and invite_criteria is None + ): + raise ValueError( + "INSPECT_AUTH_INVITE_CRITERIA must contain at least one clause when " + "INSPECT_AUTH_FILTER=NO_AFFILIATION_INVITE_CRITERIA" + ) + + invite_expiry_cutoff = ( + invite_criteria.cutoff + if invite_criteria is not None + else _standalone_invite_cutoff(now_utc, invite_expiry_days) + ) + console_limit = parse_console_limit( getattr(config, INSPECT_AUTH_CONSOLE_LIMIT_ATTR, None), INSPECT_AUTH_CONSOLE_LIMIT_ATTR, DEFAULT_INSPECT_AUTH_CONSOLE_LIMIT, ) + business_names_max_length = parse_business_names_max_length( + getattr(config, INSPECT_AUTH_BUSINESS_NAMES_MAX_LENGTH_ATTR, None) + ) inspection_path, summary_path = _resolve_inspection_paths(config) selection_mode = parse_auth_selection_mode(config) @@ -170,6 +254,7 @@ def validate_inspect_config(config: Any) -> InspectAuthSettings: batch_size=batch_size, inspect_filter=inspect_filter, console_limit=console_limit, + business_names_max_length=business_names_max_length, inspection_path=inspection_path, summary_path=summary_path, selection_mode=selection_mode, @@ -177,6 +262,10 @@ def validate_inspect_config(config: Any) -> InspectAuthSettings: auth_mig_group_ids=auth_mig_group_ids, auth_mig_batch_ids=auth_mig_batch_ids, manual_account_ids=manual_account_ids, + run_clock=now_utc, + invite_expiry_days=invite_expiry_days, + invite_expiry_cutoff=invite_expiry_cutoff, + invite_criteria=invite_criteria, ) @@ -240,10 +329,36 @@ def _submit_inspect_batch( def _format_start_message(settings: InspectAuthSettings) -> str: + criteria_fragment = "" + if settings.invite_criteria is not None: + criteria_fragment = ( + f"{format_invite_criteria(settings.invite_criteria)}, " + "CriteriaRole=filter, " + "AgeRule=timestamp-precision vs Now; " + "PositionRule=1=oldest, ties by (sent_date, entity_id, id); " + "NULL sent_date fails age clauses and all_expired; " + ) + cutoff_text = format_clock_z(settings.invite_cutoff) + expiry_role = ( + "filter-clause" + if settings.invite_criteria is not None and settings.invite_criteria.requires_expiry() + else "diagnostics-only" + ) + expiry_parameters = ( + "" + if settings.invite_criteria is not None + else f"ExpiryDays={settings.invite_expiry_days}, Cutoff={cutoff_text}, " + ) + expiry_fragment = ( + f"{expiry_parameters}ExpiryRole={expiry_role}, " + "CutoffClock=UTC once-per-run, " + ) return ( "👷 Inspect Auth starting. " f"SelectionMode={settings.selection_mode.value}, " f"InspectFilter={settings.inspect_filter}, " + f"{criteria_fragment}" + f"{expiry_fragment}" f"AuthReadChecks={','.join(settings.auth_read_checks)}, " f"ConsoleLimit={format_console_limit(settings.console_limit)}, " f"ConfiguredBatches={settings.batches}, BatchSize={settings.batch_size}, " @@ -251,12 +366,25 @@ def _format_start_message(settings: InspectAuthSettings) -> str: ) +def _resolve_inspect_batch_states(futures: list[Any]) -> list[Any]: + """Resolve submitted inspect batches into one ordered state list.""" + states = [] + for future in futures: + batch_result = resolve_task_result(future) + if isinstance(batch_result, AuthBatchResult): + states.extend(batch_result.states) + else: + # Compatibility for simple task doubles that return raw AuthBusinessState lists. + states.extend(batch_result or []) + return states + + def _run_inspect_auth_flow_with_engines( config: Any, settings: InspectAuthSettings, colin_extract_engine: Engine, auth_engine: Engine, -) -> dict[str, int | str]: +) -> dict[str, Any]: """Run inspect-auth orchestration using already-initialized engines.""" total_candidates = get_inspect_candidate_count_task(config, colin_extract_engine, settings) total_candidates = int(resolve_task_result(total_candidates) or 0) @@ -306,31 +434,47 @@ def _run_inspect_auth_flow_with_engines( ) ) - states = [] - for future in futures: - batch_result = resolve_task_result(future) - if isinstance(batch_result, AuthBatchResult): - states.extend(batch_result.states) - else: - # Compatibility for simple task doubles that return raw AuthBusinessState lists. - states.extend(batch_result or []) + states = _resolve_inspect_batch_states(futures) - matched_states = filter_inspection_states(states, settings.inspect_filter) - inspection_rows = build_inspection_rows(matched_states) - identifier_rows = build_inspection_identifier_rows(states, matched_states, settings.inspect_filter) - summary = build_inspection_summary(states, matched_states, settings.inspect_filter) + matched_states = filter_inspection_states( + states, settings.inspect_filter, settings.invite_criteria + ) + inspection_rows = build_inspection_rows( + matched_states, + settings.invite_criteria, + reference_now=settings.run_clock, + ) + identifier_rows = build_inspection_identifier_rows( + states, matched_states, settings.inspect_filter, settings.invite_criteria + ) + summary = build_inspection_summary( + states, matched_states, settings.inspect_filter, settings.invite_criteria + ) + summary["expiry_days"] = settings.invite_expiry_days + summary["expiry_cutoff"] = settings.invite_cutoff print( "🌟 Inspect Auth tasks complete. " f"TotalInspected={summary['inspected_count']}, Matched={summary['matched_count']}, " f"IdentifierGroups={len(identifier_rows)}" ) - print_inspection_summary(states, matched_states, settings.inspect_filter) + print_inspection_summary( + states, matched_states, settings.inspect_filter, settings.invite_criteria + ) + run_clock_text = format_clock_minutes_z(settings.run_clock) + invite_cutoff_text = format_clock_minutes_z(settings.invite_cutoff) + print( + "🕐 Invite clock: " + f"Now={run_clock_text} Cutoff={invite_cutoff_text} " + f"ExpiryDays={settings.invite_expiry_days} " + "(expired means sent_date < Cutoff; one clock per run)" + ) print_inspection_rows( inspection_rows, console_limit=settings.console_limit, report_path=settings.inspection_path, limit_config_key=INSPECT_AUTH_CONSOLE_LIMIT_ATTR, + business_names_max_length=settings.business_names_max_length, ) print_inspection_identifier_rows( identifier_rows, @@ -340,8 +484,27 @@ def _run_inspect_auth_flow_with_engines( limit_config_key=INSPECT_AUTH_CONSOLE_LIMIT_ATTR, ) - write_inspection_report(settings.inspection_path, matched_states) - write_inspection_summary_txt(settings.summary_path, identifier_rows, summary) + write_inspection_report( + settings.inspection_path, + matched_states, + settings.invite_criteria, + reference_now=settings.run_clock, + ) + write_inspection_summary_txt( + settings.summary_path, + identifier_rows, + summary, + handoff_filter=settings.inspect_filter, + ) + if ( + settings.inspect_filter == INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA + and identifier_rows + and int(identifier_rows[0].get("count", 0) or 0) > 0 + ): + print( + f"➡️ Reminder handoff block written to {settings.summary_path} — " + "set a new AUTH_REPEATABLE_CYCLE_KEY before running make run-auth-invite." + ) print( "🌰 Inspect Auth reports written. " f"InspectionPath={settings.inspection_path}, SummaryPath={settings.summary_path}" @@ -350,7 +513,7 @@ def _run_inspect_auth_flow_with_engines( return summary -def _initialize_and_run_inspect_auth_flow() -> dict[str, int | str]: +def _initialize_and_run_inspect_auth_flow() -> dict[str, Any]: """Validate config, initialize engines once, and run inspect-auth orchestration.""" from common.init_utils import auth_init, colin_extract_init, get_config @@ -374,7 +537,7 @@ def _initialize_and_run_inspect_auth_flow() -> dict[str, int | str]: @flow(name="Inspect-Auth-Flow", log_prints=True) -def inspect_auth_flow() -> dict[str, int | str]: +def inspect_auth_flow() -> dict[str, Any]: """Validate config, inspect selected Auth DB candidates, and write report.""" return _initialize_and_run_inspect_auth_flow() diff --git a/data-tool/flows/auth/verify_auth_flow.py b/data-tool/flows/auth/verify_auth_flow.py index 6aefea631a..3f002e95d5 100644 --- a/data-tool/flows/auth/verify_auth_flow.py +++ b/data-tool/flows/auth/verify_auth_flow.py @@ -17,8 +17,11 @@ import inspect import logging import math -from collections import defaultdict -from dataclasses import dataclass +import re +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from enum import Enum from pathlib import Path from typing import Any, Callable, Iterable, Protocol @@ -90,6 +93,7 @@ def decorator(fn): INSPECT_FILTER_ENTITY_WITHOUT_AFFILIATION = "ENTITY_WITHOUT_AFFILIATION" INSPECT_FILTER_HAS_INVITE = "HAS_INVITE" INSPECT_FILTER_ENTITY_WITHOUT_INVITE = "ENTITY_WITHOUT_INVITE" +INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA = "NO_AFFILIATION_INVITE_CRITERIA" INSPECT_FILTERS = ( INSPECT_FILTER_ALL, INSPECT_FILTER_HAS_ANY_AUTH, @@ -101,6 +105,7 @@ def decorator(fn): INSPECT_FILTER_ENTITY_WITHOUT_AFFILIATION, INSPECT_FILTER_HAS_INVITE, INSPECT_FILTER_ENTITY_WITHOUT_INVITE, + INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA, ) INSPECT_IDENTIFIER_FILTERS = tuple( inspect_filter for inspect_filter in INSPECT_FILTERS if inspect_filter != INSPECT_FILTER_ALL @@ -129,6 +134,7 @@ def decorator(fn): ANSI_BOLD_RED = "\033[1;31m" ANSI_RESET = "\033[0m" IDENTIFIER_GROUP_SEPARATOR = "─" * 88 +_BLANK_DIAGNOSTIC_BUCKET = "(blank)" _DEPENDENT_CHECKS = (CHECK_CONTACT, CHECK_AFFILIATION, CHECK_INVITE) _ALL_CHECKS = (CHECK_ENTITY, CHECK_CONTACT, CHECK_AFFILIATION, CHECK_INVITE) @@ -197,6 +203,22 @@ def decorator(fn): "invite_state", "invite_count", "blocked_by_missing_entity", + "invite_claimed_count", + "invite_unclaimed_count", + "invite_deleted_count", + "invite_type_counts", + "invite_status_counts", + "unaffiliated_unclaimed_invite_count", + "unaffiliated_unclaimed_expired_count", + "unaffiliated_unclaimed_current_count", + "unaffiliated_unclaimed_unknown_age_count", + "oldest_unclaimed_sent_date", + "newest_unclaimed_sent_date", + "latest_accepted_date", + "unaffiliated_unclaimed_sent_dates", + "unaffiliated_unclaimed_ages_days", + "invite_criteria_pass", + "invite_criteria_failed_clauses", ] _ENTITY_READ_SQL = """ @@ -223,10 +245,11 @@ def decorator(fn): """.strip() _INVITE_READ_SQL = """ - SELECT entity_id, COUNT(*) AS invite_count + SELECT id, entity_id, type, invitation_status_code, + sent_date, accepted_date, is_deleted, affiliation_id FROM affiliation_invitations WHERE entity_id IN :entity_ids - GROUP BY entity_id + ORDER BY entity_id, id """.strip() @@ -246,6 +269,11 @@ def auth_read_checks(self) -> tuple[str, ...]: """Return Auth DB table read scope for this batch.""" ... + @property + def invite_cutoff(self) -> datetime | None: + """Return the optional UTC cutoff used for invitation age diagnostics.""" + ... + class AuthCandidateSettings(Protocol): """Structural settings surface consumed by shared candidate/account helpers.""" @@ -309,6 +337,11 @@ def auth_read_checks(self) -> tuple[str, ...]: """Return Auth DB table read scope for verification.""" return self.selected_checks + @property + def invite_cutoff(self) -> datetime | None: + """Disable age-bucket diagnostics for verify-auth.""" + return None + @dataclass(frozen=True) class AuthBatchResult: @@ -364,6 +397,649 @@ def statement(self) -> TextClause: return _build_text_statement(self.sql, self.expanding_bind_names) +class InviteClauseKind(Enum): + """Supported AND-composed invitation criteria clause kinds.""" + + COUNT = "count" + ALL_EXPIRED = "all_expired" + AGE = "age" + + +@dataclass(frozen=True) +class InviteClause: + """One normalized invitation criteria clause.""" + + kind: InviteClauseKind + raw: str + op: str | None = None + value: int | None = None + position: int | None = None + + +@dataclass(frozen=True) +class InviteCriteria: + """Parsed invitation criteria sharing one reference clock.""" + + clauses: tuple[InviteClause, ...] + raw: str + now: datetime + expiry_days: int | None = None + cutoff: datetime | None = None + + def requires_expiry(self) -> bool: + """Return whether the expression contains an ``all_expired`` clause.""" + return any(clause.kind is InviteClauseKind.ALL_EXPIRED for clause in self.clauses) + + def has_age_clauses(self) -> bool: + """Return whether the expression contains an elapsed-age clause.""" + return any(clause.kind is InviteClauseKind.AGE for clause in self.clauses) + + +@dataclass(frozen=True) +class AuthInviteDiagnostics: + """Aggregated, non-sensitive diagnostics using exclusive accepted > deleted > unclaimed buckets.""" + + total_count: int = 0 + deleted_count: int = 0 + claimed_count: int = 0 + unclaimed_count: int = 0 + type_counts: tuple[tuple[str, int], ...] = () + status_counts: tuple[tuple[str, int], ...] = () + unaffiliated_unclaimed_count: int = 0 + expired_unclaimed_unaffiliated_count: int | None = None + current_unclaimed_unaffiliated_count: int | None = None + unknown_age_unclaimed_unaffiliated_count: int = 0 + oldest_unclaimed_sent_date: datetime | None = None + newest_unclaimed_sent_date: datetime | None = None + latest_accepted_date: datetime | None = None + unaffiliated_unclaimed_sent_dates: tuple[datetime, ...] = () + + +def _as_utc_naive(value: datetime | None) -> datetime | None: + """Normalize Auth timestamps to naive UTC for stable comparisons and output.""" + if value is None: + return None + if value.tzinfo is not None and value.utcoffset() is not None: + return value.astimezone(timezone.utc).replace(tzinfo=None) + return value + + +def format_clock_z(value: datetime | None) -> str: + """Render an operator-facing clock as second-precision UTC-naive ISO-8601 plus Z.""" + normalized = _as_utc_naive(value) + return f"{normalized.isoformat(timespec='seconds')}Z" if normalized is not None else "" + + +def format_clock_minutes_z(value: datetime | None) -> str: + """Render an operator-facing clock as minute-precision UTC-naive ISO-8601 plus Z.""" + normalized = _as_utc_naive(value) + return f"{normalized.isoformat(timespec='minutes')}Z" if normalized is not None else "" + + +_CRITERIA_OPERATOR_NAMES = { + "==": "EQ", + ">=": "GTE", + "<=": "LTE", + ">": "GT", + "<": "LT", +} +_CRITERIA_OPERATORS = "== >= <= > <" +_AGE_OPERATORS = ">= > <= <" +_COUNT_CLAUSE_RE = re.compile(r"^count\s*(==|>=|<=|>|<)\s*([+-]?\d+)\s*$", re.IGNORECASE) +_AGE_CLAUSE_RE = re.compile( + r"^(age\s*\[\s*([+-]?\d+)\s*\]|oldest_age|newest_age|all_ages|any_age)" + r"\s*(==|>=|<=|>|<)\s*([+-]?\d+)\s*$", + re.IGNORECASE, +) + + +def _criteria_clause_error(config_key: str, index: int, raw_clause: str, message: str) -> ValueError: + """Build a consistently contextual criteria parser error.""" + return ValueError(f"{config_key} clause {index} {raw_clause!r}: {message}") + + +def _positive_clause_value( + value_text: str, + *, + config_key: str, + index: int, + raw_clause: str, +) -> int: + """Parse the shared strictly-positive criteria value domain.""" + value = int(value_text) + if value <= 0: + raise _criteria_clause_error( + config_key, + index, + raw_clause, + "value must be a positive integer; use ENTITY_WITHOUT_INVITE for zero invites", + ) + return value + + +def _parse_count_invite_clause( + raw_clause: str, + *, + config_key: str, + index: int, +) -> InviteClause | None: + """Parse a count clause, returning None when the clause has another selector.""" + count_match = _COUNT_CLAUSE_RE.fullmatch(raw_clause) + if count_match: + operator, value_text = count_match.groups() + value = _positive_clause_value( + value_text, + config_key=config_key, + index=index, + raw_clause=raw_clause, + ) + return InviteClause( + InviteClauseKind.COUNT, + f"count{operator}{value}", + op=_CRITERIA_OPERATOR_NAMES[operator], + value=value, + ) + + lowered = raw_clause.lower() + if not lowered.startswith("count"): + return None + operator_match = re.match(r"^count\s*([^\d\s+-]+)", lowered) + if operator_match: + operator = operator_match.group(1) + raise _criteria_clause_error( + config_key, + index, + raw_clause, + f"unknown operator {operator!r}; expected one of {_CRITERIA_OPERATORS}", + ) + raise _criteria_clause_error( + config_key, + index, + raw_clause, + f"expected count followed by one of {_CRITERIA_OPERATORS} and a positive integer", + ) + + +def _age_clause_position( + selector: str, + position_text: str | None, + operator: str, + *, + config_key: str, + index: int, + raw_clause: str, +) -> int | None: + """Normalize an age selector to its oldest-first tuple position.""" + normalized_selector = selector.lower() + if normalized_selector.startswith("age"): + position = int(position_text or 0) + if position < 1: + raise _criteria_clause_error( + config_key, + index, + raw_clause, + "position must be >= 1 (1 = oldest)", + ) + return position + if normalized_selector == "oldest_age": + return 1 + if normalized_selector == "newest_age": + return None + if normalized_selector == "all_ages": + return None if operator in (">=", ">") else 1 + return 1 if operator in (">=", ">") else None + + +def _parse_age_invite_clause( + raw_clause: str, + *, + config_key: str, + index: int, +) -> InviteClause | None: + """Parse an age clause, returning None when the clause has another selector.""" + age_match = _AGE_CLAUSE_RE.fullmatch(raw_clause) + if age_match: + selector, position_text, operator, value_text = age_match.groups() + if operator == "==": + raise _criteria_clause_error( + config_key, + index, + raw_clause, + f"age clauses accept {_AGE_OPERATORS} only", + ) + value = _positive_clause_value( + value_text, + config_key=config_key, + index=index, + raw_clause=raw_clause, + ) + position = _age_clause_position( + selector, + position_text, + operator, + config_key=config_key, + index=index, + raw_clause=raw_clause, + ) + selector_text = "newest_age" if position is None else f"age[{position}]" + return InviteClause( + InviteClauseKind.AGE, + f"{selector_text}{operator}{value}", + op=_CRITERIA_OPERATOR_NAMES[operator], + value=value, + position=position, + ) + + if re.match( + r"^(age\s*\[|oldest_age|newest_age|all_ages(?:\s|[<>=]|$)|any_age(?:\s|[<>=]|$))", + raw_clause.lower(), + ): + raise _criteria_clause_error( + config_key, + index, + raw_clause, + f"expected an age selector followed by one of {_AGE_OPERATORS} and a positive integer", + ) + return None + + +def _parse_invite_clause(raw_part: str, *, config_key: str, index: int) -> InviteClause: + """Parse one criteria clause while preserving specialized validation precedence.""" + raw_clause = raw_part.strip() + if not raw_clause: + raise _criteria_clause_error( + config_key, + index, + raw_clause, + "must contain a clause between commas", + ) + + lowered = raw_clause.lower() + if lowered == "all_expired": + return InviteClause(InviteClauseKind.ALL_EXPIRED, "all_expired") + if re.match(r"^all_expired\s*[<>=]", lowered): + raise _criteria_clause_error( + config_key, + index, + raw_clause, + "all_expired takes no operator or value", + ) + + count_clause = _parse_count_invite_clause(raw_clause, config_key=config_key, index=index) + if count_clause is not None: + return count_clause + age_clause = _parse_age_invite_clause(raw_clause, config_key=config_key, index=index) + if age_clause is not None: + return age_clause + raise _criteria_clause_error( + config_key, + index, + raw_clause, + "unknown clause; expected count, all_expired, age[n], oldest_age, newest_age, all_ages, any_age", + ) + + +def _require_all_expired_config( + clauses: list[InviteClause], + raw_clauses: list[str], + *, + expiry_days: int | None, + config_key: str, +) -> None: + """Require an expiry window for the first all-expired clause.""" + if expiry_days is not None: + return + for index, clause in enumerate(clauses, start=1): + if clause.kind is InviteClauseKind.ALL_EXPIRED: + raise _criteria_clause_error( + config_key, + index, + raw_clauses[index - 1].strip(), + "all_expired requires INSPECT_AUTH_INVITE_EXPIRY_DAYS", + ) + + +def _validated_criteria_clock( + now: datetime, + clauses: list[InviteClause], + *, + config_key: str, +) -> datetime: + """Normalize the criteria clock and validate every elapsed-age threshold.""" + normalized_now = _as_utc_naive(now) + if normalized_now is None: # Defensive: the public signature requires a datetime. + raise ValueError(f"{config_key} requires a reference clock") + for index, clause in enumerate(clauses, start=1): + if clause.kind is not InviteClauseKind.AGE: + continue + try: + normalized_now - timedelta(days=clause.value or 0) + except OverflowError as exc: + raise _criteria_clause_error( + config_key, + index, + clause.raw, + "day value is outside the supported range for the reference clock", + ) from exc + return normalized_now + + +def _criteria_cutoff(normalized_now: datetime, expiry_days: int | None) -> datetime | None: + """Build the optional expiry cutoff with the established config error.""" + try: + return normalized_now - timedelta(days=expiry_days) if expiry_days is not None else None + except OverflowError as exc: + raise ValueError( + "INSPECT_AUTH_INVITE_EXPIRY_DAYS is outside the supported range for the criteria reference clock" + ) from exc + + +def parse_invite_criteria( + raw_value: str | None, + *, + now: datetime, + expiry_days: int | None, + config_key: str = "INSPECT_AUTH_INVITE_CRITERIA", +) -> InviteCriteria | None: + """Parse a comma-separated, AND-composed invitation criteria expression.""" + if raw_value is None or not str(raw_value).strip(): + return None + + raw_clauses = str(raw_value).split(",") + clauses = [ + _parse_invite_clause(raw_part, config_key=config_key, index=index) + for index, raw_part in enumerate(raw_clauses, start=1) + ] + _require_all_expired_config( + clauses, + raw_clauses, + expiry_days=expiry_days, + config_key=config_key, + ) + normalized_now = _validated_criteria_clock(now, clauses, config_key=config_key) + return InviteCriteria( + clauses=tuple(clauses), + raw=", ".join(clause.raw for clause in clauses), + now=normalized_now, + expiry_days=expiry_days, + cutoff=_criteria_cutoff(normalized_now, expiry_days), + ) + + +def _compare_invite_count(actual: int, op: str, expected: int) -> bool: + """Apply a normalized count-clause comparison.""" + comparisons = { + "EQ": actual == expected, + "GTE": actual >= expected, + "LTE": actual <= expected, + "GT": actual > expected, + "LT": actual < expected, + } + return comparisons[op] + + +def _compare_invite_age(sent_date: datetime, op: str, threshold: datetime) -> bool: + """Compare a sent timestamp against an elapsed-age threshold.""" + comparisons = { + "GTE": sent_date <= threshold, + "GT": sent_date < threshold, + "LTE": sent_date >= threshold, + "LT": sent_date > threshold, + } + return comparisons[op] + + +def _age_clause_sent_date( + dated_invites: tuple[datetime, ...], + position: int | None, +) -> datetime | None: + """Select an oldest-first positional date or the newest endpoint.""" + if position is None: + return dated_invites[-1] + if len(dated_invites) >= position: + return dated_invites[position - 1] + return None + + +def _evaluate_invite_clause( + clause: InviteClause, + diagnostics: AuthInviteDiagnostics, + criteria: InviteCriteria, + dated_invites: tuple[datetime, ...], +) -> bool: + """Evaluate one normalized invitation clause.""" + if clause.kind is InviteClauseKind.COUNT: + return _compare_invite_count( + diagnostics.unaffiliated_unclaimed_count, + clause.op or "", + clause.value or 0, + ) + if clause.kind is InviteClauseKind.ALL_EXPIRED: + return ( + diagnostics.unaffiliated_unclaimed_count > 0 + and diagnostics.unknown_age_unclaimed_unaffiliated_count == 0 + and criteria.cutoff is not None + and diagnostics.expired_unclaimed_unaffiliated_count + == diagnostics.unaffiliated_unclaimed_count + ) + if diagnostics.unknown_age_unclaimed_unaffiliated_count != 0 or not dated_invites: + return False + sent_date = _age_clause_sent_date(dated_invites, clause.position) + if sent_date is None: + return False + threshold = criteria.now - timedelta(days=clause.value or 0) + return _compare_invite_age(sent_date, clause.op or "", threshold) + + +def evaluate_invite_clauses( + diagnostics: AuthInviteDiagnostics, + criteria: InviteCriteria, +) -> tuple[tuple[InviteClause, bool], ...]: + """Evaluate every clause, preserving order for predicates and diagnostics.""" + dated_invites = tuple( + normalized + for sent_date in diagnostics.unaffiliated_unclaimed_sent_dates + if (normalized := _as_utc_naive(sent_date)) is not None + ) + return tuple( + (clause, _evaluate_invite_clause(clause, diagnostics, criteria, dated_invites)) + for clause in criteria.clauses + ) + + +def format_invite_criteria(criteria: InviteCriteria) -> str: + """Render criteria as a stable, self-describing report/log fragment.""" + rendered = f'Criteria="{criteria.raw}", Now={format_clock_z(criteria.now)}' + if criteria.expiry_days is not None: + rendered += ( + f", ExpiryDays={criteria.expiry_days}, Cutoff={format_clock_z(criteria.cutoff)}" + ) + return rendered + + +def _diagnostic_bucket(value: Any) -> str: + """Return a deterministic display bucket for invitation type/status values.""" + if value is None or not str(value).strip(): + return _BLANK_DIAGNOSTIC_BUCKET + return str(value).strip() + + +@dataclass +class _AuthInviteAccumulator: + """Mutable per-entity invitation fold discarded after state construction.""" + + total_count: int = 0 + deleted_count: int = 0 + claimed_count: int = 0 + unclaimed_count: int = 0 + type_counts: Counter[str] = field(default_factory=Counter) + status_counts: Counter[str] = field(default_factory=Counter) + unaffiliated_unclaimed_count: int = 0 + expired_unclaimed_unaffiliated_count: int = 0 + current_unclaimed_unaffiliated_count: int = 0 + unknown_age_unclaimed_unaffiliated_count: int = 0 + unaffiliated_unclaimed_dated: list[tuple[datetime, str, Any]] = field(default_factory=list) + oldest_unclaimed_sent_date: datetime | None = None + newest_unclaimed_sent_date: datetime | None = None + latest_accepted_date: datetime | None = None + + def _record_bucket_counts(self, invite_type: Any, invitation_status_code: Any) -> str: + """Record the non-deleted type/status mix and return the normalized type.""" + type_bucket = _diagnostic_bucket(invite_type) + self.type_counts[type_bucket] += 1 + self.status_counts[_diagnostic_bucket(invitation_status_code)] += 1 + return type_bucket + + def _record_accepted( + self, + invite_type: Any, + invitation_status_code: Any, + accepted_date: datetime, + ) -> None: + """Record an accepted invitation, which takes precedence over deletion.""" + self._record_bucket_counts(invite_type, invitation_status_code) + self.claimed_count += 1 + if self.latest_accepted_date is None or accepted_date > self.latest_accepted_date: + self.latest_accepted_date = accepted_date + + def _record_unclaimed_sent_date(self, sent_date: datetime) -> None: + """Update the all-type unclaimed invitation date range.""" + if self.oldest_unclaimed_sent_date is None or sent_date < self.oldest_unclaimed_sent_date: + self.oldest_unclaimed_sent_date = sent_date + if self.newest_unclaimed_sent_date is None or sent_date > self.newest_unclaimed_sent_date: + self.newest_unclaimed_sent_date = sent_date + + def _record_unaffiliated_unclaimed( + self, + sent_date: datetime | None, + invite_cutoff: datetime | None, + entity_id: Any, + invitation_id: Any, + ) -> None: + """Record qualifying unaffiliated-unclaimed age diagnostics.""" + self.unaffiliated_unclaimed_count += 1 + if sent_date is None: + self.unknown_age_unclaimed_unaffiliated_count += 1 + return + + self.unaffiliated_unclaimed_dated.append( + (sent_date, str(entity_id) if entity_id is not None else "", invitation_id) + ) + if invite_cutoff is None: + return + if sent_date < invite_cutoff: + self.expired_unclaimed_unaffiliated_count += 1 + else: + self.current_unclaimed_unaffiliated_count += 1 + + def _record_unclaimed( + self, + invite_type: Any, + invitation_status_code: Any, + sent_date: datetime | None, + invite_cutoff: datetime | None, + entity_id: Any, + invitation_id: Any, + ) -> None: + """Record one non-deleted, unaccepted invitation.""" + type_bucket = self._record_bucket_counts(invite_type, invitation_status_code) + self.unclaimed_count += 1 + normalized_sent_date = _as_utc_naive(sent_date) + if normalized_sent_date is not None: + self._record_unclaimed_sent_date(normalized_sent_date) + if type_bucket == "UNAFFILIATED_EMAIL": + self._record_unaffiliated_unclaimed( + normalized_sent_date, + invite_cutoff, + entity_id, + invitation_id, + ) + + def add_row( + self, + *, + invite_type: Any, + invitation_status_code: Any, + sent_date: datetime | None, + accepted_date: datetime | None, + is_deleted: Any, + invite_cutoff: datetime | None, + entity_id: Any = None, + invitation_id: Any = None, + ) -> None: + """Fold one row using accepted > deleted > unclaimed lifecycle precedence.""" + self.total_count += 1 + normalized_accepted_date = _as_utc_naive(accepted_date) + if normalized_accepted_date is not None: + self._record_accepted(invite_type, invitation_status_code, normalized_accepted_date) + return + if bool(is_deleted): + self.deleted_count += 1 + return + self._record_unclaimed( + invite_type, + invitation_status_code, + sent_date, + invite_cutoff, + entity_id, + invitation_id, + ) + + def merge(self, other: _AuthInviteAccumulator) -> None: + """Merge one entity accumulator into a business-level accumulator.""" + self.total_count += other.total_count + self.deleted_count += other.deleted_count + self.claimed_count += other.claimed_count + self.unclaimed_count += other.unclaimed_count + self.type_counts.update(other.type_counts) + self.status_counts.update(other.status_counts) + self.unaffiliated_unclaimed_count += other.unaffiliated_unclaimed_count + self.expired_unclaimed_unaffiliated_count += other.expired_unclaimed_unaffiliated_count + self.current_unclaimed_unaffiliated_count += other.current_unclaimed_unaffiliated_count + self.unknown_age_unclaimed_unaffiliated_count += other.unknown_age_unclaimed_unaffiliated_count + self.unaffiliated_unclaimed_dated.extend(other.unaffiliated_unclaimed_dated) + for sent_date in (other.oldest_unclaimed_sent_date, other.newest_unclaimed_sent_date): + if sent_date is not None: + if self.oldest_unclaimed_sent_date is None or sent_date < self.oldest_unclaimed_sent_date: + self.oldest_unclaimed_sent_date = sent_date + if self.newest_unclaimed_sent_date is None or sent_date > self.newest_unclaimed_sent_date: + self.newest_unclaimed_sent_date = sent_date + if other.latest_accepted_date is not None and ( + self.latest_accepted_date is None or other.latest_accepted_date > self.latest_accepted_date + ): + self.latest_accepted_date = other.latest_accepted_date + + def freeze(self, *, cutoff_configured: bool) -> AuthInviteDiagnostics: + """Return the immutable public diagnostics representation.""" + ordered_sent_dates = tuple( + sent_date + for sent_date, _entity_id, _invitation_id in sorted( + self.unaffiliated_unclaimed_dated, + key=_dated_invite_sort_key, + ) + ) + return AuthInviteDiagnostics( + total_count=self.total_count, + deleted_count=self.deleted_count, + claimed_count=self.claimed_count, + unclaimed_count=self.unclaimed_count, + type_counts=tuple(sorted(self.type_counts.items())), + status_counts=tuple(sorted(self.status_counts.items())), + unaffiliated_unclaimed_count=self.unaffiliated_unclaimed_count, + expired_unclaimed_unaffiliated_count=( + self.expired_unclaimed_unaffiliated_count if cutoff_configured else None + ), + current_unclaimed_unaffiliated_count=( + self.current_unclaimed_unaffiliated_count if cutoff_configured else None + ), + unknown_age_unclaimed_unaffiliated_count=self.unknown_age_unclaimed_unaffiliated_count, + unaffiliated_unclaimed_sent_dates=ordered_sent_dates, + oldest_unclaimed_sent_date=self.oldest_unclaimed_sent_date, + newest_unclaimed_sent_date=self.newest_unclaimed_sent_date, + latest_accepted_date=self.latest_accepted_date, + ) + + @dataclass(frozen=True) class AuthBusinessState: """Aggregated Auth DB state for one selected business identifier.""" @@ -378,6 +1054,7 @@ class AuthBusinessState: missing_account_ids: tuple[str, ...] = () invite_count: int = 0 business_names: tuple[str, ...] = () + invite_diagnostics: AuthInviteDiagnostics = field(default_factory=AuthInviteDiagnostics) @property def entity_exists(self) -> bool: @@ -399,39 +1076,85 @@ def auth_state_has_any_auth(state: AuthBusinessState) -> bool: ) -def auth_state_matches_inspect_filter(state: AuthBusinessState, inspect_filter: str) -> bool: +def _require_invite_filter_criteria( + inspect_filter: str, + invite_criteria: InviteCriteria | None, +) -> None: + """Fail fast when the criteria-dependent token is evaluated without criteria.""" + if inspect_filter == INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA and invite_criteria is None: + raise ValueError( + f"{INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA} requires " + "INSPECT_AUTH_INVITE_CRITERIA" + ) + + +def _auth_state_matches_invite_criteria( + state: AuthBusinessState, + clause_results: tuple[tuple[InviteClause, bool], ...], +) -> bool: + """Apply the composite filter floor and base predicates to evaluated clauses.""" + return ( + state.entity_exists + and not state.found_account_ids + and state.invite_diagnostics.unaffiliated_unclaimed_count >= 1 + and all(passed for _clause, passed in clause_results) + ) + + +_STANDARD_INSPECT_FILTER_MATCHERS: dict[str, Callable[[AuthBusinessState], bool]] = { + INSPECT_FILTER_ALL: lambda _state: True, + INSPECT_FILTER_HAS_ANY_AUTH: auth_state_has_any_auth, + INSPECT_FILTER_HAS_ENTITY: lambda state: state.entity_exists, + INSPECT_FILTER_MISSING_ENTITY: lambda state: not state.entity_exists, + INSPECT_FILTER_HAS_CONTACT: lambda state: state.entity_exists and state.usable_contact_count > 0, + INSPECT_FILTER_ENTITY_WITHOUT_CONTACT: ( + lambda state: state.entity_exists and state.usable_contact_count == 0 + ), + INSPECT_FILTER_HAS_AFFILIATION: lambda state: state.entity_exists and bool(state.found_account_ids), + INSPECT_FILTER_ENTITY_WITHOUT_AFFILIATION: ( + lambda state: state.entity_exists and not state.found_account_ids + ), + INSPECT_FILTER_HAS_INVITE: lambda state: state.entity_exists and state.invite_count > 0, + INSPECT_FILTER_ENTITY_WITHOUT_INVITE: lambda state: state.entity_exists and state.invite_count == 0, +} + + +def auth_state_matches_inspect_filter( + state: AuthBusinessState, + inspect_filter: str, + invite_criteria: InviteCriteria | None = None, +) -> bool: """Evaluate one AuthBusinessState against a validated inspect filter. ``HAS_CONTACT`` and ``ENTITY_WITHOUT_CONTACT`` intentionally use ``usable_contact_count`` (non-blank email), not raw ``contact_count``. """ - if inspect_filter == INSPECT_FILTER_ALL: - return True - if inspect_filter == INSPECT_FILTER_HAS_ANY_AUTH: - return auth_state_has_any_auth(state) - if inspect_filter == INSPECT_FILTER_HAS_ENTITY: - return state.entity_exists - if inspect_filter == INSPECT_FILTER_MISSING_ENTITY: - return not state.entity_exists - if inspect_filter == INSPECT_FILTER_HAS_CONTACT: - return state.entity_exists and state.usable_contact_count > 0 - if inspect_filter == INSPECT_FILTER_ENTITY_WITHOUT_CONTACT: - return state.entity_exists and state.usable_contact_count == 0 - if inspect_filter == INSPECT_FILTER_HAS_AFFILIATION: - return state.entity_exists and bool(state.found_account_ids) - if inspect_filter == INSPECT_FILTER_ENTITY_WITHOUT_AFFILIATION: - return state.entity_exists and not state.found_account_ids - if inspect_filter == INSPECT_FILTER_HAS_INVITE: - return state.entity_exists and state.invite_count > 0 - if inspect_filter == INSPECT_FILTER_ENTITY_WITHOUT_INVITE: - return state.entity_exists and state.invite_count == 0 + _require_invite_filter_criteria(inspect_filter, invite_criteria) + matcher = _STANDARD_INSPECT_FILTER_MATCHERS.get(inspect_filter) + if matcher is not None: + return matcher(state) + if inspect_filter == INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA: + assert invite_criteria is not None + return _auth_state_matches_invite_criteria( + state, + evaluate_invite_clauses(state.invite_diagnostics, invite_criteria), + ) allowed = ", ".join(INSPECT_FILTERS) raise ValueError(f"Unknown inspect filter: {inspect_filter}; expected one of {allowed}") -def filter_inspection_states(states: list[AuthBusinessState], inspect_filter: str) -> list[AuthBusinessState]: +def filter_inspection_states( + states: list[AuthBusinessState], + inspect_filter: str, + invite_criteria: InviteCriteria | None = None, +) -> list[AuthBusinessState]: """Return inspected states matching the active post-read inspection filter.""" - return [state for state in states if auth_state_matches_inspect_filter(state, inspect_filter)] + _require_invite_filter_criteria(inspect_filter, invite_criteria) + return [ + state + for state in states + if auth_state_matches_inspect_filter(state, inspect_filter, invite_criteria) + ] @dataclass(frozen=True) @@ -545,6 +1268,39 @@ def parse_console_limit(raw_value: Any, config_key: str, default: int = DEFAULT_ return ConsoleLimit(disabled=False, max_rows=parsed) +DEFAULT_INSPECT_AUTH_INVITE_EXPIRY_DAYS = 7 + + +def parse_invite_expiry_days( + raw_value: Any, + config_key: str = "INSPECT_AUTH_INVITE_EXPIRY_DAYS", +) -> int: + """Parse invitation expiry days, defaulting missing values and rejecting blanks.""" + if raw_value is None: + return DEFAULT_INSPECT_AUTH_INVITE_EXPIRY_DAYS + if isinstance(raw_value, str) and not raw_value.strip(): + raise ValueError( + f"{config_key} must not be blank; omit it to use the default of " + f"{DEFAULT_INSPECT_AUTH_INVITE_EXPIRY_DAYS} days" + ) + raw = str(raw_value).strip() if isinstance(raw_value, str) else raw_value + try: + parsed = int(raw) + except (TypeError, ValueError) as exc: + raise ValueError( + f"{config_key} must be a positive integer; use 1 for older than one day" + ) from exc + if parsed <= 0: + raise ValueError( + f"{config_key} must be a positive integer; use 1 for older than one day" + ) + try: + timedelta(days=parsed) + except OverflowError as exc: + raise ValueError(f"{config_key} is outside the supported day range") from exc + return parsed + + def parse_inspect_filter(raw_value: Any, config_key: str = "INSPECT_AUTH_FILTER") -> str: """Parse an inspect filter value using the provided config key in errors.""" raw = (blank_to_none(raw_value) or INSPECT_FILTER_ALL).upper() @@ -820,6 +1576,18 @@ def _numeric_sort_key(value: str) -> tuple[int, str]: return (int(value), value) if value.isdigit() else (0, value) +def _dated_invite_sort_key( + item: tuple[datetime, str, Any], +) -> tuple[datetime, tuple[int, str], tuple[int, str]]: + """Return the deterministic business-wide qualifying-invite order key.""" + sent_date, entity_id, invitation_id = item + return ( + sent_date, + _numeric_sort_key(entity_id), + _numeric_sort_key(str(invitation_id) if invitation_id is not None else ""), + ) + + def _build_text_statement(sql: str, expanding_bind_names: tuple[str, ...]) -> TextClause: """Return a SQLAlchemy text statement with expanding bind parameters attached.""" stmt = text(sql) @@ -858,6 +1626,7 @@ def read_auth_states_for_candidates( business_identifiers: Iterable[str], expected_accounts_by_identifier: dict[str, Iterable[str]] | None = None, selected_checks: tuple[str, ...] = _ALL_CHECKS, + invite_cutoff: datetime | None = None, ) -> list[AuthBusinessState]: """Read and aggregate Auth DB state for selected candidates. @@ -871,6 +1640,7 @@ def read_auth_states_for_candidates( return [] expected_accounts_by_identifier = expected_accounts_by_identifier or {} + normalized_invite_cutoff = _as_utc_naive(invite_cutoff) entity_ids_by_identifier: dict[str, set[Any]] = {identifier: set() for identifier in candidates} business_names_by_identifier: dict[str, set[str]] = {identifier: set() for identifier in candidates} identifier_by_entity_id: dict[str, str] = {} @@ -878,7 +1648,7 @@ def read_auth_states_for_candidates( usable_contact_count_by_entity_id: dict[str, int] = defaultdict(int) found_accounts_by_identifier: dict[str, set[str]] = {identifier: set() for identifier in candidates} found_account_names_by_identifier: dict[str, set[str]] = {identifier: set() for identifier in candidates} - invite_count_by_entity_id: dict[str, int] = defaultdict(int) + invite_diagnostics_by_entity_id: dict[str, _AuthInviteAccumulator] = defaultdict(_AuthInviteAccumulator) with auth_engine.connect() as conn: entity_rows = _execute_fetchall(conn, build_auth_entity_read_statement(candidates)) @@ -916,11 +1686,20 @@ def read_auth_states_for_candidates( found_account_names_by_identifier[identifier].add(str(org_name).strip()) if entity_ids and CHECK_INVITE in selected_checks: - invite_rows = _execute_fetchall(conn, _build_entity_id_statement(_INVITE_READ_SQL, entity_ids)) + invite_statement = _build_entity_id_statement(_INVITE_READ_SQL, entity_ids) + invite_rows = conn.execute(invite_statement.statement(), invite_statement.params) for row in invite_rows: - entity_id = str(_row_value(row, 0, "entity_id")) - invite_count = _row_value(row, 1, "invite_count") - invite_count_by_entity_id[entity_id] += int(invite_count or 0) + entity_id = str(_row_value(row, 1, "entity_id")) + invite_diagnostics_by_entity_id[entity_id].add_row( + entity_id=entity_id, + invitation_id=_row_value(row, 0, "id"), + invite_type=_row_value_or_none(row, 2, "type"), + invitation_status_code=_row_value_or_none(row, 3, "invitation_status_code"), + sent_date=_row_value_or_none(row, 4, "sent_date"), + accepted_date=_row_value_or_none(row, 5, "accepted_date"), + is_deleted=_row_value_or_none(row, 6, "is_deleted"), + invite_cutoff=normalized_invite_cutoff, + ) states: list[AuthBusinessState] = [] for identifier in candidates: @@ -934,7 +1713,12 @@ def read_auth_states_for_candidates( ) contact_count = sum(contact_count_by_entity_id.get(entity_id, 0) for entity_id in entity_ids) usable_contact_count = sum(usable_contact_count_by_entity_id.get(entity_id, 0) for entity_id in entity_ids) - invite_count = sum(invite_count_by_entity_id.get(entity_id, 0) for entity_id in entity_ids) + invite_accumulator = _AuthInviteAccumulator() + for entity_id in entity_ids: + invite_accumulator.merge(invite_diagnostics_by_entity_id.get(entity_id, _AuthInviteAccumulator())) + invite_diagnostics = invite_accumulator.freeze( + cutoff_configured=CHECK_INVITE in selected_checks and normalized_invite_cutoff is not None + ) states.append( AuthBusinessState( business_identifier=identifier, @@ -946,7 +1730,8 @@ def read_auth_states_for_candidates( found_account_ids=found_account_ids, found_account_names=found_account_names, missing_account_ids=missing_account_ids, - invite_count=invite_count, + invite_count=invite_diagnostics.total_count, + invite_diagnostics=invite_diagnostics, ) ) return states @@ -1071,6 +1856,7 @@ def run_auth_batch( candidates, expected_accounts_by_identifier=expected_accounts_by_identifier, selected_checks=settings.auth_read_checks, + invite_cutoff=getattr(settings, "invite_cutoff", None), ) verification_results = build_verification_results(states, settings.selected_checks) if settings.run_verify else [] return AuthBatchResult(states=states, verification_results=verification_results) @@ -1247,47 +2033,298 @@ def _dependent_state(entity_exists: bool, present: bool) -> str: return "present" if present else "missing" -def build_inspection_rows(states: list[AuthBusinessState]) -> list[dict[str, str | int]]: +def _format_count_pairs(counts: tuple[tuple[str, int], ...]) -> str: + """Render deterministic invitation count buckets without CSV-ambiguous commas.""" + return ";".join(f"{name}={count}" for name, count in counts) + + +def _format_diagnostic_datetime(value: datetime | None) -> str: + """Render an Auth timestamp as second-precision UTC-naive ISO-8601.""" + normalized_value = _as_utc_naive(value) + return normalized_value.isoformat(timespec="seconds") if normalized_value is not None else "" + + +_CONSOLE_STATE_ALIASES = { + "not_applicable_entity_missing": "na_entity_missing", +} +_CONSOLE_HEADER_ALIASES = {"business_identifier": "identifier"} +_INVITE_TYPE_ABBREVIATIONS = { + "EMAIL": "EM", + "UNAFFILIATED_EMAIL": "UE", + _BLANK_DIAGNOSTIC_BUCKET: "(b)", +} +_INVITE_STATUS_ABBREVIATIONS = { + "PENDING": "PEND", + "ACCEPTED": "ACC", + "EXPIRED": "EXPD", + "FAILED": "FAIL", + _BLANK_DIAGNOSTIC_BUCKET: "(b)", +} +_FOUND_ACCOUNT_NAMES_CONSOLE_MAX = 28 +INSPECTION_LEGEND_LINES: tuple[str, ...] = ( + "Legend invite_summary: q=qualifying UNAFFILIATED_EMAIL unclaimed; exp=sent=Cutoff unk=NULL sent_date", + "Legend invite_summary: clm=accepted (Auth soft-deletes); del=without acceptance; ages=days since sent, qualifying only", + "Legend invite_dates: s:=unclaimed sent range (all types); a:=latest accepted date; CSV: inspect-auth-inspection.csv", + "Legend invite_mix: T:=types EM=EMAIL UE=UNAFFILIATED_EMAIL; S:=raw Auth status(es); deleted-w/o-acceptance excluded→del=", +) + + +def _format_console_state(value: str | int) -> str: + """Return a compact console-only alias for a persisted report state value.""" + rendered = str(value) + return _CONSOLE_STATE_ALIASES.get(rendered, rendered) + + +def _format_invite_mix_segment( + raw_counts: str | int, + abbreviations: dict[str, str], +) -> str: + """Render one deterministic type/status count segment with compact bucket names.""" + if not raw_counts: + return "" + pairs: list[str] = [] + for raw_pair in str(raw_counts).split(";"): + if not raw_pair: + continue + name, count = raw_pair.split("=", maxsplit=1) + abbreviated = abbreviations.get(name, name.upper()[:5]) + pairs.append(f"{abbreviated}={count}") + return ",".join(pairs) + + +def _format_invite_mix(row: dict[str, str | int]) -> str: + """Return compact persisted type/status counts for the console table.""" + type_counts = _format_invite_mix_segment( + row.get("invite_type_counts", ""), + _INVITE_TYPE_ABBREVIATIONS, + ) + status_counts = _format_invite_mix_segment( + row.get("invite_status_counts", ""), + _INVITE_STATUS_ABBREVIATIONS, + ) + segments = [] + if type_counts: + segments.append(f"T:{type_counts}") + if status_counts: + segments.append(f"S:{status_counts}") + return " ".join(segments) + + +def _format_invite_summary(row: dict[str, str | int]) -> str: + """Return the compact invitation diagnostics used by the console table.""" + unclaimed = row.get("invite_unclaimed_count", 0) + qualifying = row.get("unaffiliated_unclaimed_invite_count", 0) + expired = row.get("unaffiliated_unclaimed_expired_count", "") + current = row.get("unaffiliated_unclaimed_current_count", "") + unknown = row.get("unaffiliated_unclaimed_unknown_age_count", 0) + expiry_summary = "" + if expired != "" and current != "": + age_parts = [f"exp={expired}", f"cur={current}"] + if unknown: + age_parts.append(f"unk={unknown}") + expiry_summary = f"({','.join(age_parts)})" + summary = ( + f"uncl={unclaimed} q={qualifying}{expiry_summary} " + f"clm={row.get('invite_claimed_count', 0)} " + f"del={row.get('invite_deleted_count', 0)}" + ) + ages = [age for age in str(row.get("unaffiliated_unclaimed_ages_days", "")).split(";") if age] + if ages: + preview = ages[:6] + ages_summary = f" ages={','.join(preview)}d" + if len(ages) > len(preview): + ages_summary += f",+{len(ages) - len(preview)}" + summary += ages_summary + return summary + + +def _format_invite_dates(row: dict[str, str | int]) -> str: + """Return all-unclaimed sent dates and the latest acceptance date across all invite types.""" + oldest = str(row.get("oldest_unclaimed_sent_date", "")) + newest = str(row.get("newest_unclaimed_sent_date", "")) + accepted = str(row.get("latest_accepted_date", "")) + populated = [value for value in (oldest, newest, accepted) if value] + if not populated: + return "" + + dates = [datetime.fromisoformat(value).date() for value in populated] + compact = len({value.year for value in dates}) == 1 + + def render(value: str) -> str: + parsed = datetime.fromisoformat(value).date() + return parsed.strftime("%m-%d" if compact else "%Y-%m-%d") + + parts: list[str] = [] + sent_start = oldest or newest + sent_end = newest or oldest + if sent_start: + sent_range = render(sent_start) + if sent_end and sent_end != sent_start: + sent_range += f"→{render(sent_end)}" + parts.append(f"s:{sent_range}") + if accepted: + parts.append(f"a:{render(accepted)}") + return " ".join(parts) + + +def _truncate_console_cell(value: str, max_len: int) -> str: + """Truncate a console-only cell to ``max_len`` characters with an ellipsis.""" + if len(value) <= max_len: + return value + if max_len <= 0: + return "" + return f"{value[: max_len - 1]}…" + + +def _format_ordered_invite_dates(sent_dates: tuple[datetime, ...]) -> str: + """Render ordered qualifying sent dates as a semicolon-delimited audit cell.""" + return ";".join(_format_diagnostic_datetime(sent_date) for sent_date in sent_dates) + + +def _invite_ages_days(sent_dates: tuple[datetime, ...], now: datetime) -> tuple[int, ...]: + """Return floor elapsed days at the reference clock, preserving date order.""" + normalized_now = _as_utc_naive(now) + assert normalized_now is not None + return tuple( + math.floor((normalized_now - normalized_date).total_seconds() / 86400) + for sent_date in sent_dates + if (normalized_date := _as_utc_naive(sent_date)) is not None + ) + + +def _inspection_invite_context( + invite_diagnostics: AuthInviteDiagnostics, + invite_criteria: InviteCriteria | None, + reference_now: datetime | None, +) -> tuple[tuple[tuple[InviteClause, bool], ...], tuple[int, ...]]: + """Evaluate optional criteria and age diagnostics using the established clock precedence.""" + criteria_results = ( + evaluate_invite_clauses(invite_diagnostics, invite_criteria) + if invite_criteria is not None + else () + ) + age_reference = invite_criteria.now if invite_criteria is not None else reference_now + if age_reference is None: + return criteria_results, () + invite_ages = _invite_ages_days( + invite_diagnostics.unaffiliated_unclaimed_sent_dates, + age_reference, + ) + return criteria_results, invite_ages + + +def _inspection_criteria_cells( + criteria_results: tuple[tuple[InviteClause, bool], ...], + invite_criteria: InviteCriteria | None, +) -> tuple[str, str]: + """Render criteria outcome cells while keeping unset criteria cells blank.""" + if invite_criteria is None: + return "", "" + return ( + _format_bool(all(passed for _clause, passed in criteria_results)), + ";".join(clause.raw for clause, passed in criteria_results if not passed), + ) + + +def _optional_diagnostic_count(value: int | None) -> str | int: + """Render an unavailable diagnostic count as the persisted blank cell.""" + return "" if value is None else value + + +def _build_inspection_row( + state: AuthBusinessState, + invite_criteria: InviteCriteria | None, + reference_now: datetime | None, +) -> dict[str, str | int]: + """Build one inspection row without changing persisted field order or formatting.""" + entity_exists = state.entity_exists + invite_diagnostics = state.invite_diagnostics + criteria_results, invite_ages = _inspection_invite_context( + invite_diagnostics, + invite_criteria, + reference_now, + ) + criteria_pass, failed_clauses = _inspection_criteria_cells( + criteria_results, + invite_criteria, + ) + return { + "business_identifier": state.business_identifier, + "business_names": _format_tuple(state.business_names), + "entity_state": "present" if entity_exists else "missing", + "entity_exists": _format_bool(entity_exists), + "entity_count": len(state.entity_ids), + "entity_ids": _format_tuple(state.entity_ids), + "contact_state": _dependent_state(entity_exists, state.usable_contact_count > 0), + "contact_count": state.contact_count, + "usable_contact_count": state.usable_contact_count, + "affiliation_state": _dependent_state(entity_exists, bool(state.found_account_ids)), + "affiliation_count": len(state.found_account_ids), + "found_account_ids": _format_tuple(state.found_account_ids), + "found_account_names": _format_tuple(state.found_account_names), + "expected_account_ids": _format_tuple(state.expected_account_ids), + "missing_account_ids": _format_tuple(state.missing_account_ids), + "invite_state": _dependent_state(entity_exists, state.invite_count > 0), + "invite_count": state.invite_count, + "blocked_by_missing_entity": _format_bool(not entity_exists), + "invite_claimed_count": invite_diagnostics.claimed_count, + "invite_unclaimed_count": invite_diagnostics.unclaimed_count, + "invite_deleted_count": invite_diagnostics.deleted_count, + "invite_type_counts": _format_count_pairs(invite_diagnostics.type_counts), + "invite_status_counts": _format_count_pairs(invite_diagnostics.status_counts), + "unaffiliated_unclaimed_invite_count": invite_diagnostics.unaffiliated_unclaimed_count, + "unaffiliated_unclaimed_expired_count": _optional_diagnostic_count( + invite_diagnostics.expired_unclaimed_unaffiliated_count + ), + "unaffiliated_unclaimed_current_count": _optional_diagnostic_count( + invite_diagnostics.current_unclaimed_unaffiliated_count + ), + "unaffiliated_unclaimed_unknown_age_count": ( + invite_diagnostics.unknown_age_unclaimed_unaffiliated_count + ), + "oldest_unclaimed_sent_date": _format_diagnostic_datetime( + invite_diagnostics.oldest_unclaimed_sent_date + ), + "newest_unclaimed_sent_date": _format_diagnostic_datetime( + invite_diagnostics.newest_unclaimed_sent_date + ), + "latest_accepted_date": _format_diagnostic_datetime(invite_diagnostics.latest_accepted_date), + "unaffiliated_unclaimed_sent_dates": _format_ordered_invite_dates( + invite_diagnostics.unaffiliated_unclaimed_sent_dates + ), + "unaffiliated_unclaimed_ages_days": ";".join(str(age) for age in invite_ages), + "invite_criteria_pass": criteria_pass, + "invite_criteria_failed_clauses": failed_clauses, + } + + +def build_inspection_rows( + states: list[AuthBusinessState], + invite_criteria: InviteCriteria | None = None, + *, + reference_now: datetime | None = None, +) -> list[dict[str, str | int]]: """Build factual inspection rows from the provided Auth DB states. ``contact_state`` is based on usable contacts with non-blank email; raw - contact row totals remain available in ``contact_count``. + contact row totals remain available in ``contact_count``. Invite criteria + clocks take precedence over ``reference_now`` when computing invite ages. """ - rows: list[dict[str, str | int]] = [] - for state in states: - entity_exists = state.entity_exists - rows.append( - { - "business_identifier": state.business_identifier, - "business_names": _format_tuple(state.business_names), - "entity_state": "present" if entity_exists else "missing", - "entity_exists": _format_bool(entity_exists), - "entity_count": len(state.entity_ids), - "entity_ids": _format_tuple(state.entity_ids), - "contact_state": _dependent_state(entity_exists, state.usable_contact_count > 0), - "contact_count": state.contact_count, - "usable_contact_count": state.usable_contact_count, - "affiliation_state": _dependent_state(entity_exists, bool(state.found_account_ids)), - "affiliation_count": len(state.found_account_ids), - "found_account_ids": _format_tuple(state.found_account_ids), - "found_account_names": _format_tuple(state.found_account_names), - "expected_account_ids": _format_tuple(state.expected_account_ids), - "missing_account_ids": _format_tuple(state.missing_account_ids), - "invite_state": _dependent_state(entity_exists, state.invite_count > 0), - "invite_count": state.invite_count, - "blocked_by_missing_entity": _format_bool(not entity_exists), - } - ) - return rows + return [ + _build_inspection_row(state, invite_criteria, reference_now) + for state in states + ] def build_inspection_identifier_rows( states: list[AuthBusinessState], matched_states: list[AuthBusinessState], inspect_filter: str, + invite_criteria: InviteCriteria | None = None, ) -> list[dict[str, str | int]]: """Build copy-friendly AUTH_CORP_NUMS rows from raw inspection state.""" if inspect_filter != INSPECT_FILTER_ALL: + _require_invite_filter_criteria(inspect_filter, invite_criteria) return [ { "inspect_filter": inspect_filter, @@ -1298,10 +2335,15 @@ def build_inspection_identifier_rows( rows: list[dict[str, str | int]] = [] for filter_token in INSPECT_IDENTIFIER_FILTERS: + if ( + filter_token == INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA + and invite_criteria is None + ): + continue identifiers = [ state.business_identifier for state in states - if auth_state_matches_inspect_filter(state, filter_token) + if auth_state_matches_inspect_filter(state, filter_token, invite_criteria) ] if identifiers: rows.append( @@ -1373,6 +2415,32 @@ def _write_identifier_copy_list_section( txt_file.write(f"SQL_IN_IDENTIFIERS={_format_sql_quoted_identifier_list(identifiers_csv)}\n") +def _write_invite_reminder_handoff_section( + txt_file: Any, + identifiers_csv: str, + count: int, +) -> None: + """Write the paste-ready environment block for a non-empty reminder cohort.""" + if count <= 0: + return + txt_file.write("\n") + txt_file.write("# ── Next step: send reminder invites to this cohort ─────────\n") + txt_file.write( + "# Review the list, then copy this block into your .env and run: make run-auth-invite\n" + ) + txt_file.write("AUTH_SELECTION_MODE=MIGRATION_FILTER\n") + txt_file.write(f"AUTH_CORP_NUMS={identifiers_csv}\n") + txt_file.write( + "# REQUIRED: set a new key per reminder round, " + "e.g. AUTH_REPEATABLE_CYCLE_KEY=saf_reminder_2\n" + ) + txt_file.write("AUTH_REPEATABLE_CYCLE_KEY=\n") + txt_file.write("AUTH_INVITE_IS_REMINDER=True\n") + txt_file.write( + "# Reminder rounds are inferred from invite count/age only; Auth stores no reminder ordinal.\n" + ) + + def _write_identifier_copy_list_txt( path: str, rows: list[dict[str, str | int]], @@ -1416,13 +2484,19 @@ def write_scenario_txt(path: str, results: list[VerificationResult]) -> None: ) -def write_inspection_csv(path: str, states: list[AuthBusinessState]) -> None: +def write_inspection_csv( + path: str, + states: list[AuthBusinessState], + invite_criteria: InviteCriteria | None = None, + *, + reference_now: datetime | None = None, +) -> None: """Write factual Auth DB inspection detail CSV rows for the provided states.""" expanded_path = _ensure_parent_dir(path) with open(expanded_path, "w", newline="", encoding="utf-8") as csv_file: writer = csv.DictWriter(csv_file, fieldnames=INSPECTION_FIELDNAMES) writer.writeheader() - writer.writerows(build_inspection_rows(states)) + writer.writerows(build_inspection_rows(states, invite_criteria, reference_now=reference_now)) _INSPECTION_FILTER_SUMMARY_METRICS = { @@ -1441,12 +2515,16 @@ def write_inspection_csv(path: str, states: list[AuthBusinessState]) -> None: }, INSPECT_FILTER_HAS_INVITE: {"label": "HasInvite", "summary_key": "has_invite_count"}, INSPECT_FILTER_ENTITY_WITHOUT_INVITE: {"label": "EntityWithoutInvite", "summary_key": "entity_without_invite_count"}, + INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA: { + "label": "NoAffiliationInviteCriteria", + "summary_key": "no_affiliation_invite_criteria_count", + }, } -def _all_inspection_summary_metric_rows(summary: dict[str, int | str]) -> list[tuple[str, int | str]]: +def _all_inspection_summary_metric_rows(summary: dict[str, Any]) -> list[tuple[str, int | str]]: """Return display-ordered full inspect summary metric rows.""" - return [ + rows = [ ("Inspected", summary.get("inspected_count", 0)), ("MatchedByFilter", summary.get("matched_count", 0)), ("HasAnyAuth", summary.get("has_any_auth_count", 0)), @@ -1457,11 +2535,30 @@ def _all_inspection_summary_metric_rows(summary: dict[str, int | str]) -> list[t ("HasAffiliation", summary.get("has_affiliation_count", 0)), ("EntityWithoutAffiliation", summary.get("entity_without_affiliation_count", 0)), ("HasInvite", summary.get("has_invite_count", 0)), - ("EntityWithoutInvite", summary.get("entity_without_invite_count", 0)), + ("HasUnclaimedInvite", summary.get("has_unclaimed_invite_count", 0)), + ("HasClaimedInvite", summary.get("has_claimed_invite_count", 0)), + ( + "HasUnaffiliatedUnclaimedInvite", + summary.get("has_unaffiliated_unclaimed_invite_count", 0), + ), ] + if summary.get("invite_criteria", ""): + rows.append( + ( + "NoAffiliationInviteCriteria", + summary.get("no_affiliation_invite_criteria_count", 0), + ) + ) + rows.extend( + [ + ("UnclaimedInviteRows(invites)", summary.get("unclaimed_invite_rows_total", 0)), + ("EntityWithoutInvite", summary.get("entity_without_invite_count", 0)), + ] + ) + return rows -def _inspection_summary_metric_rows(summary: dict[str, int | str]) -> list[tuple[str, int | str]]: +def _inspection_summary_metric_rows(summary: dict[str, Any]) -> list[tuple[str, int | str]]: """Return filter-aware inspect summary metric rows.""" inspect_filter = str(summary.get("inspect_filter", INSPECT_FILTER_ALL)) if inspect_filter == INSPECT_FILTER_ALL: @@ -1480,7 +2577,9 @@ def _inspection_summary_metric_rows(summary: dict[str, int | str]) -> list[tuple def write_inspection_summary_txt( path: str, identifier_rows: list[dict[str, str | int]], - summary: dict[str, int | str] | None = None, + summary: dict[str, Any] | None = None, + *, + handoff_filter: str | None = None, ) -> None: """Write inspect summary and copy-friendly identifier groups as plain text.""" expanded_path = _ensure_parent_dir(path) @@ -1488,6 +2587,13 @@ def write_inspection_summary_txt( if summary is not None: txt_file.write("# Inspect Auth summary\n") txt_file.write(f"Filter={summary.get('inspect_filter', '')}\n") + if summary.get("invite_criteria", ""): + txt_file.write(f"{summary.get('invite_criteria', '')}\n") + elif "expiry_days" in summary and "expiry_cutoff" in summary: + cutoff_text = format_clock_z(summary.get("expiry_cutoff")) + txt_file.write( + f"ExpiryDays={summary.get('expiry_days')}, Cutoff={cutoff_text} (diagnostics-only)\n" + ) table_rows = [ {"metric": label, "count": str(value)} for label, value in _inspection_summary_metric_rows(summary) @@ -1495,6 +2601,23 @@ def write_inspection_summary_txt( for line in _render_table_lines(table_rows, ("metric", "count"), right_aligned_columns=("count",)): txt_file.write(f"{line}\n") txt_file.write("\n") + clause_audit = summary.get("clause_audit", ()) + if clause_audit: + txt_file.write( + "# Invite criteria clause audit " + f"(businesses passing each clause, of {summary.get('inspected_count', 0)} inspected)\n" + ) + audit_rows = [ + {"clause": str(clause), "passing": str(passing)} + for clause, passing in clause_audit + ] + for line in _render_table_lines( + audit_rows, + ("clause", "passing"), + right_aligned_columns=("passing",), + ): + txt_file.write(f"{line}\n") + txt_file.write("\n") _write_identifier_copy_list_section( txt_file, @@ -1502,6 +2625,16 @@ def write_inspection_summary_txt( label_key="inspect_filter", title="Inspect Auth identifier copy lists", ) + if ( + handoff_filter == INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA + and len(identifier_rows) == 1 + ): + handoff_row = identifier_rows[0] + _write_invite_reminder_handoff_section( + txt_file, + str(handoff_row.get("identifiers_csv", "")), + int(handoff_row.get("count", 0) or 0), + ) def _table_header_border(widths: dict[str, int], columns: tuple[str, ...]) -> str: @@ -1731,38 +2864,101 @@ def print_scenario_rows( ) -def build_inspection_summary(states: list[AuthBusinessState], matched_states: list[AuthBusinessState], inspect_filter: str) -> dict[str, int | str]: +def build_inspection_summary( + states: list[AuthBusinessState], + matched_states: list[AuthBusinessState], + inspect_filter: str, + invite_criteria: InviteCriteria | None = None, +) -> dict[str, Any]: """Build full-population inspection summary counts. ``has_contact_count`` and ``entity_without_contact_count`` use usable contact email, not merely raw contact row presence. """ - return { + _require_invite_filter_criteria(inspect_filter, invite_criteria) + summary: dict[str, Any] = { "inspect_filter": inspect_filter, "inspected_count": len(states), "matched_count": len(matched_states), "emitted_count": len(matched_states), "has_any_auth_count": sum(1 for state in states if auth_state_has_any_auth(state)), - "missing_entity_count": sum(1 for state in states if auth_state_matches_inspect_filter(state, INSPECT_FILTER_MISSING_ENTITY)), - "has_entity_count": sum(1 for state in states if auth_state_matches_inspect_filter(state, INSPECT_FILTER_HAS_ENTITY)), - "has_contact_count": sum(1 for state in states if auth_state_matches_inspect_filter(state, INSPECT_FILTER_HAS_CONTACT)), + "missing_entity_count": sum( + 1 for state in states + if auth_state_matches_inspect_filter(state, INSPECT_FILTER_MISSING_ENTITY) + ), + "has_entity_count": sum( + 1 for state in states + if auth_state_matches_inspect_filter(state, INSPECT_FILTER_HAS_ENTITY) + ), + "has_contact_count": sum( + 1 for state in states + if auth_state_matches_inspect_filter(state, INSPECT_FILTER_HAS_CONTACT) + ), "entity_without_contact_count": sum( - 1 for state in states if auth_state_matches_inspect_filter(state, INSPECT_FILTER_ENTITY_WITHOUT_CONTACT) + 1 for state in states + if auth_state_matches_inspect_filter(state, INSPECT_FILTER_ENTITY_WITHOUT_CONTACT) + ), + "has_affiliation_count": sum( + 1 for state in states + if auth_state_matches_inspect_filter(state, INSPECT_FILTER_HAS_AFFILIATION) ), - "has_affiliation_count": sum(1 for state in states if auth_state_matches_inspect_filter(state, INSPECT_FILTER_HAS_AFFILIATION)), "entity_without_affiliation_count": sum( - 1 for state in states if auth_state_matches_inspect_filter(state, INSPECT_FILTER_ENTITY_WITHOUT_AFFILIATION) + 1 for state in states + if auth_state_matches_inspect_filter(state, INSPECT_FILTER_ENTITY_WITHOUT_AFFILIATION) + ), + "has_invite_count": sum( + 1 for state in states + if auth_state_matches_inspect_filter(state, INSPECT_FILTER_HAS_INVITE) + ), + "has_unclaimed_invite_count": sum( + 1 for state in states if state.invite_diagnostics.unclaimed_count > 0 + ), + "has_claimed_invite_count": sum( + 1 for state in states if state.invite_diagnostics.claimed_count > 0 + ), + "has_unaffiliated_unclaimed_invite_count": sum( + 1 for state in states if state.invite_diagnostics.unaffiliated_unclaimed_count > 0 + ), + "unclaimed_invite_rows_total": sum( + state.invite_diagnostics.unclaimed_count for state in states ), - "has_invite_count": sum(1 for state in states if auth_state_matches_inspect_filter(state, INSPECT_FILTER_HAS_INVITE)), "entity_without_invite_count": sum( - 1 for state in states if auth_state_matches_inspect_filter(state, INSPECT_FILTER_ENTITY_WITHOUT_INVITE) + 1 for state in states + if auth_state_matches_inspect_filter(state, INSPECT_FILTER_ENTITY_WITHOUT_INVITE) ), } + if invite_criteria is not None: + summary["invite_criteria"] = format_invite_criteria(invite_criteria) + state_clause_results = [ + ( + state, + evaluate_invite_clauses(state.invite_diagnostics, invite_criteria), + ) + for state in states + ] + summary["no_affiliation_invite_criteria_count"] = sum( + 1 + for state, results in state_clause_results + if _auth_state_matches_invite_criteria(state, results) + ) + summary["clause_audit"] = tuple( + ( + clause.raw, + sum(1 for _state, results in state_clause_results if results[index][1]), + ) + for index, clause in enumerate(invite_criteria.clauses) + ) + return summary -def print_inspection_summary(states: list[AuthBusinessState], matched_states: list[AuthBusinessState], inspect_filter: str) -> None: +def print_inspection_summary( + states: list[AuthBusinessState], + matched_states: list[AuthBusinessState], + inspect_filter: str, + invite_criteria: InviteCriteria | None = None, +) -> None: """Print readable summary-first inspection output.""" - summary = build_inspection_summary(states, matched_states, inspect_filter) + summary = build_inspection_summary(states, matched_states, inspect_filter, invite_criteria) metric_rows = [(label, str(value)) for label, value in _inspection_summary_metric_rows(summary)] metric_width = max(len("metric"), *(len(label) for label, _ in metric_rows)) count_width = max(len("count"), *(len(value) for _, value in metric_rows)) @@ -1775,6 +2971,23 @@ def print_inspection_summary(states: list[AuthBusinessState], matched_states: li for label, value in metric_rows: print(f"{label.ljust(metric_width)} | {value.rjust(count_width)}") print() + clause_audit = summary.get("clause_audit", ()) + if clause_audit: + audit_rows = [ + {"clause": str(clause), "passing": str(passing)} + for clause, passing in clause_audit + ] + print( + "🧮 Invite criteria clause audit " + f"(businesses passing each clause, of {summary['inspected_count']} inspected):" + ) + for line in _render_table_lines( + audit_rows, + ("clause", "passing"), + right_aligned_columns=("passing",), + ): + print(line) + print() if not states: print("🔎 Inspect Auth: no selected businesses to inspect.") elif not matched_states: @@ -1807,60 +3020,152 @@ def _print_preview_truncated_message( ) -def print_inspection_rows( - inspection_rows: list[dict[str, str | int]], - console_limit: ConsoleLimit | None = None, - report_path: str | None = None, - limit_config_key: str = "INSPECT_AUTH_CONSOLE_LIMIT", -) -> None: - """Print a compact factual inspection table for matched inspection rows.""" - if console_limit is not None and console_limit.disabled: - return - if not inspection_rows: - print("🔎 Inspect Auth detail: no matched rows to print.") - return - - preview_rows = _preview_rows(inspection_rows, console_limit) - if not preview_rows: - return - +def _inspection_console_columns_and_headers() -> tuple[tuple[str, ...], dict[str, str]]: + """Return ordered inspect console columns and their display headers.""" columns = ( + "row", "business_identifier", "business_names", "entity_state", "contact_state", "affiliation_state", "invite_state", + "invite_mix", + "invite_summary", + "invite_dates", "found_account_ids", "found_account_names", ) - table_rows = [{column: str(row.get(column, "")) for column in columns} for row in preview_rows] + headers = { + column: _CONSOLE_HEADER_ALIASES.get( + column, + column[: -len("_state")] if column.endswith("_state") else column, + ) + for column in columns + } + return columns, headers + + +def _inspection_console_cell( + row: dict[str, str | int], + row_number: int, + column: str, + business_names_max_length: int | None, +) -> str: + """Format one inspection value for console display.""" + if column == "row": + return str(row_number) + if column == "invite_mix": + return _format_invite_mix(row) + if column == "invite_summary": + return _format_invite_summary(row) + if column == "invite_dates": + return _format_invite_dates(row) + if column == "business_names" and business_names_max_length is not None: + return _truncate_console_cell(str(row.get(column, "")), business_names_max_length) + if column == "found_account_names": + return _truncate_console_cell( + str(row.get(column, "")), + _FOUND_ACCOUNT_NAMES_CONSOLE_MAX, + ) + if column in {"contact_state", "affiliation_state", "invite_state"}: + return _format_console_state(row.get(column, "")) + return str(row.get(column, "")) + + +def _inspection_console_table_row( + row: dict[str, str | int], + row_number: int, + columns: tuple[str, ...], + business_names_max_length: int | None, +) -> dict[str, str]: + """Build one numbered inspection console table row.""" + return { + column: _inspection_console_cell( + row, + row_number, + column, + business_names_max_length, + ) + for column in columns + } + + +def _inspection_console_marker_row(remaining_rows: int, limit_config_key: str) -> dict[str, str]: + """Build the marker row shown when the console preview is capped.""" + return { + "row": "", + "business_identifier": f"⚠️ {remaining_rows} MORE ROWS NOT SHOWN", + "business_names": f"console capped by {limit_config_key}", + "entity_state": "CSV complete", + "contact_state": "set ALL", + "affiliation_state": "for full console", + "invite_state": "", + "invite_mix": "", + "invite_summary": "", + "invite_dates": "", + "found_account_ids": "", + "found_account_names": "", + } + + +def _build_inspection_console_table_rows( + inspection_rows: list[dict[str, str | int]], + preview_rows: list[dict[str, str | int]], + columns: tuple[str, ...], + limit_config_key: str, + business_names_max_length: int | None, +) -> tuple[list[dict[str, str]], bool]: + """Build formatted preview rows and append a truncation marker when needed.""" + table_rows = [ + _inspection_console_table_row( + row, + row_number, + columns, + business_names_max_length, + ) + for row_number, row in enumerate(preview_rows, start=1) + ] truncated = len(preview_rows) < len(inspection_rows) if truncated: remaining_rows = len(inspection_rows) - len(preview_rows) - table_rows.append( - { - "business_identifier": f"⚠️ {remaining_rows} MORE ROWS NOT SHOWN", - "business_names": f"console capped by {limit_config_key}", - "entity_state": "CSV complete", - "contact_state": "set ALL", - "affiliation_state": "for full console", - "invite_state": "", - "found_account_ids": "", - "found_account_names": "", - } - ) - widths = { - column: max(len(column), *(len(row[column]) for row in table_rows)) + table_rows.append(_inspection_console_marker_row(remaining_rows, limit_config_key)) + return table_rows, truncated + + +def _inspection_console_widths( + headers: dict[str, str], + table_rows: list[dict[str, str]], + columns: tuple[str, ...], +) -> dict[str, int]: + """Measure inspect console columns from headers and rendered cells.""" + return { + column: max(len(headers[column]), *(len(row[column]) for row in table_rows)) for column in columns } - cap_label = _console_cap_label(console_limit, limit_config_key, unit="rows") + +def _print_inspection_console_table( + table_rows: list[dict[str, str]], + columns: tuple[str, ...], + headers: dict[str, str], + widths: dict[str, int], + shown_rows: int, + total_rows: int, + cap_label: str, +) -> None: + """Render the inspection console title, legends, borders, and table rows.""" header_border = _table_header_border(widths, columns) marker_separator = "-+-".join("-" * widths[column] for column in columns) - print(f"🔎 Inspect Auth state (contact_state uses usable contact email; {cap_label}):") + print( + "🔎 Inspect Auth state — " + f"showing {shown_rows} of {total_rows} matched " + f"({cap_label}; contact_state uses usable contact email):" + ) + for legend_line in INSPECTION_LEGEND_LINES: + print(legend_line) print(header_border) - print(" | ".join(column.ljust(widths[column]) for column in columns)) + print(" | ".join(headers[column].ljust(widths[column]) for column in columns)) print(header_border) for row in table_rows: rendered_row = " | ".join(row[column].ljust(widths[column]) for column in columns) @@ -1871,6 +3176,45 @@ def print_inspection_rows( else: print(rendered_row) + +def print_inspection_rows( + inspection_rows: list[dict[str, str | int]], + console_limit: ConsoleLimit | None = None, + report_path: str | None = None, + limit_config_key: str = "INSPECT_AUTH_CONSOLE_LIMIT", + business_names_max_length: int | None = None, +) -> None: + """Print a compact factual inspection table for matched inspection rows.""" + if console_limit is not None and console_limit.disabled: + return + if not inspection_rows: + print("🔎 Inspect Auth detail: no matched rows to print.") + return + + preview_rows = _preview_rows(inspection_rows, console_limit) + if not preview_rows: + return + + columns, headers = _inspection_console_columns_and_headers() + table_rows, truncated = _build_inspection_console_table_rows( + inspection_rows, + preview_rows, + columns, + limit_config_key, + business_names_max_length, + ) + widths = _inspection_console_widths(headers, table_rows, columns) + cap_label = _console_cap_label(console_limit, limit_config_key, unit="rows") + _print_inspection_console_table( + table_rows, + columns, + headers, + widths, + len(preview_rows), + len(inspection_rows), + cap_label, + ) + if truncated: _print_preview_truncated_message( label="Inspect Auth state", @@ -1963,11 +3307,17 @@ def write_verification_reports(settings: VerifyAuthSettings, results: list[Verif write_scenario_txt(settings.scenario_path, results) -def write_inspection_report(path: str, states: list[AuthBusinessState]) -> None: +def write_inspection_report( + path: str, + states: list[AuthBusinessState], + invite_criteria: InviteCriteria | None = None, + *, + reference_now: datetime | None = None, +) -> None: """Write an inspection CSV report to the provided path.""" if not path: raise ValueError("Inspection output path must be set before writing inspection reports") - write_inspection_csv(path, states) + write_inspection_csv(path, states, invite_criteria, reference_now=reference_now) def calculate_batch_count(total_candidates: int, batch_size: int, configured_batches: int) -> int: diff --git a/data-tool/flows/batch_delete_flow.py b/data-tool/flows/batch_delete_flow.py index 097e96a08f..93d9f9beae 100644 --- a/data-tool/flows/batch_delete_flow.py +++ b/data-tool/flows/batch_delete_flow.py @@ -731,6 +731,124 @@ def get_selected_corps_mig(db_engine: Engine, config, candidates: List[str], off # nothing found in remaining pages return None, None, curr + +@task(cache_policy=NO_CACHE) +def get_lear_business_filings(db_engine: Engine, business_ids: List[int]): + """Return all filings for the provided business IDs where source = 'LEAR'.""" + if not business_ids: + return [] + + sql = text(""" + SELECT * + FROM filings + WHERE business_id IN :business_ids + AND source = :source + """).bindparams( + bindparam('business_ids', expanding=True), + bindparam('source'), + ) + + with db_engine.connect() as conn: + rows = conn.execute(sql, { + 'business_ids': business_ids, + 'source': 'LEAR', + }).fetchall() + + return [dict(row._mapping) for row in rows] + + +@task(cache_policy=NO_CACHE) +def insert_demigrated_filings(lear_engine: Engine, colin_engine: Engine, business_ids: List[int], identifiers: List[str]): + """Fetch any LEAR filings for the provided businesses and persist them for demigration tracking.""" + if not business_ids: + return + + filings = get_lear_business_filings(lear_engine, business_ids) + if not filings: + return + + id_to_identifier = dict(zip(business_ids, identifiers)) if business_ids and identifiers else {} + + def _serialize_if_json(value): + if isinstance(value, (dict, list)): + return __import__('json').dumps(value) + return value + + normalized_filings = [] + for filing in filings: + if not filing: + continue + + filing_dict = dict(filing) if hasattr(filing, 'keys') else filing + filing_id = filing_dict.get('id') + if filing_id is None: + continue + + business_id = filing_dict.get('business_id') + normalized_filings.append({ + 'identifier': id_to_identifier.get(business_id), + 'filing_id': filing_id, + 'filing_date': filing_dict.get('filing_date'), + 'filing_type': filing_dict.get('filing_type'), + 'filing_json': _serialize_if_json(filing_dict.get('filing_json')), + 'payment_id': filing_dict.get('payment_id'), + 'transaction_id': filing_dict.get('transaction_id'), + 'business_id': business_id, + 'submitter_id': filing_dict.get('submitter_id'), + 'status': filing_dict.get('status'), + 'payment_completion_date': filing_dict.get('payment_completion_date'), + 'paper_only': filing_dict.get('paper_only'), + 'completion_date': filing_dict.get('completion_date'), + 'effective_date': filing_dict.get('effective_date'), + 'source': filing_dict.get('source') or 'LEAR', + 'parent_filing_id': filing_dict.get('parent_filing_id'), + 'payment_status_code': filing_dict.get('payment_status_code'), + 'temp_reg': filing_dict.get('temp_reg'), + 'payment_account': filing_dict.get('payment_account'), + 'tech_correction_json': _serialize_if_json(filing_dict.get('tech_correction_json')), + 'colin_only': filing_dict.get('colin_only'), + 'deletion_locked': filing_dict.get('deletion_locked'), + 'submitter_roles': _serialize_if_json(filing_dict.get('submitter_roles')), + 'meta_data': _serialize_if_json(filing_dict.get('meta_data')), + 'filing_sub_type': filing_dict.get('filing_sub_type'), + 'approval_type': filing_dict.get('approval_type'), + 'application_date': filing_dict.get('application_date'), + 'notice_date': filing_dict.get('notice_date'), + 'resubmission_date': filing_dict.get('resubmission_date'), + 'hide_in_ledger': filing_dict.get('hide_in_ledger'), + 'withdrawn_filing_id': filing_dict.get('withdrawn_filing_id'), + 'withdrawal_pending': filing_dict.get('withdrawal_pending'), + 'lear_only': filing_dict.get('lear_only') + }) + + if not normalized_filings: + return + + sql = text(""" + INSERT INTO demigrated_filings (corp_num, filing_id, filing_date, filing_type, filing_json, + payment_id, transaction_id, business_id, submitter_id, status, + payment_completion_date, paper_only, completion_date, effective_date, + source, parent_filing_id, payment_status_code, temp_reg, payment_account, + tech_correction_json, colin_only, deletion_locked, + submitter_roles, meta_data, filing_sub_type, approval_type, + application_date, notice_date, resubmission_date, hide_in_ledger, + withdrawn_filing_id, withdrawal_pending, lear_only) + VALUES (:identifier, :filing_id, :filing_date, :filing_type, :filing_json, + :payment_id, :transaction_id, :business_id, :submitter_id, :status, + :payment_completion_date, :paper_only, :completion_date, :effective_date, + :source, :parent_filing_id, :payment_status_code, :temp_reg, :payment_account, + :tech_correction_json, :colin_only, :deletion_locked, + :submitter_roles, :meta_data, :filing_sub_type, :approval_type, + :application_date, :notice_date, :resubmission_date, :hide_in_ledger, + :withdrawn_filing_id, :withdrawal_pending, :lear_only) + ON CONFLICT (id) DO NOTHING + """) + + with colin_engine.connect() as conn: + conn.execute(sql, normalized_filings) + conn.commit() + + @flow(log_prints=True) def batch_delete_flow(): try: @@ -773,6 +891,10 @@ def batch_delete_flow(): break print(f'🚀 Running round {cnt} to delete {len(business_ids)} busiesses...') + # If business has lear filings save them to demigrated filings table + if config.SAVE_DEMIGRATED_LEAR_FILINGS: + insert_demigrated_filings(lear_engine, colin_engine, business_ids, identifiers) + futures = [] futures.append(lear_delete.submit(lear_engine, business_ids)) diff --git a/data-tool/flows/common/auth_service.py b/data-tool/flows/common/auth_service.py index 644ab283ee..9855b6c3b0 100644 --- a/data-tool/flows/common/auth_service.py +++ b/data-tool/flows/common/auth_service.py @@ -379,7 +379,15 @@ def update_contact_email(cls, config, identifier: str, email: str, *, token: str return HTTPStatus.BAD_REQUEST @classmethod - def send_unaffiliated_email(cls, config, identifier: str, email: str, *, token: str | None = None) -> HTTPStatus: + def send_unaffiliated_email( + cls, + config, + identifier: str, + email: str, + *, + token: str | None = None, + is_reminder: bool = False, + ) -> HTTPStatus: """Send unaffiliated email/invite to the business (if supported by API).""" token = cls._resolve_token(config, token) if not token: @@ -389,16 +397,17 @@ def send_unaffiliated_email(cls, config, identifier: str, email: str, *, token: account_svc_affiliation_invitation_url = f'{auth_url}/affiliationInvitations' print(f'👷 Sending unaffiliated email to {email} for {identifier}...') - data = {} - rv = requests.post( - url=f'{account_svc_affiliation_invitation_url}/unaffiliated/{identifier}', - headers={ + data = {'isReminder': True} if is_reminder else {} + request_kwargs = { + 'url': f'{account_svc_affiliation_invitation_url}/unaffiliated/{identifier}', + 'headers': { **cls.CONTENT_TYPE_JSON, 'Authorization': cls.BEARER + token }, - data=json.dumps(data), - timeout=cls.get_time_out(config) - ) + 'data': json.dumps(data), + 'timeout': cls.get_time_out(config), + } + rv = requests.post(**request_kwargs) if rv.status_code in (HTTPStatus.OK, HTTPStatus.CREATED): return HTTPStatus.OK diff --git a/data-tool/flows/config.py b/data-tool/flows/config.py index 4a4ef3360b..789db8733a 100644 --- a/data-tool/flows/config.py +++ b/data-tool/flows/config.py @@ -245,10 +245,25 @@ class _Config(): # pylint: disable=too-few-public-methods # inspect Auth flow. HAS_CONTACT / ENTITY_WITHOUT_CONTACT mean usable contact email, # not merely any raw contact row. INSPECT_AUTH_FILTER = (os.getenv('INSPECT_AUTH_FILTER') or 'ALL').strip() or 'ALL' + # Invitation settings stay raw here and are validated only by inspect-auth. + # INSPECT_AUTH_INVITE_CRITERIA is an AND-composed clause list such as + # count>=3, all_expired, age[1]>=90, newest_age>30. Positions are chronological + # (1=oldest); newest_age is the smallest elapsed age. all_ages/any_age are parse-time + # aliases: for >=/> they normalize to newest_age/age[1], respectively; for <=/< they + # normalize to age[1]/newest_age. Logs and reports show those canonical endpoints. + # Bare age[*], OR/NOT/parentheses are unsupported. NULL sent_date counts toward count + # but makes every age/all_expired clause false. + INSPECT_AUTH_INVITE_CRITERIA = os.getenv('INSPECT_AUTH_INVITE_CRITERIA') + # Expiry defaults to 7 days when omitted; a positive value overrides it and an explicit + # blank is invalid. The cutoff is captured once per run; expired means sent_date < cutoff. + INSPECT_AUTH_INVITE_EXPIRY_DAYS = os.getenv('INSPECT_AUTH_INVITE_EXPIRY_DAYS', '7') # Console-only inspect preview/identifier limit: 0 suppresses optional sections, # a positive integer caps console rows/identifiers, and ALL prints all. # File outputs remain complete, including inspect-auth-inspection.csv and inspect-auth-summary.txt. INSPECT_AUTH_CONSOLE_LIMIT = os.getenv('INSPECT_AUTH_CONSOLE_LIMIT', '25') + # Console-only business_names cell cap. Unset/blank/ALL is uncapped; a positive + # integer caps the total rendered cell length. Persisted report values stay complete. + INSPECT_AUTH_BUSINESS_NAMES_MAX_LENGTH = os.getenv('INSPECT_AUTH_BUSINESS_NAMES_MAX_LENGTH', 'ALL') # freeze flow FREEZE_BATCHES = _get_int('FREEZE_BATCHES', 0) @@ -275,6 +290,8 @@ class _Config(): # pylint: disable=too-few-public-methods MIG_GROUP_IDS = os.getenv('MIG_GROUP_IDS') MIG_BATCH_IDS = os.getenv('MIG_BATCH_IDS') + SAVE_DEMIGRATED_LEAR_FILINGS = os.getenv('SAVE_DEMIGRATED_LEAR_FILINGS', 'False') == 'True' + # ------------------------------------------------------------------------------------------ # Auth-only flows (auth_processing tracking) # ------------------------------------------------------------------------------------------ @@ -296,6 +313,9 @@ class _Config(): # pylint: disable=too-few-public-methods AUTH_UPSERT_CONTACT = _get_bool('AUTH_UPSERT_CONTACT', False) AUTH_CREATE_AFFILIATIONS = _get_bool('AUTH_CREATE_AFFILIATIONS', False) AUTH_SEND_UNAFFILIATED_EMAIL = _get_bool('AUTH_SEND_UNAFFILIATED_EMAIL', False) + # Auth create/invite plans add isReminder: true to the unaffiliated invitation JSON body + # when enabled; False preserves the serialized empty object. + AUTH_INVITE_IS_REMINDER = _get_bool('AUTH_INVITE_IS_REMINDER', False) AUTH_FAIL_IF_MISSING_EMAIL = _get_bool('AUTH_FAIL_IF_MISSING_EMAIL', False) AUTH_DRY_RUN = _get_bool('AUTH_DRY_RUN', False) diff --git a/data-tool/flows/migrate_corps_flow.py b/data-tool/flows/migrate_corps_flow.py deleted file mode 100644 index a0e49fb1b2..0000000000 --- a/data-tool/flows/migrate_corps_flow.py +++ /dev/null @@ -1,475 +0,0 @@ -import os -import sys - -# Prefect 3 dependencies requires SQLAlchemy 2.x so we load what Prefect 3 needs by default. -# We override SqlAlchemy in this flow to use SQLAlchemy 1.4.44 which is still required by -# legal api dependencies. -# Add SQLAlchemy 1.4.44 to path - do this before any other imports -sqlalchemy_path = os.getenv('SQLALCHEMY_PATH') -if sqlalchemy_path: - print(f'Using SQLAlchemy from: {sqlalchemy_path}') - sys.path.insert(0, sqlalchemy_path) - from sqlalchemy import __version__, create_engine, engine, text - from sqlalchemy.exc import InvalidRequestError - print(f'SQLAlchemy version: {__version__}') - - -import pandas as pd -import prefect -from legal_api.models import db -from legal_api.models.db import init_db -from legal_api.models import Business -from prefect import flow, task, serve - - -from config import get_named_config -from flows.corps.corp_queries import get_unprocessed_corps_query -from flows.corps.event_filing_service import EventFilingService, IAEventFilings -from corps.filing_data_cleaning_utils import clean_offices_data, clean_corp_party_data, clean_corp_data, clean_event_data -from common.processing_status_service import ProcessingStatusService, ProcessingStatuses -from custom_filer.corps_filer import process_filing -from common.custom_exceptions import CustomException, CustomUnsupportedTypeException -from flows.corps.lear_data_utils import populate_filing_json_from_lear, populate_filing -from corps.filing_json_factory_service import FilingJsonFactoryService -from corps.filing_data_utils import get_previous_event_ids, \ - get_processed_event_ids, get_event_info_to_retrieve, is_in_lear -from flask import Flask -from datetime import timedelta - - - -@task( - name="colin_init", - # retries=3, - # retry_delay_seconds=60, - log_prints=True -) -def colin_init(config): - print("Initializing COLIN connection") - engine = create_engine(config.SQLALCHEMY_DATABASE_URI_COLIN_MIGR) - return engine - - -@task( - name="lear_init", - # retries=3, - # retry_delay_seconds=60, - log_prints=True -) -def lear_init(config): - print("Initializing LEAR connection") - FLASK_APP = Flask('init_lear') - FLASK_APP.config.from_object(config) - init_db(FLASK_APP) - FLASK_APP.app_context().push() - return FLASK_APP, db - - -@task( - name="get_config", - # retries=3, - # retry_delay_seconds=60, - log_prints=True -) -def get_config(): - config = get_named_config() - return config - - -@task( - name="get_unprocessed_corps", - # retries=3, - # retry_delay_seconds=60, - log_prints=True -) -def get_unprocessed_corps(config, db_engine: engine): - print("Getting unprocessed corporations") - logger = prefect.get_run_logger() - query = get_unprocessed_corps_query(config.DATA_LOAD_ENV) - sql_text = text(query) - - with db_engine.connect() as conn: - rs = conn.execute(sql_text) - df = pd.DataFrame(rs, columns=rs.keys()) - raw_data_dict = df.to_dict('records') - corp_nums = [x.get('corp_num') for x in raw_data_dict] - # logger.info(f'{len(raw_data_dict)} corp_nums to process from colin data: {corp_nums}') - status_service = ProcessingStatusService(config.DATA_LOAD_ENV, db_engine) - # TODO: optimize to update all records in one-shot - for corp_num in corp_nums: - status_service.update_flow_status(flow_name='corps-flow', - corp_num=corp_num, - processed_status=ProcessingStatuses.PROCESSING) - return raw_data_dict - - -@task( - name="get_event_filing_data", - # retries=3, - # retry_delay_seconds=60, - # cache_key_fn=task_input_hash, - # cache_expiration=timedelta(minutes=30), # Shorter expiration for data freshness - log_prints=True -) -def get_event_filing_data(config, colin_db_engine: engine, unprocessed_corp_dict: dict): - logger = prefect.get_run_logger() - corp_num = unprocessed_corp_dict.get('corp_num') - print(f"Starting event filing data processing for corp: {corp_num}") - status_service = ProcessingStatusService(config.DATA_LOAD_ENV, colin_db_engine) - event_filing_service = EventFilingService(colin_db_engine, config) - corp_name = '' - - try: - event_ids = unprocessed_corp_dict.get('event_ids') - correction_event_ids = unprocessed_corp_dict.get('correction_event_ids') - events_ids_to_process, event_filing_types_to_process = get_event_info_to_retrieve(unprocessed_corp_dict) - processed_events_ids = get_processed_event_ids(unprocessed_corp_dict) - unprocessed_corp_dict['retrieved_events_cnt'] = len(events_ids_to_process) - event_filing_data_arr = [] - - corp_comments = event_filing_service.get_corp_comments_data(corp_num) - unprocessed_corp_dict['corp_comments'] = corp_comments - unprocessed_corp_dict['correctionEventFilingMappings'] = {} - correction_event_filing_mappings = unprocessed_corp_dict['correctionEventFilingMappings'] - - prev_event_filing_data = None - for idx, event_id in enumerate(events_ids_to_process): - event_file_type = event_filing_types_to_process[idx] - is_supported_event_filing = event_filing_service.get_event_filing_is_supported(event_file_type) - # print(f'event_id: {event_id}, event_file_type: {event_file_type}, is_supported_event_filing: {is_supported_event_filing}') - prev_event_ids = get_previous_event_ids(event_ids, event_id) - event_filing_data_dict, is_corrected_event_filing, correction_event_id = \ - event_filing_service.get_event_filing_data(corp_num, - event_id, - event_file_type, - prev_event_filing_data, - prev_event_ids, - correction_event_ids, - correction_event_filing_mappings) - if is_corrected_event_filing: - correction_event_filing_mappings[correction_event_id] = { - 'correctedEventId': event_id, - 'learFilingType': event_filing_data_dict['target_lear_filing_type'] - } - - event_filing_data_arr.append({ - 'is_in_lear': is_in_lear(processed_events_ids, event_id), - 'is_supported_type': is_supported_event_filing, - 'skip_filing': event_filing_data_dict['skip_filing'], - 'data': event_filing_data_dict - }) - prev_event_filing_data = event_filing_data_dict - - unprocessed_corp_dict['event_filing_data'] = event_filing_data_arr - - except Exception as err: - error_msg = f'error getting event filing data {corp_num}, {corp_name}, {err}' - error_msg_minimal = f'error getting event filing data {corp_num}, {corp_name}' - logger.error(error_msg_minimal) - status_service.update_flow_status(flow_name='corps-flow', - corp_num=corp_num, - corp_name=corp_name, - processed_status=ProcessingStatuses.FAILED, - failed_event_id=event_id, - failed_event_file_type=event_file_type, - last_error=error_msg) - raise CustomException(error_msg_minimal) - - print(f"Completed event filing data processing for corp: {corp_num}") - return unprocessed_corp_dict - - -@task( - name="clean_event_filing_data", - # retries=3, - # retry_delay_seconds=60, - log_prints=True -) -def clean_event_filing_data(config, colin_db_engine: engine, event_filing_data_dict: dict): - logger = prefect.get_run_logger() - corp_num = event_filing_data_dict.get('corp_num') - print(f"Starting data cleaning for corp: {corp_num}") - status_service = ProcessingStatusService(config.DATA_LOAD_ENV, colin_db_engine) - corp_name = '' - event_id = None - event_filing_type = None - - try: - event_filing_data_arr = event_filing_data_dict['event_filing_data'] - for event_filing_data in event_filing_data_arr: - if event_filing_data['is_supported_type'] and not event_filing_data['skip_filing']: - filing_data = event_filing_data['data'] - event_filing_type = filing_data['event_file_type'] - event_id=filing_data['e_event_id'] - clean_event_data(filing_data) - clean_corp_data(config, filing_data) - corp_name = filing_data['curr_corp_name'] - clean_corp_party_data(filing_data) - clean_offices_data(filing_data) - except Exception as err: - error_msg = f'error cleaning business {corp_num}, {corp_name}, {err}' - error_msg_minimal = f'error cleaning business {corp_num}, {corp_name}' - logger.error(error_msg_minimal) - status_service.update_flow_status(flow_name='corps-flow', - corp_num=corp_num, - corp_name=corp_name, - processed_status=ProcessingStatuses.FAILED, - failed_event_id=event_id, - failed_event_file_type=event_filing_type, - last_error=error_msg) - raise CustomException(error_msg_minimal) - - print(f"Completed data cleaning for corp: {corp_num}") - return event_filing_data_dict - - -@task( - name="transform_event_filing_data", - # retries=3, - # retry_delay_seconds=60, - log_prints=True -) -def transform_event_filing_data(config, colin_db_engine: engine, event_filing_data_dict: dict): - logger = prefect.get_run_logger() - corp_num = event_filing_data_dict.get('corp_num') - print(f"Starting data transformation for corp: {corp_num}") - status_service = ProcessingStatusService(config.DATA_LOAD_ENV, colin_db_engine) - corp_name = '' - event_id = None - event_filing_type = None - - try: - event_filing_data_arr = event_filing_data_dict['event_filing_data'] - for event_filing_data in event_filing_data_arr: - if not event_filing_data['is_in_lear'] and event_filing_data['is_supported_type'] \ - and not event_filing_data['skip_filing']: - # process and create LEAR json filing dict - filing_data = event_filing_data['data'] - event_filing_type = filing_data['event_file_type'] - event_id=filing_data['e_event_id'] - corp_name = filing_data['curr_corp_name'] - corp_filing_json_factory_service = FilingJsonFactoryService(event_filing_data) - filing_json = corp_filing_json_factory_service.get_filing_json() - event_filing_data['filing_json'] = filing_json - except Exception as err: - error_msg = f'error transforming business {corp_num}, {corp_name}, {err}' - error_msg_minimal = f'error transforming business {corp_num}, {corp_name}' - logger.error(error_msg_minimal) - status_service.update_flow_status(flow_name='corps-flow', - corp_num=corp_num, - corp_name=corp_name, - processed_status=ProcessingStatuses.FAILED, - failed_event_id=event_id, - failed_event_file_type=event_filing_type, - last_error=error_msg) - raise CustomException(error_msg_minimal) - - print(f"Completed data transformation for corp: {corp_num}") - return event_filing_data_dict - - -@task( - name="load_event_filing", - # retries=3, - # retry_delay_seconds=60, - log_prints=True -) -def load_event_filing_data(config, app: any, colin_db_engine: engine, db_lear, event_filing_data_dict: dict): - logger = prefect.get_run_logger() - corp_num = event_filing_data_dict.get('corp_num') - print(f"Starting data load for corp: {corp_num}") - status_service = ProcessingStatusService(config.DATA_LOAD_ENV, colin_db_engine) - corp_type = event_filing_data_dict['corp_type_cd'] - filings_count = event_filing_data_dict['cnt'] - corp_name = '' - event_id = None - event_filing_type = None - filing = None - filing_processed = False - - with app.app_context(): - try: - event_filing_data_arr = event_filing_data_dict['event_filing_data'] - for idx, event_filing_data in enumerate(event_filing_data_arr): - filing_data = event_filing_data['data'] - event_id=filing_data['e_event_id'] - event_filing_type = filing_data['event_file_type'] - - if not event_filing_data['is_supported_type']: - error_msg = f'could not finish processing this corp as there is an unsupported event/filing type: {event_filing_type}' - raise CustomUnsupportedTypeException(f'{error_msg}') - - if not event_filing_data['is_in_lear'] and not event_filing_data['skip_filing']: - # the corp_processing table should already track whether an event/filing has been processed and - # saved to lear but just to be safe a final check against lear is made to ensure the event/filing - # is not already in lear - # colin_event = get_colin_event(db_lear, event_id) - # if colin_event: - # error_msg = f'colin event id ({event_id}) already exists in lear: {event_filing_type}' - # raise CustomException(f'{error_msg}') - - business = None - if not IAEventFilings.has_value(event_filing_type): - business = Business.find_by_identifier(corp_num) - populate_filing_json_from_lear(db, event_filing_data, business) - corp_name = filing_data['curr_corp_name'] - - # save filing to filing table - filing = populate_filing(business, event_filing_data, filing_data) - filing.save() - - # process filing with custom filer function - business = process_filing(config, filing.id, event_filing_data_dict, filing_data, db_lear) - filing_processed = True - - event_cnt = event_filing_data_dict['retrieved_events_cnt'] - if event_cnt == (idx + 1): - status_service.update_flow_status(flow_name='corps-flow', - corp_num=corp_num, - corp_name=corp_name, - corp_type=corp_type, - filings_count=filings_count, - processed_status=ProcessingStatuses.COMPLETED, - last_processed_event_id=event_id) - print(f"Completed data load for corp: {corp_num}") - else: - status_service.update_flow_status(flow_name='corps-flow', - corp_num=corp_num, - corp_name=corp_name, - processed_status=ProcessingStatuses.PROCESSING, - last_processed_event_id=event_id) - except CustomUnsupportedTypeException as err: - error_msg = f'Partial loading of business {corp_num}, {corp_name}, {err}' - error_msg_minimal = f'Partial loading of business {corp_num}, {corp_name}' - logger.error(error_msg_minimal) - status_service.update_flow_status(flow_name='corps-flow', - corp_num=corp_num, - corp_name=corp_name, - corp_type=corp_type, - filings_count=filings_count, - processed_status=ProcessingStatuses.PARTIAL, - failed_event_id=event_id, - failed_event_file_type=event_filing_type, - last_error=error_msg) - raise CustomException(error_msg_minimal) - except InvalidRequestError as err: - error_msg = f'error loading business InvalidRequestError: {corp_num}, {corp_name}, {err}' - error_msg_minimal = f'error loading business InvalidRequestError: {corp_num}, {corp_name}' - logger.error(error_msg_minimal) - status_service.update_flow_status(flow_name='corps-flow', - corp_num=corp_num, - corp_name=corp_name, - corp_type=corp_type, - filings_count=filings_count, - processed_status=ProcessingStatuses.FAILED, - failed_event_id=event_id, - failed_event_file_type=event_filing_type, - last_error=error_msg) - logger.info(f'filing processed: {filing_processed}') - logger.info('lear db rollback') - db_lear.session.rollback() - - raise CustomException(error_msg_minimal) - except Exception as err: - error_msg = f'error loading business {corp_num}, {corp_name}, {err}' - error_msg_minimal = f'error loading business {corp_num}, {corp_name}' - logger.error(error_msg_minimal) - status_service.update_flow_status(flow_name='corps-flow', - corp_num=corp_num, - corp_name=corp_name, - corp_type=corp_type, - filings_count=filings_count, - processed_status=ProcessingStatuses.FAILED, - failed_event_id=event_id, - failed_event_file_type=event_filing_type, - last_error=error_msg) - logger.info(f'filing processed: {filing_processed}') - logger.info('lear db rollback') - db_lear.session.rollback() - - raise CustomException(error_msg_minimal) - - - -@flow( - name="Corps-Migrate-ETL", - description="Migrate corporation data through ETL process", - version="1.0", - log_prints=True -) -def migrate_flow(): - # setup - config = get_config() - db_colin_engine = colin_init(config) - FLASK_APP, db_lear = lear_init(config) - - unprocessed_corps = get_unprocessed_corps(config, db_colin_engine) - print(f"Found {len(unprocessed_corps)} corps to process") - - # Submit all event filing tasks in parallel - event_filing_futures = [] - for corp in unprocessed_corps: - future = get_event_filing_data.submit( - config=config, - colin_db_engine=db_colin_engine, - unprocessed_corp_dict=corp - ) - event_filing_futures.append(future) - - # Process results and submit cleaning tasks - clean_futures = [] - for future in event_filing_futures: - data = future.result() # Wait for result - if data: - clean_future = clean_event_filing_data.submit( - config=config, - colin_db_engine=db_colin_engine, - event_filing_data_dict=data - ) - clean_futures.append(clean_future) - - # Process cleaning results and submit transform tasks - transform_futures = [] - for future in clean_futures: - data = future.result() # Wait for result - if data: - transform_future = transform_event_filing_data.submit( - config=config, - colin_db_engine=db_colin_engine, - event_filing_data_dict=data - ) - transform_futures.append(transform_future) - - # Process transform results and submit load tasks - load_futures = [] - for future in transform_futures: - data = future.result() # Wait for result - if data: - load_future = load_event_filing_data.submit( - config=config, - app=FLASK_APP, - colin_db_engine=db_colin_engine, - db_lear=db_lear, - event_filing_data_dict=data - ) - load_futures.append(load_future) - - # Wait for all loads to complete - for future in load_futures: - future.result() - - -if __name__ == "__main__": - migrate_flow() - - -# if __name__ == "__main__": -# # Create deployment with schedule -# deployment = migrate_flow.to_deployment( -# name="corps-migration", -# interval=timedelta(seconds=15), # Run every 15 seconds -# tags=["corps-migration"] -# ) - -# # Start serving the deployment -# serve(deployment) diff --git a/data-tool/flows/pyproject.toml b/data-tool/flows/pyproject.toml new file mode 100644 index 0000000000..0ec4d895a1 --- /dev/null +++ b/data-tool/flows/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "common" +version = "1.0.0" +description = "Shared Utilities" +authors = [] +requires-python = ">=3.11" +dependencies = [] + +[tool.setuptools] +packages = ["common"] + +[tool.setuptools.package-dir] +common = "common" + +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" diff --git a/data-tool/flows/refresh_extract_subset_flow.py b/data-tool/flows/refresh_extract_subset_flow.py index c3eedc8d6b..d57e33566b 100644 --- a/data-tool/flows/refresh_extract_subset_flow.py +++ b/data-tool/flows/refresh_extract_subset_flow.py @@ -294,6 +294,8 @@ def extract_pull_flow( corp_file = str(feed_path) result: subprocess.CompletedProcess | None = None print(f'Running CPRD subset extract generator {corp_file}') + # The generated master owns the terminal address-transpose invocation. + # Do not add a second flow-level invocation here. try: result = run_cprd_subset_extract_generator( corp_file=corp_file, diff --git a/data-tool/flows/tombstone/tombstone_base_data.py b/data-tool/flows/tombstone/tombstone_base_data.py index 390b57784d..1e11a60ef0 100644 --- a/data-tool/flows/tombstone/tombstone_base_data.py +++ b/data-tool/flows/tombstone/tombstone_base_data.py @@ -204,9 +204,7 @@ 'submitter_id': None, 'withdrawn_filing_id': None, # others - 'submitter_roles': None, - 'court_order_file_number' : None, # optional - 'court_order_effect_of_order' : None, # optional + 'submitter_roles': None }, 'jurisdiction': None, # optional 'amalgamations': None, # optional diff --git a/data-tool/flows/tombstone/tombstone_mappings.py b/data-tool/flows/tombstone/tombstone_mappings.py index 94d6c86e53..5c24011333 100644 --- a/data-tool/flows/tombstone/tombstone_mappings.py +++ b/data-tool/flows/tombstone/tombstone_mappings.py @@ -68,6 +68,9 @@ class EventFilings(str, Enum): # CONVOTHER Change of Directors CONVOTHER_OTCDR = 'CONVOTHER_OTCDR' + # Change of Name + FILE_OTNCN = 'FILE_OTNCN' + # Consent Continuation Out FILE_CONTO = 'FILE_CONTO' @@ -130,9 +133,11 @@ class EventFilings(str, Enum): FILE_ADVD2 = 'FILE_ADVD2' FILE_ADVDS = 'FILE_ADVDS' FILE_OTVDS = 'FILE_OTVDS' + FILE_OTDIS = 'FILE_OTDIS' # CONVOTHER Dissolution CONVOTHER_OTVDS = 'CONVOTHER_OTVDS' + CONVOTHER_OTDIS = 'CONVOTHER_OTDIS' ## admin SYSDA_NULL = 'SYSDA_NULL' SYSDS_NULL = 'SYSDS_NULL' @@ -279,6 +284,9 @@ def has_value(cls, value): # CONVOTHER Change of Directors EventFilings.CONVOTHER_OTCDR: 'changeOfDirectors', + # Change of Name + EventFilings.FILE_OTNCN: ['conversion', 'changeOfName'], + EventFilings.FILE_CONTO: 'consentContinuationOut', EventFilings.FILE_COUTI: 'continuationOut', @@ -329,9 +337,11 @@ def has_value(cls, value): EventFilings.FILE_ADVD2: ['dissolution', 'voluntary'], EventFilings.FILE_ADVDS: ['dissolution', 'voluntary'], EventFilings.FILE_OTVDS: ['dissolution', 'voluntary'], + EventFilings.FILE_OTDIS: ['dissolution', 'voluntary'], # CONVOTHER Dissolution EventFilings.CONVOTHER_OTVDS: ['dissolution', 'voluntary'], + EventFilings.CONVOTHER_OTDIS: ['dissolution', 'voluntary'], EventFilings.SYSDA_NULL: ['dissolution', 'administrative'], EventFilings.SYSDS_NULL: ['dissolution', 'administrative'], @@ -454,6 +464,9 @@ def has_value(cls, value): # CONVOTHER Change of Directors EventFilings.CONVOTHER_OTCDR: 'Notice of Change of Directors', + # Change of Name + EventFilings.FILE_OTNCN: 'Notice of Change of Name', + EventFilings.FILE_CONTO: '6 Months Consent to Continue Out', EventFilings.FILE_COUTI: 'Instrument of Continuation Out', @@ -488,9 +501,11 @@ def has_value(cls, value): EventFilings.FILE_ADVD2: 'Application for Dissolution (Voluntary Dissolution)', EventFilings.FILE_ADVDS: 'Application for Dissolution (Voluntary Dissolution)', EventFilings.FILE_OTVDS: 'Voluntary Dissolution', + EventFilings.FILE_OTDIS: 'Dissolution', # CONVOTHER Dissolution EventFilings.CONVOTHER_OTVDS: 'Voluntary Dissolution', + EventFilings.CONVOTHER_OTDIS: 'Dissolution', EventFilings.SYSDA_NULL: None, # admin - status Administrative Dissolution EventFilings.SYSDS_NULL: None, # admin - status Administrative Dissolution diff --git a/data-tool/flows/tombstone/tombstone_queries.py b/data-tool/flows/tombstone/tombstone_queries.py index 753035a850..f3a9a5f40e 100644 --- a/data-tool/flows/tombstone/tombstone_queries.py +++ b/data-tool/flows/tombstone/tombstone_queries.py @@ -103,7 +103,7 @@ def get_unprocessed_corps_query(flow_name, config, batch_size, *, include_accoun cte_clause, where_clause = get_unprocessed_corps_subquery(flow_name, environment) - corp_type_filter = "('BC', 'C', 'ULC', 'CUL', 'CC', 'CCC', 'QA', 'QB', 'QC', 'QD', 'QE', 'CP')" + corp_type_filter = "('BC', 'C', 'ULC', 'CUL', 'CC', 'CCC', 'QA', 'QB', 'QC', 'QD', 'QE', 'CP', 'LIB')" mig_extra_where = "" if use_mig_filter: @@ -229,7 +229,7 @@ def get_total_reservable_count_query(flow_name: str, config) -> str: cte_clause, where_clause = get_unprocessed_corps_subquery(flow_name, environment) - corp_type_filter = "('BC', 'C', 'ULC', 'CUL', 'CC', 'CCC', 'QA', 'QB', 'QC', 'QD', 'QE', 'CP')" + corp_type_filter = "('BC', 'C', 'ULC', 'CUL', 'CC', 'CCC', 'QA', 'QB', 'QC', 'QD', 'QE', 'CP', 'LIB')" mig_extra_where = "" if use_mig_filter: diff --git a/data-tool/flows/tombstone/tombstone_utils.py b/data-tool/flows/tombstone/tombstone_utils.py index b63722ea7d..a3fb3508d1 100644 --- a/data-tool/flows/tombstone/tombstone_utils.py +++ b/data-tool/flows/tombstone/tombstone_utils.py @@ -503,9 +503,7 @@ def format_filings_data(data: dict, config=None) -> dict: 'paper_only': paper_only, 'status': status, 'submitter_id': user_id, # will be updated to real user_id when loading data into db - 'submitter_roles': 'staff' if x.get('u_role_typ_cd') and x['u_role_typ_cd'].lower() == 'staff' else None, - 'court_order_file_number' : x['court_order_num'], - 'court_order_effect_of_order' : 'planOfArrangement' if x['arrangement_ind'] == True else None, + 'submitter_roles': 'staff' if x.get('u_role_typ_cd') and x['u_role_typ_cd'].lower() == 'staff' else None } # conversion still need to populate create-new-business info diff --git a/data-tool/requirements/dev.txt b/data-tool/requirements/dev.txt index e69de29bb2..e079f8a603 100755 --- a/data-tool/requirements/dev.txt +++ b/data-tool/requirements/dev.txt @@ -0,0 +1 @@ +pytest diff --git a/data-tool/restore_extract.sh b/data-tool/restore_extract.sh index 7ce1e4b3bf..9366b73152 100755 --- a/data-tool/restore_extract.sh +++ b/data-tool/restore_extract.sh @@ -15,15 +15,16 @@ PGHOST="${PGHOST:-localhost}" # or ‑‑host PGPORT="${PGPORT:-5432}" # or ‑‑port PGUSER="${PGUSER:-postgres}" # or ‑‑user PGDATABASE="${PGDATABASE:-colin-mig-corps-test}" # or ‑‑dbname +PGSCHEMA="${PGSCHEMA:-}" +# Optional PostgreSQL client binary directory. Leave empty to use PATH. +# Example: PG_BIN=/path/to/postgresql/bin ./restore_extract.sh +PG_BIN="${PG_BIN:-}" # Supply the password *either* via a .pgpass file *or* one‑shot: # PGPASSWORD=secret ./restore_extract.sh ############################################################################## -# -- Tables to restore ------------------------------------------------------------ -RESTORE=(corp_processing auth_processing auth_component_operation colin_tracking mig_group mig_batch mig_corp_batch mig_corp_account corps_with_third_party email_domain_groups bar_corps bad_emails exclude_corps excluded_emails excluded_email_domains excluded_email_domain_patterns) - # -- Runtime options ----------------------------------------------------------- -DUMP="${DUMP}" +DUMP="${DUMP:-}" DELTA_MODE="${DELTA_MODE:-false}" ############################################################################## @@ -32,14 +33,85 @@ DELTA_MODE="${DELTA_MODE:-false}" die() { printf >&2 "error: %s\n" "$*"; exit 1; } +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PRESERVED_TABLES_CONF="$SCRIPT_DIR/scripts/restore/preserved_tables.conf" +REPAIR_SEQUENCES_SQL="$SCRIPT_DIR/scripts/restore/repair_sequences.sql" +DELTA_RESTORE_SCRIPT="$SCRIPT_DIR/delta_restore_extract.sh" + +pg_tool() { + local tool="$1" + if [[ -n "$PG_BIN" ]]; then + printf '%s/%s' "${PG_BIN%/}" "$tool" + else + printf '%s' "$tool" + fi +} + +PSQL_BIN="$(pg_tool psql)" +PG_RESTORE_BIN="$(pg_tool pg_restore)" + # Build a single “‑h … ‑p … ‑U …” string so every call is consistent pg_conn_opts() { printf -- "-h %s -p %s -d %s -U %s" "$PGHOST" "$PGPORT" "$PGDATABASE" "$PGUSER" } -# Pass arrays (e.g., KEEP) as repeated --table switches +# Pass arrays (e.g., RESTORE) as repeated --table switches as_table_opts() { local t; for t in "$@"; do printf -- '--table=%s ' "$t"; done; } +load_preserved_tables() { + local table _rest + + [[ -f "$PRESERVED_TABLES_CONF" ]] || die "missing preserved table config: $PRESERVED_TABLES_CONF" + + RESTORE=() + while read -r table _rest; do + case "${table:-}" in + ''|'#'*) continue ;; + *) RESTORE+=("$table") ;; + esac + done < "$PRESERVED_TABLES_CONF" + + [[ "${#RESTORE[@]}" -gt 0 ]] || die "no preserved tables found in $PRESERVED_TABLES_CONF" +} + +############################################################################## +# ── ARGUMENTS / DELTA GUARDS # +############################################################################## + +delta_requested=false +delta_args=() +for arg in "$@"; do + case "$arg" in + --delta) + delta_requested=true + ;; + --schema=*) + PGSCHEMA="${arg#--schema=}" + ;; + *) + delta_args+=("$arg") + ;; + esac +done + +if [[ "$delta_requested" = "true" ]]; then + if [[ -f "$DELTA_RESTORE_SCRIPT" ]]; then + exec bash "$DELTA_RESTORE_SCRIPT" "${delta_args[@]}" + fi + die "--delta requested, but $DELTA_RESTORE_SCRIPT is not available yet. Use data-tool/delta_restore_extract.sh when it is added." +fi + +if [[ "${#delta_args[@]}" -gt 0 ]]; then + die "unknown argument(s): ${delta_args[*]}" +fi + +if [[ "$DELTA_MODE" = "true" ]]; then + die "DELTA_MODE=true is no longer supported by restore_extract.sh. Use data-tool/delta_restore_extract.sh instead." +fi + +[[ -n "$DUMP" ]] || die "DUMP is required, e.g. DUMP=/backups/keep_YYYY-MM-DD.dump ./restore_extract.sh" +[[ -f "$REPAIR_SEQUENCES_SQL" ]] || die "missing sequence repair SQL: $REPAIR_SEQUENCES_SQL" +load_preserved_tables ############################################################################## # ── RECREATE THE DATABASE FROM ORACLE # @@ -51,108 +123,45 @@ printf "🔄 Re‑importing Postgres from Oracle …\n" ############################################################################## # -- EMPTY the tables but keep their structure # ############################################################################## +if [[ -n "$PGSCHEMA" ]]; then + printf "Disabling RI Constraints" "$PGSCHEMA" + "$PSQL_BIN" $(pg_conn_opts) -v ON_ERROR_STOP=1 -c "CALL ${PGSCHEMA}.colin_fk_drop_all(NULL);" +fi printf "🧹 Truncating existing rows …\n" -if [ "$DELTA_MODE" = "true" ]; then - printf "⚠️ DELTA_MODE is true: skipping truncation of preserved tables.\n" -else - printf "DELTA_MODE is false: truncating preserved tables.\n" - psql $(pg_conn_opts) -v ON_ERROR_STOP=1 -q <&1 | grep -v "ERROR" || true - psql $(pg_conn_opts) -v ON_ERROR_STOP=0 <<'MERGE' -INSERT INTO corp_processing_old -SELECT * FROM corp_processing -ON CONFLICT (corp_num, flow_name, environment) DO UPDATE -SET - corp_type_cd = EXCLUDED.corp_type_cd, - corp_name = EXCLUDED.corp_name, - filings_count = EXCLUDED.filings_count, - processed_status = EXCLUDED.processed_status, - failed_event_file_type = EXCLUDED.failed_event_file_type, - last_processed_event_id = EXCLUDED.last_processed_event_id, - failed_event_id = EXCLUDED.failed_event_id, - last_error = EXCLUDED.last_error, - claimed_at = EXCLUDED.claimed_at, - flow_run_id = EXCLUDED.flow_run_id, - mig_batch_id = EXCLUDED.mig_batch_id, - account_ids = EXCLUDED.account_ids, - last_modified = EXCLUDED.last_modified; - -DROP TABLE corp_processing; -ALTER TABLE corp_processing_old RENAME TO corp_processing; -MERGE - +printf "🚚 Copying preserved rows (constraints temporarily disabled) …\n" +if [[ -n "$PGSCHEMA" ]]; then + "$PG_RESTORE_BIN" --section=data --data-only \ + --disable-triggers \ + $(as_table_opts "${RESTORE[@]}") -f - "$DUMP" \ + | sed -E "s/^SET search_path = public/SET search_path = ${PGSCHEMA}/; s/^(COPY|ALTER TABLE) public\\./\\1 ${PGSCHEMA}./" \ + | "$PSQL_BIN" $(pg_conn_opts) -v ON_ERROR_STOP=1 -q else - printf "🚚 Copying preserved rows (constraints temporarily disabled) …\n" - pg_restore $(pg_conn_opts) --section=data --data-only \ + "$PG_RESTORE_BIN" $(pg_conn_opts) --section=data --data-only \ --disable-triggers \ $(as_table_opts "${RESTORE[@]}") "$DUMP" fi - ############################################################################## # ── FIX ANY SEQUENCES # ############################################################################## printf "🛠 Advancing sequences …\n" -psql $(pg_conn_opts) <<'SQL' -DO $$ -DECLARE - r record; - max_val bigint; -BEGIN - FOR r IN - -- For SERIAL/IDENTITY columns (auto-dependency from sequence to column) - SELECT seq.relname AS seq, - tbl.relname AS tbl, - col.attname AS col - FROM pg_class seq - JOIN pg_depend d ON d.objid = seq.oid AND d.deptype = 'a' - JOIN pg_class tbl ON tbl.oid = d.refobjid - JOIN pg_attribute col - ON col.attrelid = tbl.oid AND col.attnum = d.refobjsubid - WHERE seq.relkind = 'S' - UNION ALL - -- For columns with DEFAULT nextval('sequence') (normal dependency from column default to sequence) - SELECT - seq.relname as seq, - tbl.relname as tbl, - col.attname as col - FROM pg_depend d - JOIN pg_class seq ON seq.oid = d.refobjid AND seq.relkind = 'S' - JOIN pg_attrdef ad ON ad.oid = d.objid AND d.classid = 'pg_attrdef'::regclass - JOIN pg_attribute col ON col.attrelid = ad.adrelid AND col.attnum = ad.adnum - JOIN pg_class tbl ON tbl.oid = ad.adrelid - WHERE d.deptype = 'n' - LOOP - EXECUTE format('SELECT MAX(%I) FROM %I', r.col, r.tbl) INTO max_val; - IF max_val IS NULL THEN - -- Table is empty, reset sequence to start at 1. The next call to nextval() will return 1. - EXECUTE format('SELECT setval(%L, 1, false);', r.seq); - ELSE - -- Table has data, set sequence so nextval() returns max_val + 1. - EXECUTE format('SELECT setval(%L, %s);', r.seq, max_val); - END IF; - END LOOP; -END; -$$; -SQL +if [[ -n "$PGSCHEMA" ]]; then + "$PSQL_BIN" $(pg_conn_opts) -c "SET search_path TO ${PGSCHEMA}" -f "$REPAIR_SEQUENCES_SQL" + printf "Enabling RI Constraints" "$PGSCHEMA" + "$PSQL_BIN" $(pg_conn_opts) -v ON_ERROR_STOP=1 -c "CALL ${PGSCHEMA}.colin_fk_restore_all(NULL);" +else + "$PSQL_BIN" $(pg_conn_opts) -f "$REPAIR_SEQUENCES_SQL" +fi printf "✅ Done. Preserved tables restored; sequences synchronised.\n" diff --git a/data-tool/scripts/LIB_Migration_ChangeLog.md b/data-tool/scripts/LIB_Migration_ChangeLog.md new file mode 100644 index 0000000000..7c431a5f33 --- /dev/null +++ b/data-tool/scripts/LIB_Migration_ChangeLog.md @@ -0,0 +1,68 @@ +# Libraries (LIB) Migration - Change Log + +## Overview + +Data Mapping findings and code/SQL changes needed to migrate Library +entities from COLIN Oracle to LEAR Business DB and Auth DB. + +--- + +## 1. COLIN Extract Script Changes + +### File: `data-tool/scripts/transfer_cprd_corps.sql` → copied as `transfer_cprd_lib_only.sql` + +**Entity Type Filter** + +- Libraries `corp_type_cd` in COLIN/LEAR: **`LIB`** +- Replaced `('BC', 'C', 'ULC', 'CUL', 'CC', 'CCC', 'QA', 'QB', 'QC', 'QD', 'QE')` with + `('LIB')` across all 32 occurrences in `transfer_cprd_lib_only.sql`. + +**Optional Tables - All Empty (0 rows) for Libraries** + +| Table | Join path to corp_type_cd | Row count | +| ------------------- | --------------------------------------------------- | --------- | +| `share_struct` | direct `corp_num` | 0 | +| `cont_out` | direct `corp_num` | 0 | +| `corp_restriction` | direct `corp_num` | 0 | +| `resolution` | direct `corp_num` | 0 | +| `share_struct_cls` | direct `corp_num` | 0 | +| `share_series` | direct `corp_num` | 0 | +| `submitting_party` | `event_id` → `event.corp_num` | 0 | +| `party_notification`| `party_id` → `corp_party.corp_party_id` → `corp_num` | 0 | +| `correction` | `event_id` → `event.corp_num` | 0 | + +Commented out all 9 of these transfer blocks in `transfer_cprd_lib_only.sql` +(`share_struct`, `share_struct_cls`, `share_series`, `submitting_party`, `party_notification`, +`cont_out`, `corp_restriction`, `correction`, `resolution`). + +**Optional Change: Trimmed `address` Transfer UNION** + +- Removed 2 UNION branches referencing `submitting_party` and `party_notification`. + +## 2. Filing/Event Type Mapping + +Queried `event`/`filing` joined to `corporation` for `corp_typ_cd = 'LIB'`: + +| filing_typ_cd | event_typ_cd | count | Status | +| --- | --- | --- | --- | +| OTAMA | CONVOTHER | 96 | Already mapped (`amalgamationApplication`) | +| OTDIS | FILE | 69 | **New** | +| OTINC | CONVOTHER | 13 | Already mapped (`incorporationApplication`) | +| OTNCN | FILE | 7 | **New** | +| OTDIS | CONVOTHER | 3 | **New** | + +- `FILE_OTDIS` / `CONVOTHER_OTDIS` → `['dissolution', 'voluntary']`, display "Dissolution". +- `FILE_OTNCN` → `'changeOfName'`, display "Notice of Change of Name". New mapping. + +## 3. Tombstone / Auth Flow Changes + +- `flows/tombstone/tombstone_mappings.py` — added `FILE_OTDIS`, `CONVOTHER_OTDIS`, `FILE_OTNCN` to + the `EventFilings` enum, `EVENT_FILING_LEAR_TARGET_MAPPING`, and + `EVENT_FILING_DISPLAY_NAME_MAPPING` (see Section 2). +- `flows/tombstone/tombstone_queries.py` — added `'LIB'` to `corp_type_filter` (both occurrences, + lines 106 and 232). +- `flows/auth/auth_queries.py` — added `'LIB'` to `CORP_TYPE_FILTER` (line 18). + +## 4. Next Steps + +TBD diff --git a/data-tool/scripts/README_COLIN_Corps_Extract.md b/data-tool/scripts/README_COLIN_Corps_Extract.md index b79b50ae44..850432f11c 100644 --- a/data-tool/scripts/README_COLIN_Corps_Extract.md +++ b/data-tool/scripts/README_COLIN_Corps_Extract.md @@ -10,16 +10,20 @@ ### create empty db for the first time ```bash createdb -h localhost -p 5432 -U postgres -T template0 colin-mig-corps-data-test && -psql -h localhost -p 5432 -U postgres -d colin-mig-corps-data-test -f /data-tool/scripts/colin_corps_extract_postgres_ddl +psql -v ON_ERROR_STOP=1 -h localhost -p 5432 -U postgres -d colin-mig-corps-data-test -f /data-tool/scripts/colin_corps_extract_postgres_ddl kill connection & recreate empty db psql -h localhost -p 5432 -U postgres -d colin-mig-corps-data-test -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE datname = 'colin-mig-corps-data-test' AND pid <> pg_backend_pid();" && dropdb -h localhost -p 5432 -U postgres colin-mig-corps-data-test && createdb -h localhost -p 5432 -U postgres -T template0 colin-mig-corps-data-test && -psql -h localhost -p 5432 -U postgres -d colin-mig-corps-test -f /data-tool/scripts/colin_corps_extract_postgres_ddl +psql -v ON_ERROR_STOP=1 -h localhost -p 5432 -U postgres -d colin-mig-corps-test -f /data-tool/scripts/colin_corps_extract_postgres_ddl ``` +The canonical DDL installs the address-transpose routines through +`\ir colin_address_transpose_updates.sql`. Run this DDL with `psql`; it installs +the routines but does not run the transpose. + 3. Install DbSchemaCLI (previously named DbShell) - [DbSchemaCLI | Free Universal SQL Command-Line Client](https://dbschema.com/dbschemacli.html). More general info around configuration and data transfer can be found at: [DbSchemaCLI | Universal Command Line Client](https://dbschema.com/documentation/dbschemacli.html) @@ -64,7 +68,27 @@ Transfer using 8 thread(s) filing_user ... 20889 rows in 00:27. Reader wait Notes: 1. Update number of threads (e.g. `vset cli.settings.transfer_threads=4`) to use as appropriate in `transfer_cprd_corps.sql`. -2. As of February 2026, the database extract script runs without error using DbSchema 9.7.1(DbSchemaCLI #250319). +2. After loading the data, `transfer_cprd_corps.sql` calls + `public.colin_address_transpose(NULL)` while holding the advisory lock, then + releases the lock. DbSchemaCLI displays one JSON value with eight phase counts, their total, and elapsed seconds. + +--- + +## Preserved-table backup, full restore, and delta restore + +The preserved migration/tracking/auth side-table list is centralized in `data-tool/scripts/restore/preserved_tables.conf` and is shared by: + +- `data-tool/backup_extract_tables.sh` — creates a preserved-table dump and `.manifest.json` sidecar. +- `data-tool/restore_extract.sh` — full preserved-table restore after an extract rebuild. +- `data-tool/delta_restore_extract.sh` — preview/validate/apply merge for preserved tables. + +Start with the [preserved-table overview](restore/README_add_preserved_table_overview.md). To add a table, use the [quick reference](restore/README_add_preserved_table_quickref.md) or [full guide](restore/README_add_preserved_table.md). + +For the delta workflow, see [restore/README_delta_restore.md](restore/README_delta_restore.md). It provides the golden path, a complete CLI/environment reference, and troubleshooting by exit code. + +`DELTA_MODE=true ./restore_extract.sh` has been removed by design. The full restore script now aborts with a pointer to `data-tool/delta_restore_extract.sh`; run explicit delta preview, validate, and apply commands instead. + +Delta restore uses the same PostgreSQL advisory-lock SQL as the subset/full-refresh scripts. Avoid running migration flows or other non-cooperating writers against the target database during apply. --- @@ -268,6 +292,7 @@ Core templates (always used in subset workflow): - `data-tool/scripts/subset/subset_pg_cleanup_orphan_children.sql` (refresh-only orphan child cleanup before chunked deletes) - `data-tool/scripts/subset/subset_pg_boolean_casts.sql` (required so DbSchemaCLI can insert 't'/'f' into boolean columns) - `data-tool/scripts/subset/subset_pg_purge_bcomps_excluded.sql` (run once after transfer) +- `data-tool/scripts/subset/subset_pg_call_address_transpose.sql` (runs address transpose once) Optional Postgres session performance templates (only when `--pg-fastload` is used): - `data-tool/scripts/subset/subset_pg_fastload_begin.sql` @@ -332,13 +357,12 @@ Generated scripts (gitignored by default): Optional flags: - Add `--include-cp` to opt in corp type `CP` for the subset transfer queries. - `--include-cp` affects the **subset workflow only**. Full refresh (`transfer_cprd_corps.sql`) and downstream reservation flows still use the historical corp-type cohort unless updated separately. - Optional performance flags: - Add `--pg-fastload` to enable Postgres session settings for faster bulk writes (templates `subset_pg_fastload_begin.sql` / `subset_pg_fastload_end.sql`). - `--pg-disable-method` currently accepts only `table_triggers` and `replica_role`. - The actual generator default is `table_triggers`, not `replica_role`. - In refresh mode, preserved rows in `corp_processing`, `auth_processing`, `affiliation_processing`, and `colin_tracking` still reference `corporation` / `event`, so FK enforcement must stay suppressed across delete/reload. The generator now adds refresh-only trigger suppression for those preserved FK-owning tables when `--pg-disable-method table_triggers` is used. - - Subset load/refresh scripts now acquire a session-level Postgres advisory lock at the start and release it at the end so overlapping subset runs on the same target DB serialize instead of interleaving. The same lock key is also used by the full refresh script. + - Subset load/refresh scripts acquire a session-level Postgres advisory lock at the start and release it at the end. The same lock key is used by the full refresh script. - `table_triggers` changes table trigger state globally while the refresh runs, so use it against a quiesced/disposable extract DB and with a role that can disable the relevant triggers. - If you use `replica_role`, remember it is session-local. If FK errors still occur, verify `current_setting('session_replication_role')` inside the nested delete/purge scripts being executed by DbSchemaCLI. - `address` is treated as a shared/global table during subset refresh/load. The generator now reuses the predeclared helper staging table `public.subset_address_stage`, transfers incoming Oracle addresses into it, and merges them into `public.address` by `addr_id` instead of deleting/reinserting address rows directly. The address extract also includes `notification_resend` references. @@ -368,6 +392,9 @@ Generated scripts (gitignored by default): - If `--no-cars` is used, cars* tables are skipped entirely. - Chunk templates are rendered at generation time (inline mode), so the resulting SQL is self-contained. - If you need legacy runtime substitution behavior, generate with `--render-mode vset` (uses DbSchemaCLI `vset` + &placeholders). +- The generated master calls address transpose once after all chunks are loaded + and before releasing the advisory lock. The returned JSON includes each phase count, + their total, and elapsed seconds. ### Oracle IN-list strategy (`--oracle-in-strategy`) diff --git a/data-tool/scripts/colin_address_transpose_updates.sql b/data-tool/scripts/colin_address_transpose_updates.sql index 38560c2312..1c462463b7 100644 --- a/data-tool/scripts/colin_address_transpose_updates.sql +++ b/data-tool/scripts/colin_address_transpose_updates.sql @@ -1,7 +1,7 @@ /* This file contains all functions and stored procedures for approved address record transpose updates. The changes only apply to active parties and offices belonging to active businesses. See the comment for the procedure -"colin_address_transpose" for a summary of the specific changes. +"colin_address_transpose" for a summary of the specific changes. The transpose changes are intended to be run as part of a colin data migration before the extract is loaed into the target, modernized business database. @@ -9,13 +9,13 @@ the target, modernized business database. Usage: 1. Compile all the functions and stored procedures in this file. 2. Run the procedure to transpose the addresses (takes approximately 7 minutes): - call colin_address_transpose(); + call colin_address_transpose(NULL); Example address ID's and these instructions are in this spreadsheet: https://docs.google.com/spreadsheets/d/1-BLTklhdiW0401sN6I68inF1-retATANyTZDswBwGcQ/edit?gid=642728421#gid=642728421 --- Uncomment and executed to drop all address transpose functions and procedures. +-- Uncomment and execute to drop all address transpose routines. drop procedure colin_address_transpose; drop procedure colin_address_transpose_cleanup; drop procedure colin_address_transpose_city; @@ -31,9 +31,22 @@ drop function colin_update_address_pcode; drop function colin_update_address_country; */ +-- OUT parameters cannot replace zero-OUT procedure contracts in place. +-- Recreate the procedure signatures atomically. +BEGIN; +DROP PROCEDURE IF EXISTS public.colin_address_transpose(); +DROP PROCEDURE IF EXISTS public.colin_address_transpose_cleanup(); +DROP PROCEDURE IF EXISTS public.colin_address_transpose_city(); +DROP PROCEDURE IF EXISTS public.colin_address_duplicate_region(); +DROP PROCEDURE IF EXISTS public.colin_address_transpose_region(); +DROP PROCEDURE IF EXISTS public.colin_address_duplicate_pcode(); +DROP PROCEDURE IF EXISTS public.colin_address_transpose_pcode(); +DROP PROCEDURE IF EXISTS public.colin_address_duplicate_country(); +DROP PROCEDURE IF EXISTS public.colin_address_transpose_country(); + -- --- Try and extract a Canada or US postal code from address text: city, then address_line_3, then address_line_2 +-- Try and extract a Canada or US postal code from address text: city, then address_line_3, then address_line_2 -- create or replace function public.colin_update_address_pcode(v_text character varying, v_remove boolean, @@ -63,8 +76,8 @@ begin elsif not v_us then return null; else - if position('BOX' in test_val) > 0 or position('PO ' in test_val) > 0 or position('P.O ' in test_val) > 0 - or position('POB ' in test_val) > 0 + if position('BOX' in test_val) > 0 or position('PO ' in test_val) > 0 or position('P.O ' in test_val) > 0 + or position('POB ' in test_val) > 0 or position('CP ' in test_val) > 0 then return null; end if; @@ -81,7 +94,7 @@ begin return test_code; end if; end if; - elsif REGEXP_MATCHES(REPLACE(test_code, '-', ''),'^[0-9]+$','gi') is not null then + elsif REGEXP_MATCHES(REPLACE(test_code, '-', ''),'^[0-9]+$','gi') is not null then if v_remove then return trim(substr(TRIM(v_text), 1, length(TRIM(v_text)) - length(test_code))); else @@ -169,7 +182,7 @@ begin else return country[2]; end if; - elsif country[1] in ('PEOPLES REPUBLIC OF CHINA', 'P R OF CHINA', 'TAIWAN REPUBLIC OF CHINA', 'EAST MALAYSIA', 'HONG KONG') and + elsif country[1] in ('PEOPLES REPUBLIC OF CHINA', 'P R OF CHINA', 'TAIWAN REPUBLIC OF CHINA', 'EAST MALAYSIA', 'HONG KONG') and POSITION(country[1] in test_text) > 0 then if v_remove then return trim(substr(TRIM(v_text), 1, length(TRIM(v_text)) - length(country[1]))); @@ -185,7 +198,7 @@ $$; -- --- Try and extract a Canada province/region code from address text: city, then address_line_3, then address_line_2 +-- Try and extract a Canada province/region code from address text: city, then address_line_3, then address_line_2 -- create or replace function public.colin_update_address_ca_province(v_text character varying, v_remove boolean) returns character varying immutable @@ -198,8 +211,8 @@ declare ARRAY['B .C.','BC'], ARRAY['B. C.','BC'], ARRAY['BC,','BC'], - ARRAY['B.C. )','BC'], - ARRAY['BC)','BC'], + ARRAY['B.C. )','BC'], + ARRAY['BC)','BC'], ARRAY['BC','BC'], ARRAY['B.C.,','BC'], ARRAY['B.C.','BC'], @@ -208,7 +221,7 @@ declare ARRAY['BC.','BC'], ARRAY['B,C.','BC'], ARRAY['B.C..','BC'], - ARRAY['B.C .','BC'], + ARRAY['B.C .','BC'], ARRAY['AB','AB'], ARRAY['ALBERTA','AB'], ARRAY['SK','SK'], @@ -262,7 +275,7 @@ $$; -- --- Try and extract a US state code from address text: city, then address_line_3, then address_line_2 +-- Try and extract a US state code from address text: city, then address_line_3, then address_line_2 -- create or replace function public.colin_update_address_us_state(v_text character varying, v_remove boolean) returns character varying immutable @@ -323,11 +336,16 @@ $$; -- -- Transpose corp_party address mailing and delivery country from addr_line3 and addr_line2. --- call colin_address_transpose_country(); +-- call colin_address_transpose_country(NULL); -- -CREATE OR REPLACE PROCEDURE public.colin_address_transpose_country() +CREATE OR REPLACE PROCEDURE public.colin_address_transpose_country( + OUT row_occurrences bigint +) LANGUAGE plpgsql AS $$ +DECLARE + v_step_rows bigint := 0; + v_phase_rows bigint := 0; BEGIN -- Delivery/Mailing line 3 update address @@ -372,6 +390,8 @@ BEGIN a.addr_line_3 like '%VIETNAM%') ) and colin_update_address_country(addr_line_3, false) is not null; + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Delivery/Mailing line 2 @@ -417,19 +437,27 @@ BEGIN a.addr_line_2 like '%VIETNAM%') ) and colin_update_address_country(addr_line_2, false) is not null; + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; + row_occurrences := v_phase_rows; END; $$; -- -- Remove duplicate country from city, address line 3, address line 2. --- call colin_address_duplicate_country(); +-- call colin_address_duplicate_country(NULL); -- -CREATE OR REPLACE PROCEDURE public.colin_address_duplicate_country() +CREATE OR REPLACE PROCEDURE public.colin_address_duplicate_country( + OUT row_occurrences bigint +) LANGUAGE plpgsql AS $$ +DECLARE + v_step_rows bigint := 0; + v_phase_rows bigint := 0; BEGIN -- Corp party delivery/mailing city update address @@ -481,6 +509,8 @@ BEGIN a.city like '%VIETNAM%') ) and country_typ_cd = colin_update_address_country(city, false); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Corp party delivery/mailing line 3 @@ -531,6 +561,8 @@ BEGIN a.addr_line_3 like '%VIETNAM%') ) and country_typ_cd = colin_update_address_country(addr_line_3, false); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Corp party delivery/mailing line 2 @@ -581,19 +613,27 @@ BEGIN a.addr_line_2 like '%VIETNAM%') ) and country_typ_cd = colin_update_address_country(addr_line_2, false); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; + row_occurrences := v_phase_rows; END; $$; -- -- Transpose corp_party address mailing and delivery postal/us zip code from city, addr_line3, and addr_line2. --- call colin_address_transpose_pcode(); +-- call colin_address_transpose_pcode(NULL); -- -CREATE OR REPLACE PROCEDURE public.colin_address_transpose_pcode() +CREATE OR REPLACE PROCEDURE public.colin_address_transpose_pcode( + OUT row_occurrences bigint +) LANGUAGE plpgsql AS $$ +DECLARE + v_step_rows bigint := 0; + v_phase_rows bigint := 0; BEGIN -- Corp party delivery/mailing city update address @@ -614,6 +654,8 @@ BEGIN and length(trim(a.city)) > 5 ) and colin_update_address_pcode(city, false, false) is not null; + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; update address set postal_cd = colin_update_address_pcode(city, false, true), @@ -633,6 +675,8 @@ BEGIN and length(trim(a.city)) >= 5 ) and colin_update_address_pcode(city, false, true) is not null; + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Corp party delivery/mailing line 3 @@ -654,6 +698,8 @@ BEGIN and length(trim(a.addr_line_3)) > 5 ) and colin_update_address_pcode(addr_line_3, false, false) is not null; + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; update address set postal_cd = colin_update_address_pcode(addr_line_3, false, true), @@ -673,6 +719,8 @@ BEGIN and length(trim(a.addr_line_3)) >= 5 ) and colin_update_address_pcode(addr_line_3, false, true) is not null; + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Corp party delivery/mailing line 2 @@ -694,6 +742,8 @@ BEGIN and length(trim(a.addr_line_2)) > 5 ) and colin_update_address_pcode(addr_line_2, false, false) is not null; + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; update address set postal_cd = colin_update_address_pcode(addr_line_2, false, true), @@ -713,6 +763,8 @@ BEGIN and length(trim(a.addr_line_2)) >= 5 ) and colin_update_address_pcode(addr_line_2, false, true) is not null; + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Office delivery/mailing city @@ -734,19 +786,27 @@ BEGIN and length(trim(a.city)) > 5 ) and colin_update_address_pcode(city, false, false) is not null; + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; + row_occurrences := v_phase_rows; END; $$; -- -- Format and remove duplicate CA postal codes from city, address line 3, address line 2. --- call colin_address_duplicate_pcode(); +-- call colin_address_duplicate_pcode(NULL); -- -CREATE OR REPLACE PROCEDURE public.colin_address_duplicate_pcode() +CREATE OR REPLACE PROCEDURE public.colin_address_duplicate_pcode( + OUT row_occurrences bigint +) LANGUAGE plpgsql AS $$ +DECLARE + v_step_rows bigint := 0; + v_phase_rows bigint := 0; BEGIN -- Corp party format delivery/mailing update address @@ -764,6 +824,8 @@ BEGIN and a.postal_cd is not null and length(trim(a.postal_cd)) = 6 and a.postal_cd ~ '^[A-Za-z]\d[A-Za-z]\d[A-Za-z]\d$' ); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Office format delivery/mailing @@ -782,6 +844,8 @@ BEGIN and a.postal_cd is not null and length(trim(a.postal_cd)) = 6 and a.postal_cd ~ '^[A-Za-z]\d[A-Za-z]\d[A-Za-z]\d$' ); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Corp party delivery/mailing city @@ -804,6 +868,8 @@ BEGIN and trim(right(a.city, 7)) ~ '^[A-Za-z]\d[A-Za-z][ -]?\d[A-Za-z]\d$' ) and postal_cd = colin_update_address_pcode(city, false, false); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Corp party delivery/mailing line 3 @@ -826,6 +892,8 @@ BEGIN and trim(right(a.addr_line_3, 7)) ~ '^[A-Za-z]\d[A-Za-z][ -]?\d[A-Za-z]\d$' ) and postal_cd = colin_update_address_pcode(addr_line_3, false, false); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Corp party delivery/mailing line 2 @@ -848,18 +916,27 @@ BEGIN and trim(right(a.addr_line_2, 7)) ~ '^[A-Za-z]\d[A-Za-z][ -]?\d[A-Za-z]\d$' ) and postal_cd = colin_update_address_pcode(addr_line_2, false, false); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; + + row_occurrences := v_phase_rows; END; $$; -- -- Transpose corp_party address mailing and delivery province/region code from city, addr_line3, and addr_line2. --- call colin_address_transpose_region(); +-- call colin_address_transpose_region(NULL); -- -CREATE OR REPLACE PROCEDURE public.colin_address_transpose_region() +CREATE OR REPLACE PROCEDURE public.colin_address_transpose_region( + OUT row_occurrences bigint +) LANGUAGE plpgsql AS $$ +DECLARE + v_step_rows bigint := 0; + v_phase_rows bigint := 0; BEGIN -- Corp party city - some instances after previous transpose steps. update address @@ -888,11 +965,13 @@ BEGIN a.city like '%SASKATCHEWAN' or a.city like '% ONTARIO' or a.city like '% NEWFOUNDLAND' or - a.city like '% ALBERTA%' or + a.city like '% ALBERTA%' or a.city like '% NOVA SCOTIA' or a.city like '% YUKON') ) and colin_update_address_ca_province(city, false) is not null; + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; update address set province = colin_update_address_us_state(city, false), @@ -926,6 +1005,8 @@ BEGIN a.city like '% NY %') ) and colin_update_address_us_state(city, false) is not null; + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; @@ -956,11 +1037,13 @@ BEGIN a.addr_line_3 like '%SASKATCHEWAN' or a.addr_line_3 like '% ONTARIO' or a.addr_line_3 like '% NEWFOUNDLAND' or - a.addr_line_3 like '% ALBERTA%' or + a.addr_line_3 like '% ALBERTA%' or a.addr_line_3 like '% NOVA SCOTIA' or a.addr_line_3 like '% YUKON') ) and colin_update_address_ca_province(addr_line_3, false) is not null; + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; update address set province = colin_update_address_us_state(addr_line_3, false), @@ -994,6 +1077,8 @@ BEGIN a.addr_line_3 like '% NY %') ) and colin_update_address_us_state(addr_line_3, false) is not null; + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Corp party delivery/mailing line 2 @@ -1023,11 +1108,13 @@ BEGIN a.addr_line_2 like '%SASKATCHEWAN' or a.addr_line_2 like '% ONTARIO' or a.addr_line_2 like '% NEWFOUNDLAND' or - a.addr_line_2 like '% ALBERTA%' or + a.addr_line_2 like '% ALBERTA%' or a.addr_line_2 like '% NOVA SCOTIA' or a.addr_line_2 like '% YUKON') ) and colin_update_address_ca_province(addr_line_2, false) is not null; + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; update address set province = colin_update_address_us_state(addr_line_2, false), @@ -1061,18 +1148,27 @@ BEGIN a.addr_line_2 like '% NY %') ) and colin_update_address_us_state(addr_line_2, false) is not null; + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; + + row_occurrences := v_phase_rows; END; $$; -- -- Remove duplicate corp_party address mailing and delivery province/region codes from city, addr_line3, and addr_line2. --- call colin_address_duplicate_region(); +-- call colin_address_duplicate_region(NULL); -- -CREATE OR REPLACE PROCEDURE public.colin_address_duplicate_region() +CREATE OR REPLACE PROCEDURE public.colin_address_duplicate_region( + OUT row_occurrences bigint +) LANGUAGE plpgsql AS $$ +DECLARE + v_step_rows bigint := 0; + v_phase_rows bigint := 0; BEGIN -- Corp party delivery/mailing city update address @@ -1099,11 +1195,13 @@ BEGIN a.city like '%SASKATCHEWAN' or a.city like '% ONTARIO' or a.city like '% NEWFOUNDLAND' or - a.city like '% ALBERTA%' or + a.city like '% ALBERTA%' or a.city like '% NOVA SCOTIA' or a.city like '% YUKON') ) and province = colin_update_address_ca_province(city, false); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Corp party delivery/mailing line 3 @@ -1131,11 +1229,13 @@ BEGIN a.addr_line_3 like '%SASKATCHEWAN' or a.addr_line_3 like '% ONTARIO' or a.addr_line_3 like '% NEWFOUNDLAND' or - a.addr_line_3 like '% ALBERTA%' or + a.addr_line_3 like '% ALBERTA%' or a.addr_line_3 like '% NOVA SCOTIA' or a.addr_line_3 like '% YUKON') ) and province = colin_update_address_ca_province(addr_line_3, false); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Corp party delivery/mailing line 2 @@ -1163,24 +1263,32 @@ BEGIN a.addr_line_2 like '%SASKATCHEWAN' or a.addr_line_2 like '% ONTARIO' or a.addr_line_2 like '% NEWFOUNDLAND' or - a.addr_line_2 like '% ALBERTA%' or + a.addr_line_2 like '% ALBERTA%' or a.addr_line_2 like '% NOVA SCOTIA' or a.addr_line_2 like '% YUKON') ) and province = colin_update_address_ca_province(addr_line_2, false); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; + row_occurrences := v_phase_rows; END; $$; -- -- Transpose office and corp_party address mailing and delivery null city from addr_line3, and addr_line2. --- call colin_address_transpose_city(); +-- call colin_address_transpose_city(NULL); -- -CREATE OR REPLACE PROCEDURE public.colin_address_transpose_city() +CREATE OR REPLACE PROCEDURE public.colin_address_transpose_city( + OUT row_occurrences bigint +) LANGUAGE plpgsql AS $$ +DECLARE + v_step_rows bigint := 0; + v_phase_rows bigint := 0; BEGIN -- Corp party delivery/mailing line 3 update address @@ -1203,6 +1311,8 @@ BEGIN and upper(a.addr_line_3) not like '%ALSO KNOWN %' and upper(a.addr_line_3) not like '%AKA:%' ); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Corp party delivery/mailing line 2 @@ -1226,6 +1336,8 @@ BEGIN and upper(a.addr_line_2) not like '%ALSO KNOWN %' and upper(a.addr_line_2) not like '%AKA:%' ); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Office delivery/mailing line 3 @@ -1249,6 +1361,8 @@ BEGIN and upper(a.addr_line_3) not like '%ALSO KNOWN %' and upper(a.addr_line_3) not like '%AKA:%' ); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Office delivery/mailing line 2 @@ -1272,8 +1386,11 @@ BEGIN and upper(a.addr_line_2) not like '%ALSO KNOWN %' and upper(a.addr_line_2) not like '%AKA:%' ); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; + row_occurrences := v_phase_rows; END; $$; @@ -1281,11 +1398,16 @@ $$; -- -- Final transpose step: cleanup. -- Remove trailing comma characters from city, addresses lines 2 and 3. --- call colin_address_transpose_cleanup(); +-- call colin_address_transpose_cleanup(NULL); -- -CREATE OR REPLACE PROCEDURE public.colin_address_transpose_cleanup() +CREATE OR REPLACE PROCEDURE public.colin_address_transpose_cleanup( + OUT row_occurrences bigint +) LANGUAGE plpgsql AS $$ +DECLARE + v_step_rows bigint := 0; + v_phase_rows bigint := 0; BEGIN -- Remove trailing comma corp party delivery/mailing city: 20928 update address @@ -1299,9 +1421,11 @@ BEGIN and cs.end_event_id is null and (cp.end_event_id is null or cp.end_event_id = 0) and (cp.mailing_addr_id = a.addr_id or cp.delivery_addr_id = a.addr_id) - and a.city is not null + and a.city is not null and right(a.city, 1) = ',' ); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Remove trailing comma corp party delivery/mailing line 3: 1109 @@ -1316,9 +1440,11 @@ BEGIN and cs.end_event_id is null and (cp.end_event_id is null or cp.end_event_id = 0) and (cp.mailing_addr_id = a.addr_id or cp.delivery_addr_id = a.addr_id) - and a.addr_line_3 is not null + and a.addr_line_3 is not null and right(a.addr_line_3, 1) = ',' ); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Remove trailing comma corp party delivery/mailing line 2: 14957 @@ -1333,9 +1459,11 @@ BEGIN and cs.end_event_id is null and (cp.end_event_id is null or cp.end_event_id = 0) and (cp.mailing_addr_id = a.addr_id or cp.delivery_addr_id = a.addr_id) - and a.addr_line_2 is not null + and a.addr_line_2 is not null and right(a.addr_line_2, 1) = ',' ); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Remove trailing comma office delivery/mailing city: 1665 @@ -1350,9 +1478,11 @@ BEGIN and cs.end_event_id is null and (o.end_event_id is null or o.end_event_id = 0) and (o.mailing_addr_id = a.addr_id or o.delivery_addr_id = a.addr_id) - and a.city is not null + and a.city is not null and right(a.city, 1) = ',' ); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Remove trailing comma office delivery/mailing line 3: 992 @@ -1367,9 +1497,11 @@ BEGIN and cs.end_event_id is null and (o.end_event_id is null or o.end_event_id = 0) and (o.mailing_addr_id = a.addr_id or o.delivery_addr_id = a.addr_id) - and a.addr_line_3 is not null + and a.addr_line_3 is not null and right(a.addr_line_3, 1) = ',' ); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; -- Remove trailing comma office delivery/mailing line 2: 87 @@ -1384,11 +1516,14 @@ BEGIN and cs.end_event_id is null and (o.end_event_id is null or o.end_event_id = 0) and (o.mailing_addr_id = a.addr_id or o.delivery_addr_id = a.addr_id) - and a.addr_line_2 is not null + and a.addr_line_2 is not null and right(a.addr_line_2, 1) = ',' ); + GET DIAGNOSTICS v_step_rows = ROW_COUNT; + v_phase_rows := v_phase_rows + v_step_rows; COMMIT; + row_occurrences := v_phase_rows; END; $$; @@ -1403,19 +1538,56 @@ $$; -- 6. Remove duplicate corp_party address mailing and delivery CA province/region codes from city, addr_line3, and addr_line2. -- 7. Transpose office and corp_party address mailing and delivery null city from addr_line3 and addr_line2. -- 8. Remove trailing comma characters from city, addr_line3, addr_line2. --- call colin_address_transpose(); +-- call colin_address_transpose(NULL); -- -CREATE OR REPLACE PROCEDURE public.colin_address_transpose() +CREATE OR REPLACE PROCEDURE public.colin_address_transpose( + OUT result json +) LANGUAGE plpgsql AS $$ +DECLARE + v_country_rows bigint; + v_duplicate_country_rows bigint; + v_pcode_rows bigint; + v_duplicate_pcode_rows bigint; + v_region_rows bigint; + v_duplicate_region_rows bigint; + v_city_rows bigint; + v_cleanup_rows bigint; + v_started_at timestamp with time zone := clock_timestamp(); BEGIN -call colin_address_transpose_country(); -call colin_address_duplicate_country(); -call colin_address_transpose_pcode(); -call colin_address_duplicate_pcode(); -call colin_address_transpose_region(); -call colin_address_duplicate_region(); -call colin_address_transpose_city(); -call colin_address_transpose_cleanup(); + CALL public.colin_address_transpose_country(v_country_rows); + CALL public.colin_address_duplicate_country(v_duplicate_country_rows); + CALL public.colin_address_transpose_pcode(v_pcode_rows); + CALL public.colin_address_duplicate_pcode(v_duplicate_pcode_rows); + CALL public.colin_address_transpose_region(v_region_rows); + CALL public.colin_address_duplicate_region(v_duplicate_region_rows); + CALL public.colin_address_transpose_city(v_city_rows); + CALL public.colin_address_transpose_cleanup(v_cleanup_rows); + + result := json_build_object( + 'country', v_country_rows, + 'country_duplicates', v_duplicate_country_rows, + 'postal_code', v_pcode_rows, + 'postal_code_duplicates', v_duplicate_pcode_rows, + 'region', v_region_rows, + 'region_duplicates', v_duplicate_region_rows, + 'city', v_city_rows, + 'cleanup', v_cleanup_rows, + 'total', v_country_rows + + v_duplicate_country_rows + + v_pcode_rows + + v_duplicate_pcode_rows + + v_region_rows + + v_duplicate_region_rows + + v_city_rows + + v_cleanup_rows, + 'elapsed_seconds', EXTRACT(EPOCH FROM (clock_timestamp() - v_started_at)) + ); END; $$; + +COMMENT ON PROCEDURE public.colin_address_transpose() IS +'Returns one JSON value with all eight phase row-update counts, their total, and elapsed wall-clock seconds.'; + +COMMIT; diff --git a/data-tool/scripts/colin_corps_extract_postgres_ddl b/data-tool/scripts/colin_corps_extract_postgres_ddl index 66ee2fff4d..4450cbfb5d 100644 --- a/data-tool/scripts/colin_corps_extract_postgres_ddl +++ b/data-tool/scripts/colin_corps_extract_postgres_ddl @@ -42,6 +42,10 @@ create sequence if not exists excluded_email_domain_patterns_id_seq; alter sequence excluded_email_domain_patterns_id_seq owner to postgres; +create sequence if not exists demigrated_filings_id_seq; + +alter sequence demigrated_filings_id_seq owner to postgres; + create table if not exists address ( addr_id numeric(10) @@ -755,6 +759,9 @@ create table if not exists mig_group alter table mig_group owner to postgres; +create index if not exists idx_mig_group_delta_nk + on mig_group (name, target_environment, source_db); + create table if not exists mig_batch ( @@ -776,6 +783,9 @@ create table if not exists mig_batch alter table mig_batch owner to postgres; +create index if not exists idx_mig_batch_delta_nk + on mig_batch (mig_group_id, name, target_environment); + create table if not exists mig_corp_batch ( @@ -791,6 +801,9 @@ create table if not exists mig_corp_batch alter table mig_corp_batch owner to postgres; +create index if not exists idx_mig_corp_batch_delta_nk + on mig_corp_batch (mig_batch_id, corp_num); + create table if not exists mig_corp_account ( @@ -807,6 +820,9 @@ create table if not exists mig_corp_account alter table mig_corp_account owner to postgres; +create index if not exists idx_mig_corp_account_delta_nk + on mig_corp_account (corp_num, target_environment, account_id, mig_batch_id); + create table if not exists corp_processing ( id integer default nextval('corp_processing_id_seq'::regclass) not null @@ -1140,6 +1156,9 @@ create table if not exists corps_with_third_party alter table corps_with_third_party owner to postgres; +create index if not exists idx_corps_with_third_party_delta_nk + on corps_with_third_party (corp_num, vendor); + create table if not exists bar_corps ( identifier varchar, @@ -1154,6 +1173,9 @@ create table if not exists bar_corps alter table bar_corps owner to postgres; +create index if not exists idx_bar_corps_delta_nk + on bar_corps (identifier); + -- helper so that `is_blank(text)` returns TRUE for '' (but NULL stays NULL) create or replace function is_blank(t text) @@ -1266,6 +1288,49 @@ CREATE TABLE IF NOT EXISTS exclude_corps ( alter table exclude_corps owner to postgres; + +create table if not exists demigrated_filings ( + id integer default nextval('demigrated_filings_id_seq'::regclass) not null + constraint pk_demigrated_filings primary key, + corp_num character varying(10) NOT NULL, + filing_id integer not null + constraint unq_demigrated_filings_filing_id unique, + filing_date timestamp with time zone, + filing_type character varying(30), + filing_json jsonb, + payment_id character varying(4096), + transaction_id bigint, + business_id integer, + submitter_id integer, + status character varying(20), + payment_completion_date timestamp with time zone, + paper_only boolean, + completion_date timestamp with time zone, + effective_date timestamp with time zone, + source character varying(15), + parent_filing_id integer, + payment_status_code character varying(50), + temp_reg character varying(10), + payment_account character varying(30), + tech_correction_json jsonb, + colin_only boolean, + deletion_locked boolean, + submitter_roles character varying(200), + meta_data jsonb, + filing_sub_type character varying(30), + approval_type character varying(15), + application_date timestamp with time zone, + notice_date timestamp with time zone, + resubmission_date timestamp with time zone, + hide_in_ledger boolean NOT NULL DEFAULT false, + withdrawn_filing_id integer, + withdrawal_pending boolean NOT NULL DEFAULT false, + lear_only boolean, + notes varchar(600) +); + +alter table demigrated_filings owner to postgres; + CREATE INDEX if not exists ix_conv_event_event_id ON conv_event (event_id); CREATE INDEX if not exists ix_conv_ledger_event_id ON conv_ledger (event_id); @@ -1441,3 +1506,171 @@ CREATE INDEX IF NOT EXISTS ix_mig_batch_target_env_dates CREATE INDEX IF NOT EXISTS ix_event_corpnum_eventid_cover ON event (corp_num, event_id) INCLUDE (event_type_cd, event_timerstamp); + +-- Associate standalone ID sequences with their owning columns. +-- This allows pg_get_serial_sequence() and delta ID reallocation to resolve them. +ALTER SEQUENCE public.affiliation_processing_id_seq + OWNED BY public.affiliation_processing.id; +ALTER SEQUENCE public.auth_processing_id_seq + OWNED BY public.auth_processing.id; +ALTER SEQUENCE public.auth_component_operation_id_seq + OWNED BY public.auth_component_operation.id; +ALTER SEQUENCE public.corp_processing_id_seq + OWNED BY public.corp_processing.id; +ALTER SEQUENCE public.colin_tracking_id_seq + OWNED BY public.colin_tracking.id; +ALTER SEQUENCE public.mig_group_id_seq + OWNED BY public.mig_group.id; +ALTER SEQUENCE public.mig_batch_id_seq + OWNED BY public.mig_batch.id; +ALTER SEQUENCE public.mig_corp_batch_id_seq + OWNED BY public.mig_corp_batch.id; +ALTER SEQUENCE public.mig_corp_account_id_seq + OWNED BY public.mig_corp_account.id; +ALTER SEQUENCE public.bad_emails_id_seq + OWNED BY public.bad_emails.id; +ALTER SEQUENCE public.excluded_email_domain_patterns_id_seq + OWNED BY public.excluded_email_domain_patterns.id; +ALTER SEQUENCE public.demigrated_filings_id_seq + OWNED BY public.demigrated_filings.id; + +-- Address-transpose routine module (psql-only include; see header of the +-- included file). DbSchemaCLI/SQLAlchemy must NOT be pointed at this DDL. +\ir colin_address_transpose_updates.sql + +-- Procedures for Dynamic Generation of FKs + +CREATE TABLE IF NOT EXISTS public.colin_dropped_fks +( + table_schema name NOT NULL, + table_name name NOT NULL, + conname name NOT NULL, + condef text NOT NULL, + dropped_at timestamptz NOT NULL DEFAULT current_timestamp, + CONSTRAINT pk_colin_dropped_fks + PRIMARY KEY (table_schema, table_name, conname) +); + +CREATE OR REPLACE PROCEDURE public.colin_fk_drop_all( + OUT dropped_count bigint +) +LANGUAGE plpgsql +SECURITY INVOKER +AS $$ +DECLARE + fk record; + constraint_definition text; +BEGIN + dropped_count := 0; + + FOR fk IN + SELECT + constraint_row.oid AS constraint_oid, + namespace.nspname AS table_schema, + relation.relname AS table_name, + constraint_row.conname + FROM pg_catalog.pg_constraint AS constraint_row + JOIN pg_catalog.pg_class AS relation + ON relation.oid = constraint_row.conrelid + JOIN pg_catalog.pg_namespace AS namespace + ON namespace.oid = relation.relnamespace + LEFT JOIN public.colin_dropped_fks AS dropped + ON dropped.table_schema = namespace.nspname + AND dropped.table_name = relation.relname + AND dropped.conname = constraint_row.conname + WHERE constraint_row.contype = 'f' + AND relation.relkind = 'r' + AND namespace.nspname = 'public' + AND dropped.conname IS NULL + ORDER BY namespace.nspname, relation.relname, constraint_row.conname + LOOP + -- Force pg_get_constraintdef() to schema-qualify every non-catalog + -- relation, independent of the caller's search_path. SET LOCAL is + -- intentionally repeated because each loop iteration commits. + PERFORM pg_catalog.set_config('search_path', 'pg_catalog', true); + constraint_definition := pg_catalog.pg_get_constraintdef(fk.constraint_oid); + + INSERT INTO public.colin_dropped_fks ( + table_schema, + table_name, + conname, + condef + ) + VALUES ( + fk.table_schema, + fk.table_name, + fk.conname, + constraint_definition + ); + + EXECUTE pg_catalog.format( + 'ALTER TABLE %I.%I DROP CONSTRAINT %I', + fk.table_schema, + fk.table_name, + fk.conname + ); + + dropped_count := dropped_count + 1; + COMMIT; + END LOOP; +END; +$$; + +CREATE OR REPLACE PROCEDURE public.colin_fk_restore_all( + OUT restored_count bigint +) +LANGUAGE plpgsql +SECURITY INVOKER +AS $$ +DECLARE + fk record; + constraint_definition text; +BEGIN + restored_count := 0; + + FOR fk IN + SELECT + table_schema, + table_name, + conname, + condef + FROM public.colin_dropped_fks + ORDER BY table_schema, table_name, conname + LOOP + -- New captures contain qualified relation names. Keep public on a + -- controlled path as backward-compatible recovery for helper rows + -- captured by an earlier module version with unqualified names. + PERFORM pg_catalog.set_config('search_path', 'pg_catalog, public', true); + + constraint_definition := pg_catalog.rtrim(fk.condef); + IF constraint_definition !~* '[[:space:]]+NOT[[:space:]]+VALID[[:space:]]*$' THEN + constraint_definition := constraint_definition || ' NOT VALID'; + END IF; + + EXECUTE pg_catalog.format( + 'ALTER TABLE %I.%I ADD CONSTRAINT %I %s', + fk.table_schema, + fk.table_name, + fk.conname, + constraint_definition + ); + + DELETE FROM public.colin_dropped_fks + WHERE table_schema = fk.table_schema + AND table_name = fk.table_name + AND conname = fk.conname; + + restored_count := restored_count + 1; + COMMIT; + END LOOP; +END; +$$; + +COMMENT ON TABLE public.colin_dropped_fks IS +'Foreign key definitions captured while a COLIN extract refresh is in progress.'; + +COMMENT ON PROCEDURE public.colin_fk_drop_all() IS +'Captures and drops every foreign key on ordinary tables in public, committing each drop for resumability.'; + +COMMENT ON PROCEDURE public.colin_fk_restore_all() IS +'Restores captured foreign keys as NOT VALID constraints, committing and removing each saved definition on success.'; diff --git a/data-tool/scripts/generate_cprd_subset_extract.py b/data-tool/scripts/generate_cprd_subset_extract.py index 479aba2f58..81f3dab835 100644 --- a/data-tool/scripts/generate_cprd_subset_extract.py +++ b/data-tool/scripts/generate_cprd_subset_extract.py @@ -111,6 +111,7 @@ class tmpl_TemplateSpec: class tmpl_TemplateBundle: pg_acquire_advisory_lock: tmpl_TemplateSpec pg_release_advisory_lock: tmpl_TemplateSpec + pg_call_address_transpose: tmpl_TemplateSpec pg_prepare_address_stage: tmpl_TemplateSpec pg_cleanup_address_stage: tmpl_TemplateSpec pg_cleanup_orphan_children: tmpl_TemplateSpec @@ -303,6 +304,10 @@ def tmpl_default_bundle(repo_root: Path) -> tmpl_TemplateBundle: name="subset_pg_release_advisory_lock", path=subset_dir / "subset_pg_release_advisory_lock.sql", ) + pg_call_address_transpose = tmpl_TemplateSpec( + name="subset_pg_call_address_transpose", + path=subset_dir / "subset_pg_call_address_transpose.sql", + ) pg_prepare_address_stage = tmpl_TemplateSpec( name="subset_pg_prepare_address_stage", path=subset_dir / "subset_pg_prepare_address_stage.sql", @@ -361,6 +366,7 @@ def tmpl_default_bundle(repo_root: Path) -> tmpl_TemplateBundle: return tmpl_TemplateBundle( pg_acquire_advisory_lock=pg_acquire_advisory_lock, pg_release_advisory_lock=pg_release_advisory_lock, + pg_call_address_transpose=pg_call_address_transpose, pg_prepare_address_stage=pg_prepare_address_stage, pg_cleanup_address_stage=pg_cleanup_address_stage, pg_cleanup_orphan_children=pg_cleanup_orphan_children, @@ -704,15 +710,18 @@ def gen_build_master_script_inline( lines.append("-- Cleanup shared address staging table") lines.append(f"execute {templates.pg_cleanup_address_stage.path.as_posix()}") lines.append("") - lines.append("-- Release subset-run advisory lock") - lines.append(f"execute {templates.pg_release_advisory_lock.path.as_posix()}") - lines.append("") if cfg.pg_fastload: lines.append("-- Reset Postgres fast-load session settings") lines.append(f"execute {templates.pg_fastload_end.path.as_posix()}") lines.append("") + lines.append("-- Run address transpose while the subset-run advisory lock is held") + lines.append(f"execute {templates.pg_call_address_transpose.path.as_posix()}") + lines.append("") + lines.append("-- Release subset-run advisory lock") + lines.append(f"execute {templates.pg_release_advisory_lock.path.as_posix()}") + return "\n".join(lines) @@ -839,15 +848,18 @@ def _vset_predicates(target_values: Sequence[str], oracle_values: Sequence[str]) lines.append("-- Cleanup shared address staging table") lines.append(f"execute {templates.pg_cleanup_address_stage.path.as_posix()}") lines.append("") - lines.append("-- Release subset-run advisory lock") - lines.append(f"execute {templates.pg_release_advisory_lock.path.as_posix()}") - lines.append("") if cfg.pg_fastload: lines.append("-- Reset Postgres fast-load session settings") lines.append(f"execute {templates.pg_fastload_end.path.as_posix()}") lines.append("") + lines.append("-- Run address transpose while the subset-run advisory lock is held") + lines.append(f"execute {templates.pg_call_address_transpose.path.as_posix()}") + lines.append("") + lines.append("-- Release subset-run advisory lock") + lines.append(f"execute {templates.pg_release_advisory_lock.path.as_posix()}") + return "\n".join(lines) @@ -1045,7 +1057,7 @@ def run(cfg: cfg_GenerationConfig) -> int: # Ensure the execute-only templates exist too (even though we don't render them). for spec in ( templates.pg_acquire_advisory_lock, - templates.pg_release_advisory_lock, + templates.pg_call_address_transpose, templates.pg_prepare_address_stage, templates.pg_cleanup_address_stage, templates.pg_cleanup_orphan_children, @@ -1173,6 +1185,7 @@ def run(cfg: cfg_GenerationConfig) -> int: else: print(" - cars* tables will NOT be refreshed (--no-cars was set).") print(f" - CP corp type inclusion: {'ENABLED' if cfg.include_cp else 'disabled'} (--include-cp)") + print(" - address transpose CALL returns JSON metrics before the advisory lock is explicitly released.") print(f" - Postgres fast-load session settings: {'ENABLED' if cfg.pg_fastload else 'disabled'} (--pg-fastload)") print(f" - Postgres trigger suppression: {cfg.pg_disable_method.value} (--pg-disable-method)") print(" - subset runs acquire a session-level advisory lock on the target DB to prevent overlap.") @@ -1227,6 +1240,7 @@ def run(cfg: cfg_GenerationConfig) -> int: if cfg.oracle_in_strategy == cfg_OracleInStrategy.AUTO: print(f" - auto threshold (--or-of-in-max-ids): {cfg.or_of_in_max_ids}") print(" - Prefer --render-mode inline for faster runs (inline generates static SQL per chunk).") + print(" - address transpose CALL returns JSON metrics before the advisory lock is explicitly released.") print(f" - Postgres fast-load session settings: {'ENABLED' if cfg.pg_fastload else 'disabled'} (--pg-fastload)") print(f" - Postgres trigger suppression: {cfg.pg_disable_method.value} (--pg-disable-method)") print(" - subset runs acquire a session-level advisory lock on the target DB to prevent overlap.") diff --git a/data-tool/scripts/restore/README_add_preserved_table.md b/data-tool/scripts/restore/README_add_preserved_table.md new file mode 100644 index 0000000000..73b49d2307 --- /dev/null +++ b/data-tool/scripts/restore/README_add_preserved_table.md @@ -0,0 +1,507 @@ +# Adding a preserved table (backup, full restore, and delta restore) + +[Overview](README_add_preserved_table_overview.md) · [Quick reference](README_add_preserved_table_quickref.md) · **Full guide** · [Delta runbook](README_delta_restore.md) + +This guide walks you through adding a new preserved table so it participates correctly in all three scripts: + +- `data-tool/backup_extract_tables.sh` — dumps preserved tables to a custom-format archive. +- `data-tool/restore_extract.sh` — restores preserved-table data into a rebuilt extract database. +- `data-tool/delta_restore_extract.sh` — previews/validates/applies a merge of a preserved-table dump into an existing extract database. + +Section 3 is the **golden path** for a simple table — if your table is standalone with a straightforward key, you can follow it top to bottom and stop. Section 5 covers every advanced pattern the codebase supports, each anchored to a live example table and the test that proves the behavior. + +--- + +## 0. Scope: which dump this guide is about + +This guide concerns the **preserved-table custom-format dump** created by `backup_extract_tables.sh`: + +- `pg_dump -Fc` with one `--table=` selector per entry in `preserved_tables.conf`, plus `--no-owner --no-acl`. +- Output: `keep_YYYY-MM-DD.dump` and a `.manifest.json` sidecar (sha256, table list, pg_dump version). When the sidecar is present, delta preview verifies its dump hash. Keep the files together even though preview currently permits a missing sidecar. + +It is **not** about `data-tool/dump_colin_extract_without_views.sh`, which produces a *whole-extract* tar archive excluding the derived view/materialized-view layer. That is a separate workflow; nothing in this guide applies to it. + +## 1. Mental model + +1. **`data-tool/scripts/colin_corps_extract_postgres_ddl` is the schema source of truth.** The backup archive technically contains schema objects (pg_dump without `--data-only` includes them), but **restore never uses them**: `restore_extract.sh` runs `pg_restore --section=data --data-only` into tables that must already exist from the current DDL. If your table isn't in the DDL, restore has nowhere to put the data. +2. **Delta restore copies your local DDL, not the dump's.** Stage shells are created as `LIKE public.
INCLUDING DEFAULTS`, and a schema-drift preflight compares dump columns against local columns — drift exits with code `2`, a missing local table fails stage-shell creation with a pointer at the DDL file. Current local DDL is a hard prerequisite. +3. **`preserved_tables.conf` grants membership only.** Backup and full restore read just the table name (column 1). Delta restore also reads `load_phase` and requires effective metadata in `delta_ctl.complete_table_config()`. A conf entry alone does *not* make delta restore work. +4. **Keep three delta registrations aligned.** Runtime classification uses `delta_ctl.complete_table_config()` (`scripts/restore/delta/10_functions.sql`). The mirrored seed entry and staging index in `delta_restore_extract.sh` are repository conventions that keep bootstrap metadata and performance support aligned with that effective config. Fixtures validate the registration. +5. **Use the supported sequence pattern.** For a surrogate ID, production DDL should declare an explicit sequence, a `nextval(...)` column default, and `ALTER SEQUENCE … OWNED BY`. The default lets full restore repair the sequence; ownership makes it discoverable through `pg_get_serial_sequence()` when delta restore must reallocate a colliding ID. + +## 2. Decision tree + +Answer these before touching any file; each answer routes you to a step or an advanced branch. + +``` +Q1. Surrogate integer id column? + ├─ yes → you need sequence + default + OWNED BY in DDL ......... §3 step 1 + └─ no → no configured surrogate ID ............................ §5.1 + +Q2. Is the row-matching "natural key" enforced by the database + (unique constraint or unique index on exactly that key)? + ├─ yes → nk_enforced = true + └─ no → nk_enforced = false; duplicates classify AMBIGUOUS_NK .. §5.3 + +Q3. What shape is the natural key? + ├─ plain columns .............. nk_cols + matching s./l. exprs .. §3 step 3 + ├─ normalized expressions ..... nk_*_exprs with nk_cols='{}' .... §5.2 + └─ includes a preserved-parent FK → asymmetric map_fk exprs ..... §5.4 + +Q4. Foreign keys? + ├─ to another preserved table → fk_map internal entry + phase ... §5.4 + ├─ to corporation/event/etc. → fk_map "external:…" entry ........ §5.5 + └─ none → fk_map = '{}' + +Q5. Has a last_modified column that should protect newer local edits? + └─ yes → has_last_modified = true ............................... §5.6 + +Q6. Append-only audit child (rows identified by content, not key)? + └─ yes → match_mode = 'hash_parent' ............................. §5.7 +``` + +## 3. Golden path: a simple table end to end + +Worked example: a hypothetical standalone table `demigrated_filings` with a surrogate `id`, a database-enforced unique `filing_id`, no FKs, and no `last_modified`. Substitute your table's names throughout. Eight steps; each names the exact file and location. + +### Step 1 — DDL in `data-tool/scripts/colin_corps_extract_postgres_ddl` + +Three placements in this one file. + +**(a) Sequence**, in the sequence block at the top of the file (alongside `bad_emails_id_seq` etc.): + +```sql +create sequence if not exists demigrated_filings_id_seq; + +alter sequence demigrated_filings_id_seq owner to postgres; +``` + +**(b) Table**, with PK, the enforced natural key, and ownership: + +```sql +create table if not exists demigrated_filings +( + id integer default nextval('demigrated_filings_id_seq'::regclass) not null + constraint pk_demigrated_filings primary key, + filing_id integer not null + constraint unq_demigrated_filings_filing_id unique, + notes varchar(600), + create_date timestamp with time zone default current_timestamp not null +); + +alter table demigrated_filings + owner to postgres; +``` + +Index rule for the natural key: the NK must have a supporting index on the **local** table. Here the unique constraint on `filing_id` already provides one, so nothing more is needed. If your NK is *not* covered by a unique constraint (unenforced or composite keys), add one following the existing convention: + +```sql +create index if not exists idx_demigrated_filings_delta_nk + on demigrated_filings (filing_id); +``` + +(Live examples: `idx_mig_group_delta_nk`, `idx_bar_corps_delta_nk`, and the normalized `ux_bad_emails_email_normalized`.) + +**(c) Sequence association**, in the terminal `ALTER SEQUENCE … OWNED BY` block at the end of the file: + +```sql +ALTER SEQUENCE public.demigrated_filings_id_seq + OWNED BY public.demigrated_filings.id; +``` + +This line makes `pg_get_serial_sequence('public.demigrated_filings','id')` resolve when delta restore must reallocate a colliding ID. The `nextval(...)` default is what generic full-restore sequence repair reads. Use this explicit sequence pattern in production DDL; identity columns are used only by the slim test fixtures. + +### Step 2 — Membership in `data-tool/scripts/restore/preserved_tables.conf` + +Add one line: + +```text +demigrated_filings 10 +``` + +Phase rules (delta apply/classification order is `ORDER BY load_phase, table_name`; backup and full restore ignore the number entirely): + +| Band | Current occupants | Use for | +|---|---|---| +| 10 | reference/exclusion tables (`bad_emails`, `bar_corps`, …) | standalone tables with no preserved-parent FKs | +| 20–40 | `mig_group` → `mig_batch` → `mig_corp_batch`/`mig_corp_account` | the migration hierarchy | +| 50 | `corp_processing`, `colin_tracking`, `auth_processing` | tracking tables referencing `mig_batch` | +| 60 | `auth_component_operation` | audit children of phase-50 tables | + +- A child must have a **strictly higher** phase than every preserved parent it references. +- Same-phase tables must not depend on each other. +- Use increments of 10 to leave room. +- Do **not** expect the phase to fix FK ordering in full restore — full restore uses one multi-table `TRUNCATE … RESTART IDENTITY` plus `pg_restore --disable-triggers`; phases play no role there. + +### Step 3 — Effective delta config in `delta_ctl.complete_table_config()` + +File: `data-tool/scripts/restore/delta/10_functions.sql`, inside `complete_table_config()`. Add an `UPDATE` block alongside the existing per-table blocks: + +```sql +UPDATE delta_ctl.table_config SET + pk_col = 'id', + nk_stage_exprs = ARRAY['s.filing_id'], + nk_local_exprs = ARRAY['l.filing_id'], + nk_cols = ARRAY['filing_id'], + nk_enforced = true +WHERE table_name = 'demigrated_filings'; +``` + +Two things to know about this function: + +- **It resets every table first.** The function begins by resetting all rows to defaults (`pk_col = NULL`, `nk_enforced = false`, empty exprs, `fk_map = '{}'`, …) and re-derives each table. If a dumped table has no block here, classification fails because its NK expressions are empty. Because of the reset you only need to set non-default fields. +- **Reuse the local constants** (`c_stage_corp_num`, `c_stage_flow_name`, etc.) when your expressions match them, following the surrounding code style. + +Field reference: + +| Field | Meaning | Set when | +|---|---|---| +| `pk_col` | Surrogate PK column name | table has one; `NULL` when no surrogate ID is configured (§5.1) | +| `nk_stage_exprs` / `nk_local_exprs` | Matching expressions over staged (`s.`) and local (`l.`) rows | always, in the same order; asymmetric for mapped parents (§5.4) | +| `nk_cols` | Plain NK column names | key is raw columns. Also excludes those columns from CHANGED updates via `update_columns()`. Use `'{}'` for expression keys (§5.2) | +| `nk_enforced` | Key uniqueness is database-enforced | **only** if a unique constraint/index exists on exactly that key (§5.3) | +| `fk_map` | `{"fk_col":"parent_table"}` or `{"fk_col":"external:table.column"}` | table has FKs (§5.4 / §5.5) | +| `has_last_modified` | Enables `CHANGED_LOCAL_NEWER` protection | column exists and should guard local edits (§5.6) | +| `match_mode` | `'nk'` (default) or `'hash_parent'` | audit children only (§5.7) | +| `compare_ignore_cols` | Columns excluded from content comparison | e.g. `['id']` for hash-parent tables | + +### Step 4 — Seed mirror in `delta_restore_extract.sh` + +File: `data-tool/delta_restore_extract.sh`, function `seed_table_config()`, inside the `seed_table_config_metadata.sql` heredoc (before its closing `SQL` marker). The heredoc uses grouped `IN (...)` lists — extend the matching lists: + +- Add `'demigrated_filings'` to the `SET pk_col = 'id' WHERE table_name IN (…)` list. +- Add `'demigrated_filings'` to the `SET nk_enforced = true WHERE table_name IN (…)` list. +- (Only if applicable) extend the `has_last_modified`, `match_mode`, and `fk_map` statements. + +Relationship between step 3 and step 4: **keep them in sync; the SQL function is authoritative.** `complete_table_config()` resets and rewrites the config at the start of every classification run, so the values that actually drive classification come from step 3. The shell seed is the current convention for having sane metadata present the moment the control schemas are installed. Consolidating the two is deliberately out of scope here. + +### Step 5 — Staging index in `create_stage_indexes()` + +File: `data-tool/delta_restore_extract.sh`, function `create_stage_indexes()`. Add one line in the block of `add_stage_index` calls: + +```bash +add_stage_index demigrated_filings idx_delta_stage_demigrated_filings_nk "filing_id" +``` + +The index must cover the staged NK columns and reproduce intrinsic normalization — e.g. `bad_emails` uses `"lower(btrim(email))"`. For a mapped parent term such as `map_fk(...)`, index the raw staged FK column, following `mig_batch`. Keep this performance-supporting index aligned with the effective config. + +### Step 6 — Roll the DDL out to existing extract databases + +New DBs get the table automatically from the DDL file. Existing DBs need the new statements applied (sequence, table, indexes, `OWNED BY`) before you run any of the three scripts against them: + +- Full restore into a DB missing the table aborts at the multi-table `TRUNCATE`; keep DDL current. +- Before full restore, confirm the dump contains a `TABLE DATA` entry for every current roster table. An older dump can omit a newly registered table, leaving it truncated without replacement data. +- Delta preview against a DB missing the table dies at stage-shell creation with `apply the latest colin_corps_extract_postgres_ddl first`. +- Delta preview against a DB with an older column set exits `2` with `drift_dump_only.tsv` / `drift_local_required.tsv` in the run directory. + +### Step 7 — Test fixtures + +Files under `data-tool/tests/delta_restore/fixtures/`: + +**(a) `minimal_schema.sql`** — add a `DROP TABLE IF EXISTS demigrated_filings CASCADE;` line to the drop block *and* a create block using the fixture convention (identity column instead of an explicit sequence): + +```sql +CREATE TABLE demigrated_filings ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + filing_id integer UNIQUE, + notes text +); +``` + +Keep only the columns delta config, classification, and apply reference — the fixture is intentionally slimmer than the real DDL. + +**(b) `t01_identical.sql`** — seed at least one row so the integration path exercises backup, staging, and `UNCHANGED` classification: + +```sql +INSERT INTO demigrated_filings(id, filing_id, notes) +VALUES (1, 90001, 'seed'); +``` + +**(c)** t13 (selector diagnostics) and t16 (selection manifest) enumerate tables in their fixtures — extend them **only** if your table introduces novel selector or manifest behavior (a new corp-bearing lookup path, a new no-surrogate-ID pattern, etc.). Ordinary tables don't need them. + +### Step 8 — Validate + +Run the runbook in [§7](#7-validation-runbook). Minimum bar: `make -C data-tool test-delta-restore` passes and the identical fixture reaches `UNCHANGED`. Add a divergent or source-only fixture when you need to prove NEW insertion, FK mapping, or ID reallocation. + +--- + +## 4. Class semantics and merge behavior (quick reference) + +Full operator semantics live in the [delta restore runbook](README_delta_restore.md); this is the one-line version plus the three invariants people ask about. + +| Class | Meaning | Applyable? | +|---|---|---| +| `NEW` | No safe local match | yes (default) | +| `CHANGED` | NK match, different payload; dump wins | yes (default) | +| `CHANGED_LOCAL_NEWER` | NK match but local `last_modified` is newer | only when explicitly selected | +| `UNCHANGED` | Payloads equivalent | no | +| `LOCAL_ONLY` | Local row with no staged match (count-only) | no | +| `BLOCKED_FK` | External or preserved-parent FK can't be resolved | no | +| `BLOCKED_PARENT` | Parent is blocked/ambiguous | no | +| `AMBIGUOUS_NK` | Duplicate NK on an unenforced-key table | no | +| `SKIPPED_ABSENT` | Configured table absent from the dump | no | + +Invariants: + +1. **ID collisions are safe.** If a selected NEW row's staged `id` collides with an unrelated local row, apply allocates a fresh local ID from the table's sequence (`disposition = NEW_REALLOCATED`); collision-free IDs are preserved (`NEW_PRESERVED`). This is the mechanism that requires `pg_get_serial_sequence()` to resolve (§3 step 1c). +2. **Children can't outrun parents.** A selected NEW child whose preserved parent is also NEW requires that parent row to stay selected — `validate_dependencies()` fails the run (exit `4`, `dependency_violations.tsv`) otherwise. +3. **Delta restore never deletes.** `LOCAL_ONLY` rows are counted for diagnostics and left untouched. + +--- + +## 5. Advanced branches + +Each subsection: when it applies, what to configure differently from §3, the live exemplar to copy from, and the test that proves the behavior. + +### 5.1 Tables without a configured surrogate ID + +*Exemplars: `excluded_emails`, `bar_corps`, `corps_with_third_party`, `email_domain_groups` (natural PK, no surrogate).* + +- DDL: no sequence, no `OWNED BY`, no `id` column. Skip §3 steps 1a/1c. +- Config: `pk_col` stays `NULL` (the reset default) — just don't set it. +- Consequences: no ID map is built; CHANGED updates target rows by classification-time `ctid`; row selectors in selection files use `row:` staging ordinals instead of `id:`. +- Fixture note: `email_domain_groups` and `excluded_emails` show the shapes in `minimal_schema.sql`. + +### 5.2 Expression keys (normalized matching) + +*Exemplar: `bad_emails` — key is `lower(btrim(email))`, enforced by `ux_bad_emails_email_normalized`.* + +- Config: put the normalization in the expressions and leave `nk_cols` empty: + + ```sql + nk_stage_exprs = ARRAY['lower(btrim(s.email))'], + nk_local_exprs = ARRAY['lower(btrim(l.email))'], + nk_cols = '{}', + nk_enforced = true + ``` + +- Why `nk_cols` still matters even when empty: it feeds `update_columns()`, which excludes NK columns from CHANGED updates. With `nk_cols = '{}'`, the underlying column (`email`) **is** updated on CHANGED — usually what you want for normalized keys, where the raw value can legitimately differ in case/whitespace. +- DDL: the enforced key must be a unique **expression index** matching the normalization exactly. +- Stage index (§3 step 5) must use the same expression: `"lower(btrim(email))"`. + +### 5.3 Unenforced keys and ambiguity + +*Exemplars: `bar_corps`, `corps_with_third_party`, the whole `mig_*` hierarchy. Proof: fixtures/asserts t09.* + +- Hard rule: **`nk_enforced = true` requires a database-enforced unique constraint or index on exactly the NK.** Nothing else counts. When `nk_enforced = true`, `detect_ambiguity()` skips the table entirely — a false claim silently allows duplicate-key misclassification. +- With `nk_enforced = false`, duplicated NK values (staged or local side) classify as `AMBIGUOUS_NK` and are never applyable. This is safe-by-default behavior, not an error to eliminate: resolve the duplicates locally or leave those rows unapplied. +- Composite unenforced keys (`corps_with_third_party`: `corp_num, vendor`) work identically — list all columns in the same order in all three `nk_*` arrays. + +### 5.4 Parent–child tables (preserved-parent FKs) + +*Exemplars: `mig_group` (20) → `mig_batch` (30) → `mig_corp_batch` / `mig_corp_account` (40). Proof: t08 (blocked parents), t14 (dependency validation), t12 (parent+children apply).* + +- Phase: child strictly above parent (§3 step 2). +- `fk_map`: internal entry per parent FK — `'{"mig_group_id":"mig_group"}'::jsonb`. +- **Asymmetric NK expressions** when the parent FK is part of the key. Staged parent IDs may be reallocated, so the stage side must translate through the ID map while the local side reads the raw column (copy `mig_batch`): + + ```sql + nk_stage_exprs = ARRAY['delta_ctl.map_fk(''mig_group'', s.mig_group_id::bigint)', 's.name', 's.target_environment'], + nk_local_exprs = ARRAY['l.mig_group_id', 'l.name', 'l.target_environment'], + nk_cols = ARRAY['mig_group_id', 'name', 'target_environment'] + ``` + +- Behavior you get for free: children of blocked/ambiguous parents become `BLOCKED_FK`/`BLOCKED_PARENT`; the generated `selection.conf` annotates the table with `parents=…`; deselecting a NEW parent while keeping its NEW child fails validation (exit `4`). +- Apply rewrites the FK value itself via `insert_expr()` → `map_fk`, so reallocated parent IDs propagate into inserted children automatically. + +### 5.5 External FKs (rows outside the preserved set) + +*Exemplars: `corp_processing` (`corporation`, `event`), `colin_tracking`/`auth_processing` (`corporation`). Code: `detect_external_fk_blocking()`.* + +- `fk_map` syntax: `"corp_num":"external:corporation.corp_num"`, `"failed_event_id":"external:event.event_id"`. +- Semantics: if the referenced local row is missing, the staged row classifies `BLOCKED_FK` with a `… not in local extract` reason. **External targets are never created** — the remediation is to load/refresh the missing `corporation`/`event` rows (e.g. via the subset workflow), then rerun preview. +- These FKs are checks only; no ID mapping or rewriting occurs for external targets. + +### 5.6 `has_last_modified` and locally-newer protection + +*Exemplars: `corp_processing`, `colin_tracking`, `auth_processing`.* + +- Set `has_last_modified = true` only when the column exists **and** newer local edits should win by default. +- Effect: an NK match where `l.last_modified > s.last_modified` classifies `CHANGED_LOCAL_NEWER` instead of `CHANGED`. It is excluded from the default selection, and the generated manifest deliberately never emits ready-to-uncomment selectors for it — overwriting newer local values must be an explicit operator decision (`include=new,changed,changed_local_newer`); add row selectors only to narrow that set. +- Full restore is unaffected (truncate + reload replaces everything regardless). + +### 5.7 Hash-parent audit children (append-only) + +*Exemplar: `auth_component_operation` (phase 60, child of `auth_processing`). T10 proves `LOCAL_ONLY` scoping to matched parents.* + +Use this mode for immutable audit/log rows identified by their content under a parent, not by a stable key. + +- Config: `match_mode = 'hash_parent'`, `compare_ignore_cols = ARRAY['id']`, `fk_map = '{"auth_processing_id":"auth_processing"}'::jsonb`. No NK arrays. +- Semantics: staged rows are matched by `md5` content hash (all columns except `id` and the parent FK, plus the **mapped** parent ID). Novel rows insert as NEW; matching rows are UNCHANGED; **there is no UPDATE path** — `apply_table()` returns after inserts for this mode, by design. +- `LOCAL_ONLY` is scoped to matched parents only, so unrelated local audit rows don't inflate counts. +- Caveat: the hash expression currently enumerates `public.auth_component_operation` columns specifically (`auth_component_hash_expr`); adding a *second* hash-parent table requires generalizing that function — treat it as an implementation task, not a config-only change, and `classify_table()` will raise `unsupported hash_parent table` until you do. + +--- + +## 6. Complex worked example + +A hypothetical `filing_reconciliation` table: child of `mig_batch`, external corp FK, enforced composite key, `last_modified`. Modeled on `corp_processing` — copy that table's blocks and adjust. Phase **50**. + +**DDL** (`colin_corps_extract_postgres_ddl`; sequence block, table, terminal `OWNED BY`): + +```sql +create sequence if not exists filing_reconciliation_id_seq; +alter sequence filing_reconciliation_id_seq owner to postgres; + +create table if not exists filing_reconciliation +( + id integer default nextval('filing_reconciliation_id_seq'::regclass) not null + constraint pk_filing_reconciliation primary key, + corp_num varchar(10) not null + constraint fk_filing_reconciliation_corporation + references corporation (corp_num), + flow_name varchar(100) not null, + environment varchar(25) not null, + mig_batch_id integer + constraint fk_filing_reconciliation_batch + references mig_batch, + processed_status varchar(25) not null, + last_error varchar(1000), + create_date timestamp with time zone default current_timestamp not null, + last_modified timestamp with time zone default current_timestamp not null, + constraint unq_filing_reconciliation + unique (corp_num, flow_name, environment) +); + +alter table filing_reconciliation owner to postgres; + +-- terminal block: +ALTER SEQUENCE public.filing_reconciliation_id_seq + OWNED BY public.filing_reconciliation.id; +``` + +The unique constraint covers the NK, so `nk_enforced = true` is licensed and no extra `idx_…_delta_nk` index is required. + +**Conf** (`preserved_tables.conf`) — strictly above `mig_batch` (30): + +```text +filing_reconciliation 50 +``` + +**`complete_table_config()`** (reuse the function's `c_stage_corp_num`-style constants as the surrounding code does): + +```sql +UPDATE delta_ctl.table_config SET + pk_col = 'id', + nk_stage_exprs = ARRAY['s.corp_num', 's.flow_name', 's.environment'], + nk_local_exprs = ARRAY['l.corp_num', 'l.flow_name', 'l.environment'], + nk_cols = ARRAY['corp_num', 'flow_name', 'environment'], + nk_enforced = true, + has_last_modified = true, + fk_map = '{"mig_batch_id":"mig_batch","corp_num":"external:corporation.corp_num"}'::jsonb +WHERE table_name = 'filing_reconciliation'; +``` + +Note the NK here does **not** include `mig_batch_id`, so the expressions stay symmetric. If your key *did* include the parent FK, use the asymmetric `map_fk` form from §5.4. + +**Seed mirror** (`seed_table_config()` heredoc): add to the `pk_col`, `nk_enforced`, and `has_last_modified` `IN` lists, and add an `fk_map` statement mirroring the above. + +**Stage index** (`create_stage_indexes()`): + +```bash +add_stage_index filing_reconciliation idx_delta_stage_filing_reconciliation_nk "corp_num, flow_name, environment" +``` + +**Fixtures**: identity-column create in `minimal_schema.sql` (drop block + create, keeping `corp_num`/`flow_name`/`environment`/`mig_batch_id`/`processed_status`/`last_modified`), and a seed row in `t01_identical.sql` referencing the existing seeded `corporation` and `mig_batch` rows. + +**What you get from this shape**: rows for corps missing locally → `BLOCKED_FK`; children of unselected NEW batches → dependency violations; locally edited rows with newer `last_modified` → `CHANGED_LOCAL_NEWER`, protected unless explicitly selected; colliding staged IDs → cleanly reallocated. + +--- + +## 7. Validation runbook + +Prerequisites: local PostgreSQL client tools (`PG_BIN=/path/to/bin` if not on `PATH`), a reachable server, and **throwaway databases only** — never validate against a shared extract DB. `` = your table. + +**7.1 Static + SQL smoke + integration harness** (from repo root; extends automatically to your table once fixtures are updated per §3 step 7): + +```bash +make -C data-tool test-delta-restore +data-tool/tests/delta_restore/run_tests.sh --integration +``` + +Success: `Summary: N passed, 0 failed`. PostgreSQL-backed checks skip (with a message) if tooling/server is unavailable — a skip is not a pass for your change; run them somewhere Postgres is available. + +**7.2 Clean-database DDL check** (proves the DDL file applies from scratch): + +```bash +createdb -h localhost -p 5432 -U postgres -T template0 preserved-ddl-check +psql -v ON_ERROR_STOP=1 -h localhost -p 5432 -U postgres -d preserved-ddl-check \ + -f data-tool/scripts/colin_corps_extract_postgres_ddl +``` + +Success: no errors. For a table with a sequence-backed surrogate ID, confirm sequence discoverability (must return the sequence name, not NULL); otherwise skip this check: + +```bash +psql -h localhost -p 5432 -U postgres -d preserved-ddl-check -qAt \ + -c "SELECT pg_get_serial_sequence('public.', 'id');" +``` + +**7.3 Backup + archive inspection** (preserved custom dump; source DB has current DDL and a few rows in ``): + +```bash +PGDATABASE= BACKUP_DIR=/tmp/preserved ./data-tool/backup_extract_tables.sh +pg_restore -l /tmp/preserved/keep_*.dump | grep -E "TABLE DATA public " +``` + +Success: exactly one `TABLE DATA` line for ``, and the manifest sidecar lists the table: + +```bash +grep '""' /tmp/preserved/keep_*.dump.manifest.json +``` + +**7.4 Full restore round-trip** (target = a throwaway rebuilt extract with current DDL and the base `corporation`/`event` rows required by preserved-table external FKs): + +```bash +DUMP=/tmp/preserved/keep_YYYY-MM-DD.dump PGDATABASE= ./data-tool/restore_extract.sh +``` + +Verify row counts match the source. For a table with a sequence-backed surrogate ID, also verify the sequence is ahead of the data; otherwise skip the sequence query: + +```bash +psql -d -qAt -c "SELECT count(*) FROM public.;" +psql -d -qAt -c \ + "SELECT nextval(pg_get_serial_sequence('public.','id')) > COALESCE(max(id),0) FROM public.;" +``` + +Success: counts equal; the second query returns `t`. (The `nextval` consumes one value — this is a throwaway DB.) + +**7.5 Delta golden path** (follow the [delta restore runbook](README_delta_restore.md) end to end against a throwaway local DB that has diverging data in ``): + +```bash +export PGDATABASE= +cd data-tool +./delta_restore_extract.sh --dump "$DUMP" --mode preview +# capture RUN from the printed "Artifacts:" line; review preview.txt and selection.conf +cp "$RUN/selection.conf" my_selection.conf +./delta_restore_extract.sh --dump "$DUMP" --mode validate --selection-file my_selection.conf +# paste the printed apply command (equivalent to): +./delta_restore_extract.sh --dump "$DUMP" --mode apply --selection-file my_selection.conf --yes +``` + +Success checkpoints: + +- Preview: `` appears in class counts; `drift_dump_only.tsv` and `drift_local_required.tsv` are empty; expected NEW/CHANGED rows are visible in `details/.*.tsv`. +- Validate: exit `0`, `Selection is valid`. +- Apply: aggregate `expected` and `affected` counts match for each table/class/action line; ID-map dispositions show `NEW_PRESERVED`/`NEW_REALLOCATED`/`MATCHED` as expected. + +Failure routing: exit `2` → drift/DDL (§3 step 6); exit `4` → `selection_diagnostics.tsv` / `dependency_violations.tsv`; exit `5` → inspect `apply_transaction.err`. For a colliding NEW ID, also verify `OWNED BY` per §3 step 1c. See the [full exit-code table](README_delta_restore.md#exit-codes). + +**7.6 FK and row-level spot checks** (after apply, for tables with FKs): + +```bash +# no orphaned children (example for a mig_batch child): +psql -d -qAt -c \ + "SELECT count(*) FROM public. c LEFT JOIN public.mig_batch p ON p.id = c.mig_batch_id + WHERE c.mig_batch_id IS NOT NULL AND p.id IS NULL;" +# selected counts vs actual delta: +cat "$RUN_APPLY/selected_counts.tsv" +``` + +Success: orphan count `0`; table deltas consistent with `selected_counts.tsv`. + +**7.7 Cleanup**: + +```bash +PGDATABASE= ./delta_restore_extract.sh --cleanup +dropdb -h localhost -U postgres preserved-ddl-check +``` diff --git a/data-tool/scripts/restore/README_add_preserved_table_overview.md b/data-tool/scripts/restore/README_add_preserved_table_overview.md new file mode 100644 index 0000000000..6d8bca4e39 --- /dev/null +++ b/data-tool/scripts/restore/README_add_preserved_table_overview.md @@ -0,0 +1,82 @@ +# Preserved tables — how it all works (orientation) + +**Overview** · [Quick reference](README_add_preserved_table_quickref.md) · [Full guide](README_add_preserved_table.md) · [Delta runbook](README_delta_restore.md) + +This is the read-first overview. When you're ready to make the edits, use the [quick reference](README_add_preserved_table_quickref.md); for depth, see the [full guide](README_add_preserved_table.md). + +## The picture + +```mermaid +flowchart TD + O[("Oracle (COLIN)")] -- "full extract refresh — wipes everything" --> X[("Extract DB")] + DDL["📐 Blueprint
colin_corps_extract_postgres_ddl
defines every table; applied to every DB"] -- "creates schema" --> X + X --> P["Your preserved side-tables
migration batches · trackers · auth audit · email lists
no home in Oracle — a refresh would destroy them"] + + ROSTER["📋 Roster
preserved_tables.conf
names the preserved tables (+ delta phase)"] --> BK + P --> BK["BACKUP
backup_extract_tables.sh"] + BK --> DUMP["💾 dump + manifest"] + + DUMP --> FR["FULL RESTORE
restore_extract.sh
truncate and reload all configured tables"] + DUMP --> DR["DELTA RESTORE
delta_restore_extract.sh
stage → classify → apply selected rows
never deletes local rows"] + + FR -- "rows only (data-only)" --> T[("rebuilt target DB")] + DR -- "selective merge" --> L[("existing local DB
with data you care about")] + DDL -. "schema must already exist here" .-> T + DDL -. "schema must already exist here" .-> L +``` + +## Why any of this exists + +The extract database is disposable by design — it gets rebuilt from Oracle whenever a fresh extract is needed. But over time we've bolted our own tables onto it: migration batches, processing trackers, auth audit rows, email exclusion lists. Those tables have no home in Oracle; a rebuild would destroy them. The preserved-table machinery is the answer: dump those tables to a file before a rebuild, and put the data back afterward. + +## Roster, schema, and delta metadata + +Two files define membership and schema; delta restore also has an authoritative metadata configuration. + +**The roster** (`preserved_tables.conf`) says *which* tables are preserved. One line per table. Backup and full restore read only the names. Delta restore also reads the number next to each name — a phase that orders tables so parents are handled before their children. + +**The blueprint** (the big `colin_corps_extract_postgres_ddl` file) says *what* each table looks like — columns, keys, sequences. Here's the part worth internalizing: the backup file technically contains table definitions, but **restore never uses them**. Restoring is data-only — rows are poured into tables that must *already exist*, built from the blueprint. So the blueprint isn't optional paperwork; it's where the table actually comes from in every database. If the blueprint hasn't been applied to a target, restore has nowhere to put the rows, and delta restore refuses to run. + +**The delta metadata authority** is `delta_ctl.complete_table_config()`. It defines how delta restore matches and relates rows; the seed mirror and stage indexes should stay aligned with it. + +## Full replacement and selective merge + +**Full restore** truncates every roster table in the target and reloads the dump. Use it when the target's existing preserved rows do not need to be retained. + +**Delta restore** is for a target that already has preserved data you care about — for example, a local extract with tracking rows absent from the dump. It merges in three moves: + +First it **stages**: the dump's rows are loaded into a scratch area shaped like your local tables, without touching real data. + +Then it **classifies** every staged row by comparing it to your local table. To compare, it needs to know how to *recognize* the same logical row in both places — that's the **natural key**, the stable identity of a row (a filing id, an email address, a corp+flow+environment combo). Given the key, every row lands in a bucket: brand new here; changed, dump wins; changed, but your local copy is *newer* (protected by default); identical; blocked because something it references is missing; or ambiguous because the key isn't actually unique. Rows that exist only locally are counted and left alone — delta restore **never deletes**. + +Finally it **applies** — but only what you select. You get a preview report and a selection file; by default only new and safely-changed rows are included, and you can narrow further. A few niceties happen automatically: if a dump row's ID would collide with an unrelated local row, a fresh ID is minted from the table's sequence; if a new child row depends on a new parent row, you can't apply one without the other. + +## So why does adding a table take six edits? + +Because the three scripts need three different levels of knowledge about your table, and the tests need a fourth. + +Backup and full restore only need the table to *exist* and be *listed* — that's the blueprint and the roster, edits one and two. After those two, backup and full restore already work. + +Delta restore needs effective metadata that identifies the natural key, whether the database enforces it, and which columns reference parent tables. That's edit three. Keep the bootstrap mirror (edit four) and performance-supporting stage index (edit five) aligned with it as repository conventions. + +Edit six adds the table to the test fixtures. The identical fixture exercises backup, staging, and `UNCHANGED` classification; add a divergent or source-only case when you need to prove insertion, FK mapping, or ID reallocation. + +The natural-key expressions and staging index must describe the same key. When `nk_enforced=true`, the blueprint must also contain a matching unique constraint or unique index. For supported unenforced keys, duplicates classify as ambiguous instead. + +## Orienting yourself for a new table + +Before touching anything, answer three questions in your head: + +*What is this row's identity?* — the natural key. Is it truly unique, and is that uniqueness enforced by the database? If not, delta will treat duplicates as ambiguous and refuse to merge them, which is the safe default, not a bug. + +*What does this row depend on?* — parents inside the preserved set (their phase must come first) or rows outside it, like corporations and events (delta checks those exist but will never create them). + +*Does it have a surrogate id?* — if yes, the blueprint needs the sequence properly declared and tied to the column, because that's what powers both sequence repair after restore and fresh-ID minting during delta merges. + +With those three answers, the six edits are mostly transcription — the quick reference gives the exact locations and paste-ready shapes, and every existing table in the config is a worked example. + +## Where to go next + +Doing it → [Quick reference](README_add_preserved_table_quickref.md) (six edits, run-to-completion, failure routing). +Understanding it deeply → [Full guide](README_add_preserved_table.md) (rationale, advanced patterns, validation runbook). +Operating delta day-to-day → [Delta runbook](README_delta_restore.md) (golden path, selection grammar, exit codes). diff --git a/data-tool/scripts/restore/README_add_preserved_table_quickref.md b/data-tool/scripts/restore/README_add_preserved_table_quickref.md new file mode 100644 index 0000000000..1de7242945 --- /dev/null +++ b/data-tool/scripts/restore/README_add_preserved_table_quickref.md @@ -0,0 +1,196 @@ +# Add a preserved table — quick reference + +[Overview](README_add_preserved_table_overview.md) · **Quick reference** · [Full guide](README_add_preserved_table.md) · [Delta runbook](README_delta_restore.md) + +One table in, six edits, three working scripts. Do the steps **top to bottom** — later steps assume earlier ones. + +``` + new table + │ + 1 DDL ──┬──► backup ✓ full restore ✓ + 2 conf line ──┘ + 3 delta config ──┐ + 4 seed mirror ├──► delta restore ✓ (needs 1–5) + 5 stage index ──┘ + 6 fixtures ─────► automated validation + │ + run: backup ─► restore ─► delta preview/validate/apply + │ + ✔ apply_summary: expected = affected +``` + +Two constants run through all six edits: +- the **table name** appears in every step (the final grep check counts on this); +- the **natural key** in the `nk_*` expressions and stage index must match. When `nk_enforced=true`, the DDL must also define the same key as unique. + +Example table throughout: `demigrated_filings` (surrogate `id`, unique `filing_id`). Substitute your names. + +--- + +### 1 ▸ DDL · `data-tool/scripts/colin_corps_extract_postgres_ddl` — three spots, one file + +```sql +-- ① TOP (sequence block) +create sequence if not exists demigrated_filings_id_seq; +alter sequence demigrated_filings_id_seq owner to postgres; + +-- ② BODY (table; the unique key here IS your delta "natural key" — steps 3 and 5 reuse it) +create table if not exists demigrated_filings ( + id integer default nextval('demigrated_filings_id_seq'::regclass) not null + constraint pk_demigrated_filings primary key, + filing_id integer not null + constraint unq_demigrated_filings_filing_id unique, + notes varchar(600) +); +alter table demigrated_filings owner to postgres; + +-- ③ BOTTOM (association block) — required to reallocate colliding IDs +ALTER SEQUENCE public.demigrated_filings_id_seq OWNED BY public.demigrated_filings.id; +``` + +⚑ Restores are **data-only** — apply this DDL to every DB you'll run the scripts against, or the table simply isn't there to receive rows. (No surrogate `id`? Skip ① and ③.) + +### 2 ▸ Conf · `data-tool/scripts/restore/preserved_tables.conf` — one line + +```text +demigrated_filings 10 +``` + +Phase = delta ordering only (backup/full restore ignore it): **10** standalone · **20–40** mig hierarchy · **50** tables referencing `mig_batch` · **60** audit children. A child's phase must be **higher** than every preserved parent it references. + +*Backup and full restore already work at this point — steps 3–6 are the delta half.* + +### 3 ▸ Delta config · `complete_table_config()` in `data-tool/scripts/restore/delta/10_functions.sql` + +**The template** — one `UPDATE` block next to the existing ones. Every line annotated; this is the simple case complete: + +```sql +UPDATE delta_ctl.table_config SET + pk_col = 'id', -- surrogate ID column (omit if none is configured) + nk_stage_exprs = ARRAY['s.filing_id'], -- match key, dump side ─┐ same key, same order, + nk_local_exprs = ARRAY['l.filing_id'], -- match key, local side ─┤ = step 1's constraint + nk_cols = ARRAY['filing_id'], -- plain NK column names ─┘ = step 5's index + nk_enforced = true -- true ONLY if a real unique constraint/index exists +WHERE table_name = 'demigrated_filings'; -- on exactly that key; else false → duplicates + -- classify AMBIGUOUS_NK (the safe outcome) +``` + +**Not the simple case?** Apply the diff for your situation on top of the template — `+` add a line, `~` change a line, `−` drop a line. Situations **stack** (e.g. `corp_processing` = parent FK + external FK + last_modified). The named table is the live block in the same function — copy-check yours against it. + +```diff +No configured surrogate ID check against: excluded_emails, bar_corps +- pk_col = 'id' (rows match/update by ctid; selectors use row:) + +Normalized key (case/space-insensitive) check against: bad_emails +~ nk_stage_exprs = ARRAY['lower(btrim(s.email))'] +~ nk_local_exprs = ARRAY['lower(btrim(l.email))'] +~ nk_cols = '{}' (steps 1 + 5 must use the SAME expression) + +Child of a preserved table check against: mig_batch ++ fk_map = '{"mig_group_id":"mig_group"}'::jsonb +~ nk_stage_exprs FK term → delta_ctl.map_fk('mig_group', s.mig_group_id::bigint) + (local side stays l.mig_group_id — asymmetry + is deliberate: staged parent IDs get remapped. + conf phase must be > the parent's) + +References corporation / event check against: corp_processing ++ fk_map entry → "corp_num":"external:corporation.corp_num" + (missing target rows → BLOCKED_FK; never created) + +Has a last_modified guard check against: corp_processing ++ has_last_modified = true (newer local rows → CHANGED_LOCAL_NEWER, + protected unless explicitly selected) + +Append-only audit child check against: auth_component_operation ++ match_mode = 'hash_parent' ⚠ NOT config-only — a 2nd hash_parent table ++ compare_ignore_cols = ARRAY['id'] needs code changes (hash fn is table-specific) +``` + +### 4 ▸ Seed mirror · `seed_table_config()` heredoc in `data-tool/delta_restore_extract.sh` + +Add `'demigrated_filings'` to the `pk_col = 'id' … IN (…)` and `nk_enforced = true … IN (…)` lists (plus `fk_map` / `has_last_modified` if step 3 used them). Keep in sync with step 3; step 3 wins at classification time. + +### 5 ▸ Stage index · `create_stage_indexes()` in `data-tool/delta_restore_extract.sh` + +```bash +add_stage_index demigrated_filings idx_delta_stage_demigrated_filings_nk "filing_id" +``` + +⚑ Cover the staged NK columns and intrinsic normalization (for example, `"lower(btrim(email))"`). For `map_fk(...)`, index the raw staged FK column, following `mig_batch`. + +### 6 ▸ Fixtures · `data-tool/tests/delta_restore/fixtures/` + +`minimal_schema.sql` — add to the DROP block, then (identity form, per fixture convention): + +```sql +CREATE TABLE demigrated_filings ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + filing_id integer UNIQUE, + notes text +); +``` + +`t01_identical.sql` — one seed row to exercise backup, staging, and `UNCHANGED` classification: + +```sql +INSERT INTO demigrated_filings(id, filing_id, notes) VALUES (1, 90001, 'seed'); +``` + +--- + +## Run to completion + +Throwaway DBs only. `PG_BIN=/path/to/bin` if Postgres tools aren't on `PATH`. + +```bash +# tests — expect "Summary: N passed, 0 failed" (skips ≠ passes) +make -C data-tool test-delta-restore +data-tool/tests/delta_restore/run_tests.sh --integration + +# BACKUP (source DB) +PGDATABASE= BACKUP_DIR=/tmp/preserved ./data-tool/backup_extract_tables.sh +DUMP=/tmp/preserved/keep_YYYY-MM-DD.dump # path printed by the backup +pg_restore -l "$DUMP" | grep "TABLE DATA public demigrated_filings " # no match → recheck step 2 +# Before full restore, confirm the dump has TABLE DATA for every current roster table. + +# FULL RESTORE (rebuilt target that already has current DDL) +PGDATABASE= DUMP="$DUMP" ./data-tool/restore_extract.sh +# ends "Done. Preserved tables restored; sequences synchronised." — table errors → recheck step 1 + +# DELTA (existing local DB with diverging rows) +export PGDATABASE=; cd data-tool +./delta_restore_extract.sh --dump "$DUMP" --mode preview # note printed "Artifacts: " +cp /selection.conf my.conf +./delta_restore_extract.sh --dump "$DUMP" --mode validate --selection-file my.conf +./delta_restore_extract.sh --dump "$DUMP" --mode apply --selection-file my.conf --yes +``` + +✔ **Done when** aggregate `expected` and `affected` counts match for each table/class/action line. The identical fixture should classify the seed as `UNCHANGED`; use a divergent or source-only fixture to prove NEW insertion, FK mapping, or ID reallocation. + +Dot-check — the table name must appear in all six edits; a missing file = the step you skipped: + +```bash +grep -rln demigrated_filings \ + data-tool/scripts/colin_corps_extract_postgres_ddl \ + data-tool/scripts/restore/preserved_tables.conf \ + data-tool/scripts/restore/delta/10_functions.sql \ + data-tool/delta_restore_extract.sh \ + data-tool/tests/delta_restore/fixtures/minimal_schema.sql \ + data-tool/tests/delta_restore/fixtures/t01_identical.sql +# expect all 6 files (delta_restore_extract.sh covers steps 4 AND 5 — name appears twice inside it) +``` + +## If it stops + +``` +exit 2 ............ dump/local columns disagree → step 1 not rolled out to this DB; fix, rerun preview +exit 4 ............ bad selection or NEW child without its NEW parent → selection_diagnostics.tsv / + dependency_violations.tsv in the printed run dir +exit 5 ............ inspect apply_transaction.err; for a colliding NEW ID, verify step 1③ + (OWNED BY) on the target +AMBIGUOUS_NK ...... genuine duplicates on an unenforced key → resolve locally or leave unapplied; + never "fix" by setting nk_enforced=true without a real constraint +classification .... empty NK expression error → step 3 effective config block is missing +``` + +Deep dives: [golden path and exit codes](README_delta_restore.md) · [rationale, advanced patterns, and full runbook](README_add_preserved_table.md). diff --git a/data-tool/scripts/restore/README_delta_restore.md b/data-tool/scripts/restore/README_delta_restore.md new file mode 100644 index 0000000000..10bd8261ec --- /dev/null +++ b/data-tool/scripts/restore/README_delta_restore.md @@ -0,0 +1,621 @@ +# Delta restore for preserved COLIN extract tables + +`data-tool/delta_restore_extract.sh` merges a preserved-table dump into an existing COLIN PostgreSQL extract database without truncating preserved tables. Use it when a developer/local extract already has migration, tracking, or auth side-table data that must be reconciled with a newer preserved-table dump. + +The preserved table set and phase order are centralized in `data-tool/scripts/restore/preserved_tables.conf`. The same file is consumed by: + +- `data-tool/backup_extract_tables.sh` — creates the preserved-table dump and sidecar manifest. +- `data-tool/restore_extract.sh` — full restore path. +- `data-tool/delta_restore_extract.sh` — delta preview, validate, and apply path. + +For orientation, see the [preserved-table overview](README_add_preserved_table_overview.md). To add a table, use the [quick reference](README_add_preserved_table_quickref.md) or [full guide](README_add_preserved_table.md); a config-only change is not sufficient for delta restore. + +## Quick reference + +### Contents + +- [Golden path: merge a delta dump end to end](#golden-path-merge-a-delta-dump-end-to-end) +- [Connection and environment](#connection-and-environment) +- [Run directory artifacts](#run-directory-artifacts) +- [Modes: preview, validate, apply](#modes-preview-validate-apply) +- [Class semantics](#class-semantics) +- [Selection manifest and cookbook](#selection-manifest-and-cookbook) + - [Row-selector grammar](#row-selector-grammar) + - [Binding and validation](#binding-and-validation) +- [CLI and environment reference](#cli-and-environment-reference) +- [Detail artifacts and viewing](#detail-artifacts-and-viewing) +- [Querying the staged run directly](#querying-the-staged-run-directly) +- [Troubleshooting by exit code](#troubleshooting-by-exit-code) +- [Recovery, cleanup, and concurrency](#recovery-cleanup-and-concurrency) +- [DELTA_MODE removal and restore_extract.sh delegation](#delta_mode-removal-and-restore_extractsh-delegation) +- [Exit codes](#exit-codes) +- [Tests](#tests) + +The five-command path, run from `data-tool/`, is: + +```bash +# 1. Back up the preserved tables on the source. +PGDATABASE=source_extract BACKUP_DIR=/tmp/preserved ./backup_extract_tables.sh + +# Export the target connection inherited by preview and its printed follow-up commands. +export PGDATABASE=local_extract + +# 2. Preview against the target; keep the final "Artifacts:" path. +DUMP=/tmp/preserved/keep_$(date +%F).dump; ./delta_restore_extract.sh --dump "$DUMP" --mode preview + +# 3. Set RUN to the printed path (example shown), then copy and edit the selection. +RUN=scripts/generated/delta_restore/20260716_183022; cp "$RUN/selection.conf" my_selection.conf; vi my_selection.conf + +# 4. Validate the copy until this exits 0. +./delta_restore_extract.sh --dump "$DUMP" --mode validate --selection-file my_selection.conf + +# 5. Paste the apply command printed by validate (shown here in full). +./delta_restore_extract.sh --dump "$DUMP" --mode apply --selection-file my_selection.conf --yes +``` + +> After a run directory is initialized, the invocation prints `Artifacts: ` on success or `Artifacts retained for inspection: ` on failure. Parse/help failures before initialization do not print an artifact path. Capture the preview path as `RUN` for review and selection editing; validate and apply each print their own diagnostic run directory. + +## Golden path: merge a delta dump end to end + +This walkthrough supplies every command and handoff needed for a normal merge. Follow links only when you need additional detail. + +### Step 0: Prepare the target and connection + +The target extract database must already exist with the current `colin_corps_extract_postgres_ddl` applied. Stop migration flows and avoid other non-cooperating writers against the target while applying. Export the target's [libpq settings](#connection-and-environment), at minimum `PGDATABASE`; set `PG_BIN` explicitly if the PostgreSQL client tools are not available through `PATH`. + +Export the target connection so the preview command and its printed validate/apply follow-ups inherit the same database, then run the remaining commands from `data-tool/`: + +```bash +export PGDATABASE=local_extract +cd data-tool +``` + +### Step 1: Create the dump on the source + +```bash +PGDATABASE=source_extract \ +BACKUP_DIR=/tmp/preserved \ +./backup_extract_tables.sh +``` + +By default this creates `keep_YYYY-MM-DD.dump` under `BACKUP_DIR` and prints: + +```text +✅ Wrote and .manifest.json +``` + +Keep the dump and sidecar together. Preview computes the dump sha256 and verifies it against the sidecar manifest. + +### Step 2: Preview against the target + +Use the exact dump path printed in Step 1: + +```bash +DUMP=/tmp/preserved/keep_YYYY-MM-DD.dump +./delta_restore_extract.sh \ + --dump "$DUMP" \ + --mode preview +``` + +A successful run ends with a concrete path such as: + +```text +Artifacts: scripts/generated/delta_restore/20260716_183022 +``` + +Capture it: + +```bash +RUN=scripts/generated/delta_restore/20260716_183022 +``` + +`RUN` is the preview run directory used in Steps 3–4. The console also prints a ready-to-paste validate command for Step 5. + +### Step 3: Review the preview in order + +1. Read `$RUN/preview.txt` for class counts, per-table samples, and the detail index with rendered/total/truncation state. +2. Read the blast-radius comment immediately after `[*]` in `$RUN/selection.conf` for the number of rows selected by default. +3. For surprising counts, open `$RUN/details/
..txt`; its canonical `.tsv` is beside it. +4. For large or targeted reviews, use the recipes in [Querying the staged run directly](#querying-the-staged-run-directly). + +### Step 4: Copy, then edit, the selection + +Do not edit the generated manifest in place: + +```bash +cp "$RUN/selection.conf" my_selection.conf +vi my_selection.conf +``` + +Copying keeps the generated manifest pristine for later diffing and keeps the run directory immutable. The copy retains the dump and staged-count binding headers required by validate/apply. The full grammar and 13 worked examples are in `$RUN/selection_cookbook.txt`. For `id:` selectors, use staged IDs from `selector_id`—never local-database IDs; see [Row-selector grammar](#row-selector-grammar). + +### Step 5: Validate; repeat until exit 0 + +```bash +./delta_restore_extract.sh \ + --dump "$DUMP" \ + --mode validate \ + --selection-file my_selection.conf +``` + +Validate normally takes seconds because it reuses the resident preview instead of restaging. After each attempt, use that invocation's printed artifact path: + +- Exit `4`: read `selection_diagnostics.tsv` and `dependency_violations.tsv` in the new validate run directory, edit `my_selection.conf`, and retry. +- Exit `2`: the resident preview is missing, incomplete, or stale; rerun Step 2 before retrying. + +On exit `0`, validate prints the exact apply command to paste, carrying forward the current selection and selector flags. + +### Step 6: Apply + +Paste validate's printed command. For the commands above it is equivalent to: + +```bash +./delta_restore_extract.sh \ + --dump "$DUMP" \ + --mode apply \ + --selection-file my_selection.conf \ + --yes +``` + +Apply deliberately re-verifies the dump hash, restages, reclassifies, binds, stamps, and checks dependencies against current local data. A validate result that has become stale therefore cannot bypass the apply safety checks. + +### Step 7: Confirm + +Capture the successful apply's final `Artifacts: ` path and read `/apply_summary.txt`. Confirm every row's `expected` count equals `affected`. The summary also records the touched tables; apply analyzes those tables and repairs their sequences. On success, the `delta_*` schemas are dropped unless `--keep-artifacts` was supplied. + +### Step 8: Clean up only when needed + +After diagnosing an abandoned preview or failed apply, drop the resident delta schemas: + +```bash +PGDATABASE=local_extract ./delta_restore_extract.sh --cleanup +``` + +Cleanup does not remove any run directory. + +## Connection and environment + +The script passes these libpq settings to PostgreSQL tools. Export the target settings before preview so the printed validate/apply commands inherit the same connection: + +```bash +PGHOST=localhost +PGPORT=5432 +PGUSER=postgres +PGDATABASE=colin-mig-corps-test +PGPASSWORD=... # optional; or use .pgpass +``` + +`PG_BIN` optionally names the directory containing PostgreSQL client tools. If it is unset or explicitly empty (`PG_BIN=""`), `psql`, `pg_restore`, and related clients are resolved through `PATH`. If it is nonempty, clients are resolved from that directory. + +The default run-directory base is `data-tool/scripts/generated/delta_restore/`. `DUMP` can supply the archive path instead of `--dump`, and `LOCK_TIMEOUT_SECONDS` controls the advisory-lock wait. See the consolidated [CLI and environment reference](#cli-and-environment-reference) for all defaults. + +## Run directory artifacts + +Artifacts are written under `data-tool/scripts/generated/delta_restore//` by default. The process sets `umask 077`, so new run directories and files default to modes 700 and 600. + +| Artifact | Purpose | +| --- | --- | +| `preview.txt` | Class samples plus an index of detail artifacts with rendered/total counts, truncation state, aligned paths, and viewing tips. | +| `selection.conf` | Concise active v2 manifest. Binding headers protect row selectors; class-only v1 files remain supported. | +| `selection_cookbook.txt` | Full selector grammar and 13 commented examples generated for the same run. | +| `details/
.new.tsv` | Canonical staged selector, row ordinal, and public-column values for NEW rows. | +| `details/
.changed.tsv` | Canonical local-to-dump changed-column records for CHANGED and CHANGED_LOCAL_NEWER rows. | +| `details/
.blocked.tsv` | Canonical staged values, block class, and reason. | +| `details/
..txt` | Derived fixed-width companion for double-click/editor review; never consumed by apply. | +| `classification.out/.err` | Classification stdout and timing notices. | +| `temp_stats.tsv` | Before/after PostgreSQL temp file and byte counters. | +| `selection_diagnostics.tsv` | Selector match counts and any validation problem. | +| `dependency_violations.tsv` | Selected-child/preserved-parent violations. | +| `selected_counts.tsv` | Counts selected by table and class. | +| `apply_transaction.err` | Apply transaction errors, including verification failures. | +| `apply_summary.txt` | Successful apply's expected/affected counts and touched-table summary. | + +Preview samples replace LF, tab, CR, and ESC with visible markers (`␤`, `␉`, `␍`, `␛`) and sanitize other terminal controls. Review `preview.txt` first, then the selection blast radius, then table/class details. Every exit prints the applicable run-directory path; failure paths remain available for diagnosis. + +## Modes: preview, validate, apply + +### Preview mode + +Preview is the default. It: + +1. Computes the dump sha256 and verifies it against optional `.manifest.json` metadata. +2. Filters the dump TOC to the preserved table list. +3. Creates `delta_stage`, `delta_map`, `delta_diff`, and `delta_ctl`. +4. Streams data into staging with guarded COPY-target rewriting. +5. Runs schema-drift preflight. +6. Classifies staged rows and builds ID maps. +7. Writes the operator and diagnostic artifacts. + +`--sample-size` controls only how many rows per class appear in the *Samples* section of `preview.txt` for each table. It does not cap detail files; `--details-limit` does that. + +### Validate mode + +Validate is the fast selection-editing loop. It requires the same target database and dump used by a preceding preview whose delta schemas are still resident. It: + +1. Acquires the normal advisory lock. +2. Confirms the resident control/diff objects are complete. +3. Hashes `--dump` and compares it with the resident `dump_sha256`. +4. Reconstructs staged-count bindings from `delta_ctl.run_counts`. +5. Reuses the normal bind → verify → stamp → dependency-check → selected-count workflow. +6. Writes diagnostics in a new run directory and prints a ready-to-paste apply command. + +Validate does **not** read the dump TOC, restage, reclassify, prompt, or modify public tables. Its transient selection/stamp state is replaced by the next preview or apply. + +A valid selection exits `0`. A missing or incomplete resident run, unreadable dump, or resident/dump sha mismatch is stale/preflight state and exits `2`; rerun preview. Lock contention exits `3`. Binding, selector, or dependency errors exit `4` and retain `selection_diagnostics.tsv` or `dependency_violations.tsv`. + +Validate is advisory rather than the correctness gate. Local public tables can change after preview without changing the dump sha, so validate can become optimistic. Apply intentionally repeats dump verification, staging, classification, binding, stamping, and dependency checks against current local data. + +### Apply mode + +Apply re-runs staging and classification fresh, stamps the requested selection, validates dependencies, and applies selected rows in one `REPEATABLE READ` transaction. It repairs sequences, analyzes touched tables, writes `apply_summary.txt`, and drops the delta schemas unless `--keep-artifacts` is supplied. + +Apply requires `--yes` or an interactive `yes` confirmation. Expected-versus-affected mismatches exit `5` and roll back the transaction. + +### Schema and run-directory lifecycle + +| After… | `delta_*` schemas | Run directory | +| --- | --- | --- | +| Preview success | Resident | Retained | +| Preview failure | Absent, incomplete, or resident depending on the failure stage | Retained | +| Validate success or exit `4` | Existing resident run remains; selection state is transient | Retained | +| Validate exit `2` | Existing state is unchanged and may be absent, incomplete, stale, or for a different dump | Retained | +| Apply success | Dropped unless `--keep-artifacts` | Retained | +| Apply failure (exit `4` or `5`) | Resident; public DML is rolled back | Retained | +| `--cleanup` | Dropped | Untouched | + +## Class semantics + +| Class | Meaning | Applyable? | +| --- | --- | --- | +| `NEW` | No safe local match. A staged ID may be preserved or reallocated. | Yes, by default | +| `CHANGED` | Natural-key/hash match with a different payload; dump wins. | Yes, by default | +| `CHANGED_LOCAL_NEWER` | Local `last_modified` is newer. | Only when explicitly included | +| `UNCHANGED` | Staged and local compared values are equivalent. | No | +| `LOCAL_ONLY` | Local row has no staged match; count-only diagnostic. | No | +| `BLOCKED_FK` | Required external or preserved-parent FK cannot be resolved safely. | No | +| `BLOCKED_PARENT` | Child depends on a blocked or ambiguous preserved parent. | No | +| `AMBIGUOUS_NK` | Natural key is duplicated for an unenforced-NK table. | No | +| `SKIPPED_ABSENT` | Configured table is absent from the dump. | No | + +Natural-key matching is hash-accelerated while retaining the null-safe residual check. `auth_component_operation` is append-only: novel hash-parent rows can be inserted, but changed updates are not applied. + +Classification emits grep-friendly timing notices to the console and `classification.err`: + +```text +NOTICE: delta_restore table=auth_processing phase=classify_nk_table start +NOTICE: delta_restore table=auth_processing phase=classify_nk_table done elapsed_ms=1234.567 +``` + +## Selection manifest and cookbook + +Preview writes a concise `selection.conf`. The generated shape is: + +```text +# delta-selection v2 +# dump_sha256= +# staged bad_emails=3 mig_group=128 ... +# ... +# ACTIVE SELECTION +[*] include=new,changed +# Default selection ([*] new,changed) currently matches 97342 rows across 9 tables (new=96105 changed=1237). + +[mig_batch] include=new,changed # new=3 changed=2 changed_local_newer=0 parents=mig_group +# ... + +# Small NEW sets receive ready-to-uncomment suggestions: + +# [bad_emails] Exact NEW set: rows=14 singletons=4 ranges=3 +# [bad_emails] new.rows include=id:101-104,200-203 +# Range counts: 101-104 (4 rows), 200-203 (4 rows) +# [bad_emails] new.rows include=id:300-301 +# Range counts: 300-301 (2 rows) + +# [bad_emails] new.rows include=id:401,500,700 +# [bad_emails] new.rows include=id:900 + +# Small CHANGED sets receive ready-to-uncomment suggestions: + +# [mig_batch] Exact CHANGED set: rows=3 singletons=1 ranges=1 +# [mig_batch] changed.rows include=row:150-151 +# Range counts: 150-151 (2 rows) + +# [mig_batch] changed.rows include=row:160 + +# Large CHANGED sets receive a useful pointer rather than no guidance: +# corp_processing has 4812 CHANGED rows; choose selector_id or corp values from: +# details/corp_processing.changed.tsv +# Example syntax: +# [corp_processing] changed.rows include=id:100,200-210 +# [corp_processing] changed.rows include=corp:BC0000001,BC0000002 + +# Full selector grammar and 13 worked examples: selection_cookbook.txt (this run dir) +``` + +Operator cues: + +- The blast-radius comment immediately after `[*]` totals the default-selected NEW and CHANGED rows before any edit. +- A per-table `parents=` suffix lists preserved parents. A selected NEW child whose preserved parent is also NEW requires that parent row to remain selected. +- By default, sets of 1–50 NEW or CHANGED rows receive exact commented `id:` or `row:` suggestions; the limit is inclusive and applies independently to each table/class set. The summary reports total rows, isolated singleton values, and consecutive ranges (a range counts as one run, not as its number of rows). +- Actual blank lines separate table blocks and, for mixed sets, separate the range selectors from singleton selectors. A range selector always remains directly adjacent to its `Range counts` comment. +- Long categories wrap at a soft maximum of 120 characters. Every wrapped line repeats the complete `[table] class.rows include=kind:` prefix, no token is split, and an indivisible prefix-plus-token may exceed the soft maximum. Singleton lists producing more than eight selector lines receive another actual blank line after every four lines for visual grouping. There is no continuation syntax. +- Repeated complete `include=` lines are unioned by the existing selector semantics. Uncomment every suggested line to select the entire summarized exact set; uncommenting only some lines deliberately selects only that subset. +- Sets above the resolved limit receive a pointer to the corresponding canonical TSV and table-aware examples. Set `--selector-suggestion-limit 0` to make every positive NEW/CHANGED set use this generic guidance. +- `CHANGED_LOCAL_NEWER` never receives ready-to-uncomment guidance because overwriting newer local values must stay deliberate. +- `selection_cookbook.txt` holds the complete, commented grammar/examples; it is generated with the manifest so the two cannot drift. + +The three preview-output controls are independent: `--sample-size` affects only `preview.txt` samples, `--details-limit` caps detail TSV/TXT artifacts, and `--selector-suggestion-limit` controls only commented manifest assistance. A generic pointer can therefore refer to a detail file truncated by a lower `--details-limit`; the full staged data remains queryable. Changing the selector-suggestion limit never changes active selection or apply behavior. + +To generate a conservative manifest that selects nothing until explicitly edited: + +```bash +./delta_restore_extract.sh \ + --dump keep.dump \ + --mode preview \ + --manifest-default none +``` + +This emits empty wildcard and per-table `include=` rules plus a zero-row blast-radius total. The default remains `new,changed` for compatibility. + +### Row-selector grammar + +Class-only v1 files remain valid and do not require binding headers. Optional row lines use: + +```text +[table] .rows =: +``` + +- Classes are `new`, `changed`, or `changed_local_newer`; the class must also be enabled by a class line. +- `id:` selects non-negative staged primary-key integers and inclusive ranges. +- `row:` selects staging ordinals for PK-less tables with the same bigint grammar. +- `corp:` selects exact identifiers on corp-bearing tables. +- Multiple include lines union their matches; excludes subtract afterward. +- With excludes only, the enabled class remains selected except for matching rows. +- `--only-corps` is a final intersection and never widens selection. + +> **`id:` values are staged (dump) primary keys** — the `selector_id` column in `details/
.new.tsv` / `.changed.tsv` and the `id …` shown in preview samples. They are never local-database IDs. When a staged ID collides with an unrelated local row, apply allocates a fresh local ID; the selector still refers to the staged value you reviewed. + +Example: + +```text +[*] include=new,changed +[mig_batch] include=new,changed +[mig_corp_account] include=new,changed +[mig_corp_account] new.rows exclude=id:23 +[corp_processing] include=new,changed +[corp_processing] changed.rows include=corp:BC0000001,BC0000002 +``` + +### Binding and validation + +The dump sha256 is computed for every run. Any file containing a `.rows` line must retain the generated `# dump_sha256=` header. A `row:` selector also requires the generated staged-count binding for that table. Duplicate hash headers, duplicate table counts, non-decimal counts, and binding mismatches exit `4`. + +Every selector must match at least one classified row after the `--only-corps` intersection. Unsupported selector kinds, selectors on disabled classes, duplicate staged primary keys, zero matches, and scalar multi-matches exit `4` with line-numbered entries in `selection_diagnostics.tsv`. Parent dependency failures are written to `dependency_violations.tsv`, including up to ten child selector IDs. + +Use validate repeatedly while editing: + +```bash +./delta_restore_extract.sh --dump keep.dump --mode validate \ + --selection-file my_selection.conf +``` + +A selection file supplies the authored candidate set. Explicit CLI filters are then enforced as a hard ceiling: + +1. `--tables a,b,c` removes every candidate from other tables. +2. A supplied `--include-classes` removes candidates outside those classes. +3. `--exclude-classes` removes its classes after the include ceiling. +4. Row selectors and `--only-corps` can narrow the remaining candidates but never widen them. + +With no selection file, the default candidate classes remain `new,changed` unless `--include-classes` changes them. With a selection file and no explicit table/class filters, the file retains its previous full authoring behavior. A generated `selection.conf` remains a full preserved-set review artifact, so after a filtered preview use the printed validate command: it carries the same filters, and validate/apply cannot re-enable file entries outside that scope. Staging and classification still use the full preserved set so parent maps exist. + +## CLI and environment reference + +### Flags + +| Flag | Argument | Default | Effect and details | +| --- | --- | --- | --- | +| `--dump` | `` | `DUMP`; otherwise required | Uses the named `pg_dump -Fc` archive for preview, validate, or apply. | +| `--mode` | `preview\|validate\|apply` | `preview` | Chooses staging/reporting, fast resident selection validation, or transactional apply; see [Modes](#modes-preview-validate-apply). | +| `--tables` | `all\|` | `all` | Hard selection ceiling, including with `--selection-file`; staging and classification still use every preserved table so parent maps exist. | +| `--include-classes` | `` | None | When supplied, limits candidates to these classes even with `--selection-file`; otherwise a file chooses its own classes. | +| `--exclude-classes` | `` | None | Removes classes after selection-file/include processing and cannot widen selection. | +| `--selection-file` | `` | None | Supplies generated/edited candidates for validate or apply; explicit table/class filters remain a hard ceiling, and row selections retain their bindings. | +| `--only-corps` | `` | None | Intersects selected corp-bearing rows with identifiers from the file and never widens selection; parent lookup tables are not restricted. | +| `--sample-size` | `` | `20` | Sets rows per class in each table's `preview.txt` *Samples* section. It affects `preview.txt` only; the detail TSV/TXT row cap is `--details-limit`. | +| `--details-limit` | `` | `10000` | Caps rows per table/class detail artifact. Full totals remain in `preview.txt` and `TRUNCATED` markers identify capped output. | +| `--selector-suggestion-limit` | `` | `50` | Controls exact commented selector ranges independently for each NEW/CHANGED table-class set. Accepts decimal integers `0..100000` inclusive; `0` makes every positive set use generic TSV guidance. This is CLI-only and does not alter active selection. | +| `--no-aligned-details` | — | Aligned `.txt` enabled | Suppresses derived fixed-width `.txt` companions; canonical TSVs are still written. | +| `--align-width` | `` | `40`; floor `6` | Sets the maximum aligned-detail cell width; values 1–5 are raised to 6. | +| `--manifest-default` | `new,changed\|none` | `new,changed` | Chooses the generated manifest's active wildcard selection; see the [conservative manifest example](#selection-manifest-and-cookbook). | +| `--report-dir` | `` | `data-tool/scripts/generated/delta_restore` | Changes the base directory under which the UTC timestamped run directory is created. | +| `--keep-artifacts` | — | Off | Retains `delta_*` schemas after a successful apply; run-directory files are retained regardless. | +| `--yes` | — | Off | Skips the interactive `yes` confirmation for apply. | +| `--cleanup` | — | Off | Drops `delta_stage`, `delta_map`, `delta_diff`, and `delta_ctl`, then exits; run directories are untouched. | +| `-h`, `--help` | — | — | Prints usage, flags, exit codes, environment settings, and artifact handoff lines. | + +### Environment variables + +| Variable | Default | Effect | +| --- | --- | --- | +| `PGHOST` | `localhost` | PostgreSQL server host used by libpq clients. | +| `PGPORT` | `5432` | PostgreSQL server port. | +| `PGUSER` | `postgres` | PostgreSQL user. | +| `PGDATABASE` | `colin-mig-corps-test` | Target database. | +| `PGPASSWORD` / `.pgpass` | None | Optional libpq password source. Prefer `.pgpass` where appropriate. | +| `PG_BIN` | Unset or empty (`PATH`) | Optional directory containing PostgreSQL client tools. A nonempty value resolves clients from that directory. | +| `DUMP` | Empty | Alternative to `--dump`; the CLI flag replaces it. | +| `LOCK_TIMEOUT_SECONDS` | `30` | Seconds to wait for the shared subset/full-refresh advisory lock before exit `3`. | + +## Detail artifacts and viewing + +Only interesting classes generate detail files; `UNCHANGED` and `LOCAL_ONLY` do not. The TSV is canonical and remains grep/awk/spreadsheet-importable. The `.txt` file is derived from it, is never parsed by validate/apply, and can be regenerated; if the two ever disagree, trust the TSV. + +The aligned companion preserves the TSV's visible COPY escapes rather than unescaping data. It uses a per-cell width of 40 by default, a minimum of 6, a two-space gutter, and a clipped `…` marker. Change the cap with `--align-width ` or disable companions with `--no-aligned-details`. + +Each aligned file ends with a footer such as: + +```text +# rendered 812 of 812 CHANGED rows +# rendered 10000 of 97105 NEW rows — TRUNCATED +``` + +Canonical truncated TSVs end with `# TRUNCATED at rows`; `preview.txt` repeats rendered/total values and `(TRUNCATED)`. Values use COPY-style tab/newline/backslash escaping and are capped at 8 KiB. Rows are ordered by staged primary key and then staging ordinal. For PK-less CHANGED rows, local display uses the classification-time `ctid`. + +Viewing options: + +- Double-click/open `details/
..txt` in a text editor and turn off word wrap. +- Use `column -s $'\t' -t < details/
..tsv | less -S`. +- Import the canonical TSV into a spreadsheet. +- Query the resident staged run directly for large or targeted reviews. + +## Querying the staged run directly + +Preview leaves the delta schemas installed. Run these recipes against the same target database before cleanup or a successful apply without `--keep-artifacts`. + +Filter NEW `corp_processing` rows by corporation prefix: + +```sql +SELECT s.* +FROM delta_diff.corp_processing_class AS d +JOIN delta_stage.corp_processing AS s USING (_delta_row_id) +WHERE d.class = 'NEW' + AND s.corp_num LIKE 'BC08%' +ORDER BY d.staged_pk NULLS LAST, d._delta_row_id; +``` + +List changed columns for one staged row: + +```sql +SELECT d.staged_pk, d._delta_row_id, changed.column_name +FROM delta_diff.corp_processing_class AS d +CROSS JOIN LATERAL unnest(d.changed_cols) AS changed(column_name) +WHERE d.class IN ('CHANGED', 'CHANGED_LOCAL_NEWER') + AND d.staged_pk = 881 +ORDER BY changed.column_name; +``` + +Count blocking reasons for one table: + +```sql +SELECT d.class, d.block_reason, count(*) AS rows +FROM delta_diff.corp_processing_class AS d +WHERE d.class IN ('BLOCKED_FK', 'BLOCKED_PARENT', 'AMBIGUOUS_NK') +GROUP BY d.class, d.block_reason +ORDER BY d.class, rows DESC; +``` + +For cross-table totals, query `delta_ctl.run_counts`: + +```sql +SELECT table_name, count_name, row_count +FROM delta_ctl.run_counts +WHERE count_name IN ('NEW', 'CHANGED', 'CHANGED_LOCAL_NEWER', + 'BLOCKED_FK', 'BLOCKED_PARENT', 'AMBIGUOUS_NK') +ORDER BY table_name, count_name; +``` + +## Troubleshooting by exit code + +Use the final `Artifacts:` or `Artifacts retained for inspection:` line to find the run directory named below. + +### Exit 0: successful run + +**Symptom:** The command completed successfully. + +**Read:** For preview, read `preview.txt` and copy `selection.conf`. For validate, use the printed ready-to-paste apply command. For apply, read `apply_summary.txt` and confirm `expected` equals `affected`. + +**Next step:** Continue at the corresponding next step in the [Golden path](#golden-path-merge-a-delta-dump-end-to-end). No remediation is required. + +### Exit 2: preflight, drift, dump, or stale-preview failure + +**Symptom:** Preflight rejected schema drift, an unreadable/corrupt dump, a manifest sha mismatch, or validate found no complete resident preview matching the dump. + +**Read:** Start with console stderr and the retained run directory. Check `dump.sha256` and any copied `dump.manifest.json`, `toc.list`/TOC errors, `drift_dump_only.tsv`, and `drift_local_required.tsv` as applicable. + +**Remediation:** Fix the connection or input, apply the current DDL when the dump contains newer columns, regenerate the dump when it lacks required columns, or restore the matching dump/manifest pair. Then rerun preview; validate cannot repair stale or missing resident state. + +### Exit 3: advisory lock busy or unavailable + +**Symptom:** The shared subset/full-refresh advisory lock was not acquired within `LOCK_TIMEOUT_SECONDS`. + +**Read:** Read console stderr and `advisory_lock.err` in the retained run directory to distinguish contention from a connection/lock error. + +**Remediation:** Let the cooperating subset/full-refresh/delta operation finish, or investigate an unexpected holder before retrying. Increase `LOCK_TIMEOUT_SECONDS` only when a longer wait is appropriate; its default is 30 seconds. + +### Exit 4: invalid selection, binding, or dependency + +If validate or apply exits `4`, inspect the run artifacts and resident diagnostics: + +```sql +SELECT * FROM delta_ctl.selection_diagnostics WHERE problem IS NOT NULL; +SELECT * FROM delta_ctl.dependency_violations; +SELECT table_name, count_name, row_count +FROM delta_ctl.run_counts +WHERE count_name IN ('BLOCKED_FK', 'BLOCKED_PARENT', 'AMBIGUOUS_NK') +ORDER BY table_name, count_name; +``` + +Common remediations: + +- Include the required preserved parent, or exclude the selected child. +- Load/refresh missing external `corporation` or `event` rows. +- Resolve duplicated natural keys locally, or leave ambiguous rows unapplied. +- Regenerate the dump if a configured table was unintentionally absent. +- Rerun preview if validate reports absent, incomplete, or stale resident state. + +The file equivalents are `selection_diagnostics.tsv`, `dependency_violations.tsv`, and `selected_counts.tsv` in the retained run directory. Edit the copied selection—not the generated `selection.conf`—then validate again. + +### Exit 5: apply verification failed + +**Symptom:** `APPLY_VERIFICATION_FAILED` means an expected count differed from its affected count, or a selected NEW row had no allocated local ID. The transaction rolled back, so **no public data changed**; the `delta_*` schemas and run directory are retained. + +**Read:** Start with `apply_transaction.err`, which durably records the table/class and expected/affected counts. The failed transaction also rolls back its writes to `delta_ctl.apply_counts`, so that table is not a post-failure count diagnostic. For a missing-ID failure, use this read-only resident query, replacing `
` with the table named by the error: + +```sql +-- disposition of every selected NEW row that would insert without an ID: +SELECT s.id AS staged_id, d.class, d.selected, m.local_id, m.disposition +FROM delta_stage.
s +JOIN delta_diff.
_class d USING (_delta_row_id) +LEFT JOIN delta_map.
_id_map m ON m.staged_id = s.id +WHERE d.selected AND d.class = 'NEW' AND m.local_id IS NULL; +``` + +**Remediation:** Do not hand-edit `delta_map` or staged rows. Capture the error and query output, run `--cleanup`, address the underlying cause, and rerun from preview. If `disposition = 'NEW_REALLOCATED'` with `local_id IS NULL`, verify the target table's ID sequence is discoverable through `pg_get_serial_sequence()`. The current DDL associates sequences with `ALTER SEQUENCE … OWNED BY`; databases created from older DDL may need that association applied. + +## Recovery, cleanup, and concurrency + +Use the [schema lifecycle table](#schema-and-run-directory-lifecycle) to determine whether resident state should exist. Staging failures do not modify public tables, and apply failures inside the transaction roll back all public-table DML. Run directories are retained regardless of schema cleanup. + +Delta restore uses the same PostgreSQL advisory lock as the subset/full-refresh scripts. This protects against cooperating maintenance scripts, not unrelated application or migration-flow sessions. Avoid non-cooperating writers during apply. + +After capturing any diagnostics, clean up resident schemas with: + +```bash +PGDATABASE=local_extract ./delta_restore_extract.sh --cleanup +``` + +## DELTA_MODE removal and restore_extract.sh delegation + +`DELTA_MODE=true ./restore_extract.sh` is intentionally unsupported and aborts with a pointer to `data-tool/delta_restore_extract.sh`. For CLI compatibility, `restore_extract.sh --delta ` delegates directly to `delta_restore_extract.sh` after removing `--delta`. Prefer explicit preview, validate, and apply commands in operator runbooks. + +## Exit codes + +| Code | Meaning and troubleshooting | +| --- | --- | +| `0` | Preview/apply succeeded, or validate found the selection valid. [Next steps](#exit-0-successful-run). | +| `2` | Preflight/schema drift/corrupt or unreadable dump; for validate, no complete matching resident preview. [Troubleshoot exit 2](#exit-2-preflight-drift-dump-or-stale-preview-failure). | +| `3` | Advisory lock busy or unavailable. [Troubleshoot exit 3](#exit-3-advisory-lock-busy-or-unavailable). | +| `4` | Selection binding, selector, or dependency invalid. [Troubleshoot exit 4](#exit-4-invalid-selection-binding-or-dependency). | +| `5` | Apply verification failed; transaction rolled back. [Troubleshoot exit 5](#exit-5-apply-verification-failed). | + +## Tests + +Run the broad harness from the repo root: + +```bash +make -C data-tool test-delta-restore +data-tool/tests/delta_restore/run_tests.sh --integration +``` + +The always-on static suite covers shell syntax, guarded COPY rewriting, aligned rendering and footer semantics, CLI validation, manifest plumbing, v1/v2 parsing, bindings, and the validate/apply orchestration boundary. PostgreSQL smoke tests cover classification, performance, selection stamping, diagnostics, dependencies, unchanged TSV renderers, manifest structure (T16), and the exact cookbook (T17). Integration covers backup → preview → valid/invalid validate without restaging → apply, private artifacts, real detail truncation/counts, and dump-hash rejection. + +PostgreSQL-backed checks are skipped only when the required local tools or server are unavailable. See `docs/plans/delta_restore_extract_test_plan.md` for the matrix and deferred manual checks. diff --git a/data-tool/scripts/restore/delta/00_install.sql b/data-tool/scripts/restore/delta/00_install.sql new file mode 100644 index 0000000000..268f7d0de4 --- /dev/null +++ b/data-tool/scripts/restore/delta/00_install.sql @@ -0,0 +1,117 @@ +-- Delta restore control schemas and run metadata. +-- Functions are installed separately from 10_functions.sql. + +DROP SCHEMA IF EXISTS delta_stage CASCADE; +DROP SCHEMA IF EXISTS delta_map CASCADE; +DROP SCHEMA IF EXISTS delta_diff CASCADE; +DROP SCHEMA IF EXISTS delta_ctl CASCADE; + +CREATE SCHEMA delta_stage; +CREATE SCHEMA delta_map; +CREATE SCHEMA delta_diff; +CREATE SCHEMA delta_ctl; + +CREATE TABLE delta_ctl.table_config ( + table_name text PRIMARY KEY, + load_phase int NOT NULL, + pk_col text NULL, + nk_exprs text[] NULL, + nk_stage_exprs text[] NOT NULL DEFAULT '{}', + nk_local_exprs text[] NOT NULL DEFAULT '{}', + nk_cols text[] NOT NULL DEFAULT '{}', + nk_enforced boolean NOT NULL DEFAULT false, + fk_map jsonb NOT NULL DEFAULT '{}'::jsonb, + has_last_modified boolean NOT NULL DEFAULT false, + match_mode text NOT NULL DEFAULT 'nk', + classify_ignore_cols text[] NOT NULL DEFAULT '{}', + compare_ignore_cols text[] NOT NULL DEFAULT '{}' +); + +CREATE TABLE delta_ctl.run_metadata ( + key text PRIMARY KEY, + value text, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE delta_ctl.dump_columns ( + table_name text NOT NULL, + column_name text NOT NULL, + ordinal int NOT NULL, + PRIMARY KEY (table_name, column_name) +); + +CREATE TABLE delta_ctl.drift_warnings ( + table_name text NOT NULL, + column_name text NOT NULL, + warning text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE delta_ctl.run_counts ( + table_name text NOT NULL, + count_name text NOT NULL, + row_count bigint NOT NULL DEFAULT 0, + details jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (table_name, count_name) +); + +CREATE TABLE delta_ctl.selection ( + table_name text NOT NULL, + class text NOT NULL, + PRIMARY KEY (table_name, class) +); + +CREATE TABLE delta_ctl.row_selection ( + table_name text NOT NULL, + class text NOT NULL, + mode text NOT NULL CHECK (mode IN ('include', 'exclude')), + kind text NOT NULL CHECK (kind IN ('id', 'row', 'corp')), + value_from bigint, + value_to bigint, + corp_num text, + is_range boolean NOT NULL DEFAULT false, + source_line int NOT NULL, + CHECK ( + (kind IN ('id', 'row') AND value_from IS NOT NULL AND value_to IS NOT NULL AND corp_num IS NULL) + OR (kind = 'corp' AND value_from IS NULL AND value_to IS NULL AND corp_num IS NOT NULL) + ) +); +CREATE INDEX row_selection_lookup_idx + ON delta_ctl.row_selection (table_name, class, mode, kind); + +CREATE TABLE delta_ctl.selection_diagnostics ( + table_name text, + class text, + mode text, + kind text, + selector text, + source_line int, + matched bigint, + problem text +); + +CREATE TABLE delta_ctl.only_corps ( + corp_num text PRIMARY KEY +); + +CREATE TABLE delta_ctl.dependency_violations ( + child_table text NOT NULL, + parent_table text NOT NULL, + reason text NOT NULL, + row_count bigint NOT NULL, + sample_ids text +); + +CREATE TABLE delta_ctl.apply_counts ( + table_name text NOT NULL, + class text NOT NULL, + action text NOT NULL, + expected_count bigint NOT NULL DEFAULT 0, + affected_count bigint NOT NULL DEFAULT 0, + PRIMARY KEY (table_name, class, action) +); + +CREATE TABLE delta_ctl.touched_tables ( + table_name text PRIMARY KEY +); diff --git a/data-tool/scripts/restore/delta/10_functions.sql b/data-tool/scripts/restore/delta/10_functions.sql new file mode 100644 index 0000000000..dd1743a0f9 --- /dev/null +++ b/data-tool/scripts/restore/delta/10_functions.sql @@ -0,0 +1,2810 @@ +-- Delta restore classification, selection, and apply functions. + +CREATE OR REPLACE FUNCTION delta_ctl.assert_identifier(p_value text, p_kind text DEFAULT 'identifier') +RETURNS text +LANGUAGE plpgsql +AS $$ +BEGIN + IF p_value IS NULL OR p_value !~ '^[a-z_][a-z0-9_]*$' THEN + RAISE EXCEPTION 'invalid %: %', p_kind, p_value; + END IF; + RETURN p_value; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.public_rel(p_name text) +RETURNS text +LANGUAGE sql +IMMUTABLE PARALLEL SAFE +AS $$ SELECT format('public.%I', p_name) $$; + +CREATE OR REPLACE FUNCTION delta_ctl.map_table_name(p_table text) +RETURNS text +LANGUAGE sql +IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT p_table || '_id_map' $$; + +CREATE OR REPLACE FUNCTION delta_ctl.map_rel(p_table text) +RETURNS text +LANGUAGE sql +IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT format('delta_map.%I', delta_ctl.map_table_name(p_table)) $$; + +CREATE OR REPLACE FUNCTION delta_ctl.temp_rel(p_name text) +RETURNS text +LANGUAGE sql +IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT format('pg_temp.%I', p_name) $$; + +CREATE OR REPLACE FUNCTION delta_ctl.drop_table_sql(p_rel text) +RETURNS text +LANGUAGE sql +IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT format('DROP TABLE IF EXISTS %s', p_rel) $$; + +CREATE OR REPLACE FUNCTION delta_ctl.analyze_sql(p_rel text) +RETURNS text +LANGUAGE sql +IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT format('ANALYZE %s', p_rel) $$; + +CREATE OR REPLACE FUNCTION delta_ctl.stage_fk_join(p_fk_temp text) +RETURNS text +LANGUAGE sql +IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT format('LEFT JOIN %s fk ON fk._delta_row_id = s._delta_row_id', p_fk_temp) $$; + +CREATE OR REPLACE FUNCTION delta_ctl.aliased_expr(p_expr text, p_alias text) +RETURNS text +LANGUAGE sql +IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT format('%s AS %I', p_expr, p_alias) $$; + +CREATE OR REPLACE FUNCTION delta_ctl.nk_hash_index_sql(p_rel text) +RETURNS text +LANGUAGE sql +IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT format('CREATE INDEX ON %s (nk_hash)', p_rel) $$; + +CREATE OR REPLACE FUNCTION delta_ctl.null_bigint_expr() +RETURNS text +LANGUAGE sql +IMMUTABLE PARALLEL SAFE +AS $$ SELECT 'NULL::bigint' $$; + +CREATE OR REPLACE FUNCTION delta_ctl.class_table_name(p_base_table text) +RETURNS text +LANGUAGE sql +IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT p_base_table || '_class' $$; + +CREATE OR REPLACE FUNCTION delta_ctl.duplicate_table_name(p_table text, p_scope text) +RETURNS text +LANGUAGE sql +IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT 'delta_dup_' || p_table || '_' || p_scope $$; + +CREATE OR REPLACE FUNCTION delta_ctl.false_expr() +RETURNS text +LANGUAGE sql +IMMUTABLE PARALLEL SAFE +AS $$ SELECT 'false' $$; + +CREATE OR REPLACE FUNCTION delta_ctl.hash_parent_mode() +RETURNS text +LANGUAGE sql +IMMUTABLE PARALLEL SAFE +AS $$ SELECT 'hash_parent' $$; + +CREATE OR REPLACE FUNCTION delta_ctl.auth_processing_table() +RETURNS text +LANGUAGE sql +IMMUTABLE PARALLEL SAFE +AS $$ SELECT 'auth_processing' $$; + +CREATE OR REPLACE FUNCTION delta_ctl.table_kind() +RETURNS text +LANGUAGE sql +IMMUTABLE PARALLEL SAFE +AS $$ SELECT 'table' $$; + +CREATE OR REPLACE FUNCTION delta_ctl.class_header() +RETURNS text +LANGUAGE sql +IMMUTABLE PARALLEL SAFE +AS $$ SELECT 'class' $$; + +CREATE OR REPLACE FUNCTION delta_ctl.truncated_rows_marker(p_limit integer) +RETURNS text +LANGUAGE sql +IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT format('# TRUNCATED at %s rows', p_limit) $$; + +CREATE OR REPLACE FUNCTION delta_ctl.parent_table_kind() +RETURNS text +LANGUAGE sql +IMMUTABLE PARALLEL SAFE +AS $$ SELECT 'parent table' $$; + +CREATE OR REPLACE FUNCTION delta_ctl.and_separator() +RETURNS text +LANGUAGE sql +IMMUTABLE PARALLEL SAFE +AS $$ SELECT ' AND ' $$; + +CREATE OR REPLACE FUNCTION delta_ctl.external_fk_pattern() +RETURNS text +LANGUAGE sql +IMMUTABLE PARALLEL SAFE +AS $$ SELECT 'external:%' $$; + +CREATE OR REPLACE FUNCTION delta_ctl.fk_alias_kind() +RETURNS text +LANGUAGE sql +IMMUTABLE PARALLEL SAFE +AS $$ SELECT 'FK alias' $$; + +CREATE OR REPLACE FUNCTION delta_ctl.local_id_col(p_fk_col text) +RETURNS text +LANGUAGE sql +IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT p_fk_col || '_local_id' $$; + +CREATE OR REPLACE FUNCTION delta_ctl.map_fk_expr(p_parent_table text, p_fk_col text) +RETURNS text +LANGUAGE sql +IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT format('delta_ctl.map_fk(%L, s.%I::bigint)', p_parent_table, p_fk_col) $$; + +CREATE OR REPLACE FUNCTION delta_ctl.assert_table_name(p_table text) +RETURNS text +LANGUAGE sql +AS $$ SELECT delta_ctl.assert_identifier(p_table, delta_ctl.table_kind()) $$; + +CREATE OR REPLACE FUNCTION delta_ctl.parent_pending_col(p_fk_col text) +RETURNS text +LANGUAGE sql +IMMUTABLE STRICT PARALLEL SAFE +AS $$ SELECT p_fk_col || '_parent_pending' $$; + +CREATE OR REPLACE FUNCTION delta_ctl.auth_component_op_table() +RETURNS text +LANGUAGE sql +IMMUTABLE PARALLEL SAFE +AS $$ SELECT 'auth_component_operation' $$; + +CREATE OR REPLACE FUNCTION delta_ctl.stage_table_exists(p_table text) +RETURNS boolean +LANGUAGE sql +AS $$ +SELECT to_regclass(format('delta_stage.%I', p_table)) IS NOT NULL; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.map_table_exists(p_table text) +RETURNS boolean +LANGUAGE sql +AS $$ +SELECT to_regclass(delta_ctl.map_rel(p_table)) IS NOT NULL; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.phase_notice(p_table text, p_phase text, p_started_at timestamptz DEFAULT NULL) +RETURNS timestamptz +LANGUAGE plpgsql +AS $$ +DECLARE + v_now timestamptz := clock_timestamp(); +BEGIN + IF p_started_at IS NULL THEN + RAISE NOTICE 'delta_restore table=% phase=% start', COALESCE(p_table, ''), p_phase; + ELSE + RAISE NOTICE 'delta_restore table=% phase=% done elapsed_ms=%', + COALESCE(p_table, ''), p_phase, + round((EXTRACT(epoch FROM (v_now - p_started_at)) * 1000)::numeric, 3); + END IF; + RETURN v_now; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.map_fk(p_parent_table text, p_staged_id bigint) +RETURNS bigint +LANGUAGE plpgsql +AS $$ +DECLARE + v_map regclass; + v_local_id bigint; + v_row_count bigint; +BEGIN + PERFORM delta_ctl.assert_identifier(p_parent_table, delta_ctl.parent_table_kind()); + IF p_staged_id IS NULL THEN + RETURN NULL; + END IF; + + v_map := to_regclass(delta_ctl.map_rel(p_parent_table)); + IF v_map IS NOT NULL THEN + EXECUTE format('SELECT local_id FROM %s WHERE staged_id = $1', v_map) + INTO v_local_id + USING p_staged_id; + GET DIAGNOSTICS v_row_count = ROW_COUNT; + IF v_row_count > 0 THEN + RETURN v_local_id; + END IF; + + -- A map table exists, so a miss means the staged parent was not safely mapped + -- (for example AMBIGUOUS_NK/BLOCKED_FK). Do not fall back to same-number + -- public IDs because those may belong to unrelated local rows. + RETURN NULL; + END IF; + + IF to_regclass(delta_ctl.public_rel(p_parent_table)) IS NOT NULL THEN + EXECUTE format('SELECT id::bigint FROM public.%I WHERE id = $1', p_parent_table) + INTO v_local_id + USING p_staged_id; + GET DIAGNOSTICS v_row_count = ROW_COUNT; + IF v_row_count > 0 THEN + RETURN v_local_id; + END IF; + END IF; + + RETURN NULL; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.parent_pending(p_parent_table text, p_staged_id bigint) +RETURNS boolean +LANGUAGE plpgsql +AS $$ +DECLARE + v_map regclass; + v_disposition text; +BEGIN + PERFORM delta_ctl.assert_identifier(p_parent_table, delta_ctl.parent_table_kind()); + IF p_staged_id IS NULL THEN + RETURN false; + END IF; + + v_map := to_regclass(delta_ctl.map_rel(p_parent_table)); + IF v_map IS NULL THEN + RETURN false; + END IF; + + EXECUTE format('SELECT disposition FROM %s WHERE staged_id = $1', v_map) + INTO v_disposition + USING p_staged_id; + + RETURN COALESCE(v_disposition LIKE 'NEW%', false); +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.jsonb_array_expr(p_exprs text[]) +RETURNS text +LANGUAGE plpgsql +AS $$ +BEGIN + IF p_exprs IS NULL OR array_length(p_exprs, 1) IS NULL THEN + RETURN '''[]''::jsonb'; + END IF; + RETURN 'jsonb_build_array(' || array_to_string(p_exprs, ', ') || ')'; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.null_safe_join(p_left_exprs text[], p_right_exprs text[]) +RETURNS text +LANGUAGE plpgsql +AS $$ +DECLARE + i int; + parts text[] := '{}'; +BEGIN + IF COALESCE(array_length(p_left_exprs, 1), 0) <> COALESCE(array_length(p_right_exprs, 1), 0) + OR COALESCE(array_length(p_left_exprs, 1), 0) = 0 THEN + RAISE EXCEPTION 'cannot build null-safe join for mismatched/empty expression lists: % vs %', p_left_exprs, p_right_exprs; + END IF; + + FOR i IN 1..array_length(p_left_exprs, 1) LOOP + parts := parts || format('(%s) IS NOT DISTINCT FROM (%s)', p_left_exprs[i], p_right_exprs[i]); + END LOOP; + RETURN array_to_string(parts, delta_ctl.and_separator()); +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.nk_hash_expr(p_exprs text[]) +RETURNS text +LANGUAGE plpgsql +AS $$ +BEGIN + IF COALESCE(array_length(p_exprs, 1), 0) = 0 THEN + RAISE EXCEPTION 'cannot build NK hash expression for empty expression list: %', p_exprs; + END IF; + + RETURN 'md5(' || delta_ctl.jsonb_array_expr(p_exprs) || '::text)'; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.stage_compare_expr(p_table text, p_col text, p_fk_alias text DEFAULT NULL) +RETURNS text +LANGUAGE plpgsql +AS $$ +DECLARE + v_parent text; +BEGIN + SELECT value INTO v_parent + FROM delta_ctl.table_config c, jsonb_each_text(c.fk_map) + WHERE c.table_name = p_table + AND key = p_col + AND value NOT LIKE delta_ctl.external_fk_pattern(); + + IF v_parent IS NOT NULL THEN + PERFORM delta_ctl.assert_identifier(v_parent, delta_ctl.parent_table_kind()); + IF p_fk_alias IS NOT NULL THEN + PERFORM delta_ctl.assert_identifier(p_fk_alias, delta_ctl.fk_alias_kind()); + RETURN format('%I.%I', p_fk_alias, delta_ctl.local_id_col(p_col)); + END IF; + RETURN delta_ctl.map_fk_expr(v_parent, p_col); + END IF; + RETURN format('s.%I', p_col); +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.effective_stage_nk_exprs(p_table text, p_fk_alias text DEFAULT NULL) +RETURNS text[] +LANGUAGE plpgsql +AS $$ +DECLARE + v_cfg delta_ctl.table_config%ROWTYPE; + v_expr text; + v_col text; + v_parent text; + v_out text[] := '{}'; +BEGIN + SELECT * INTO v_cfg FROM delta_ctl.table_config WHERE table_name = p_table; + IF NOT FOUND THEN + RAISE EXCEPTION 'missing table_config row for %', p_table; -- NOSONAR + END IF; + + IF p_fk_alias IS NOT NULL THEN + PERFORM delta_ctl.assert_identifier(p_fk_alias, delta_ctl.fk_alias_kind()); + END IF; + + FOREACH v_expr IN ARRAY v_cfg.nk_stage_exprs LOOP + v_col := NULL; + v_parent := NULL; + SELECT key, value INTO v_col, v_parent + FROM jsonb_each_text(v_cfg.fk_map) + WHERE value NOT LIKE delta_ctl.external_fk_pattern() + AND v_expr = delta_ctl.map_fk_expr(value, key) + LIMIT 1; + + IF p_fk_alias IS NOT NULL AND v_col IS NOT NULL THEN + v_out := v_out || format('%I.%I', p_fk_alias, delta_ctl.local_id_col(v_col)); + ELSE + v_out := v_out || v_expr; + END IF; + END LOOP; + + RETURN v_out; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.local_compare_expr(p_col text) +RETURNS text +LANGUAGE sql +AS $$ +SELECT format('l.%I', p_col); +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.compare_columns(p_table text) +RETURNS text[] +LANGUAGE plpgsql +AS $$ +DECLARE + v_cfg delta_ctl.table_config%ROWTYPE; + v_cols text[]; +BEGIN + SELECT * INTO v_cfg FROM delta_ctl.table_config WHERE table_name = p_table; + IF NOT FOUND THEN + RAISE EXCEPTION 'missing table_config row for %', p_table; + END IF; + + SELECT COALESCE(array_agg(a.attname ORDER BY a.attnum), '{}') + INTO v_cols + FROM pg_attribute a + WHERE a.attrelid = delta_ctl.public_rel(p_table)::regclass + AND a.attnum > 0 + AND NOT a.attisdropped + AND a.attname <> '_delta_row_id' + AND (v_cfg.pk_col IS NULL OR a.attname <> v_cfg.pk_col) + AND NOT (a.attname = ANY(v_cfg.nk_cols)) + AND NOT (a.attname = ANY(v_cfg.classify_ignore_cols)) + AND NOT (a.attname = ANY(v_cfg.compare_ignore_cols)); + + RETURN v_cols; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.complete_table_config() +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + c_stage_corp_num CONSTANT text := 's.corp_num'; + c_local_corp_num CONSTANT text := 'l.corp_num'; + c_corp_num CONSTANT text := 'corp_num'; + c_target_environment CONSTANT text := 'target_environment'; + c_stage_target_environment CONSTANT text := 's.target_environment'; + c_local_target_environment CONSTANT text := 'l.target_environment'; + c_stage_flow_name CONSTANT text := 's.flow_name'; + c_stage_environment CONSTANT text := 's.environment'; + c_local_flow_name CONSTANT text := 'l.flow_name'; + c_local_environment CONSTANT text := 'l.environment'; + c_flow_name CONSTANT text := 'flow_name'; + c_environment CONSTANT text := 'environment'; +BEGIN + UPDATE delta_ctl.table_config + SET pk_col = NULL, + nk_enforced = false, + match_mode = 'nk', + nk_stage_exprs = '{}', + nk_local_exprs = '{}', + nk_cols = '{}', + fk_map = '{}'::jsonb, + has_last_modified = false, + compare_ignore_cols = '{}' + WHERE table_name IS NOT NULL; + + UPDATE delta_ctl.table_config SET + pk_col = 'id', + nk_stage_exprs = ARRAY['s.filing_id'], + nk_local_exprs = ARRAY['l.filing_id'], + nk_cols = ARRAY['filing_id'], + nk_enforced = true + WHERE table_name = 'demigrated_filings'; + + UPDATE delta_ctl.table_config SET + nk_stage_exprs = ARRAY['s.email_domain'], + nk_local_exprs = ARRAY['l.email_domain'], + nk_cols = ARRAY['email_domain'], + nk_enforced = true + WHERE table_name = 'email_domain_groups'; + + UPDATE delta_ctl.table_config SET + pk_col = 'id', + nk_stage_exprs = ARRAY['lower(btrim(s.email))'], + nk_local_exprs = ARRAY['lower(btrim(l.email))'], + nk_cols = '{}', + nk_enforced = true + WHERE table_name = 'bad_emails'; + + UPDATE delta_ctl.table_config SET + nk_stage_exprs = ARRAY['lower(btrim(s.email))'], + nk_local_exprs = ARRAY['lower(btrim(l.email))'], + nk_cols = '{}', + nk_enforced = true + WHERE table_name = 'excluded_emails'; + + UPDATE delta_ctl.table_config SET + nk_stage_exprs = ARRAY['lower(btrim(s.email_domain))'], + nk_local_exprs = ARRAY['lower(btrim(l.email_domain))'], + nk_cols = '{}', + nk_enforced = true + WHERE table_name = 'excluded_email_domains'; + + UPDATE delta_ctl.table_config SET + pk_col = 'id', + nk_stage_exprs = ARRAY['lower(btrim(s.email_domain))', 'lower(btrim(s.local_part_pattern))'], + nk_local_exprs = ARRAY['lower(btrim(l.email_domain))', 'lower(btrim(l.local_part_pattern))'], + nk_cols = '{}', + nk_enforced = true + WHERE table_name = 'excluded_email_domain_patterns'; + + UPDATE delta_ctl.table_config SET + nk_stage_exprs = ARRAY[c_stage_corp_num], + nk_local_exprs = ARRAY[c_local_corp_num], + nk_cols = ARRAY[c_corp_num], + nk_enforced = true + WHERE table_name = 'exclude_corps'; + + UPDATE delta_ctl.table_config SET + nk_stage_exprs = ARRAY[c_stage_corp_num, 's.vendor'], + nk_local_exprs = ARRAY[c_local_corp_num, 'l.vendor'], + nk_cols = ARRAY[c_corp_num, 'vendor'], + nk_enforced = false + WHERE table_name = 'corps_with_third_party'; + + UPDATE delta_ctl.table_config SET + nk_stage_exprs = ARRAY['s.identifier'], + nk_local_exprs = ARRAY['l.identifier'], + nk_cols = ARRAY['identifier'], + nk_enforced = false + WHERE table_name = 'bar_corps'; + + UPDATE delta_ctl.table_config SET + pk_col = 'id', + nk_stage_exprs = ARRAY['s.name', c_stage_target_environment, 's.source_db'], + nk_local_exprs = ARRAY['l.name', c_local_target_environment, 'l.source_db'], + nk_cols = ARRAY['name', c_target_environment, 'source_db'], + nk_enforced = false + WHERE table_name = 'mig_group'; + + UPDATE delta_ctl.table_config SET + pk_col = 'id', + nk_stage_exprs = ARRAY['delta_ctl.map_fk(''mig_group'', s.mig_group_id::bigint)', 's.name', c_stage_target_environment], + nk_local_exprs = ARRAY['l.mig_group_id', 'l.name', c_local_target_environment], + nk_cols = ARRAY['mig_group_id', 'name', c_target_environment], + nk_enforced = false, + fk_map = '{"mig_group_id":"mig_group"}'::jsonb + WHERE table_name = 'mig_batch'; + + UPDATE delta_ctl.table_config SET + pk_col = 'id', + nk_stage_exprs = ARRAY['delta_ctl.map_fk(''mig_batch'', s.mig_batch_id::bigint)', c_stage_corp_num], + nk_local_exprs = ARRAY['l.mig_batch_id', c_local_corp_num], + nk_cols = ARRAY['mig_batch_id', c_corp_num], + nk_enforced = false, + fk_map = '{"mig_batch_id":"mig_batch"}'::jsonb + WHERE table_name = 'mig_corp_batch'; + + UPDATE delta_ctl.table_config SET + pk_col = 'id', + nk_stage_exprs = ARRAY[c_stage_corp_num, c_stage_target_environment, 's.account_id', 'delta_ctl.map_fk(''mig_batch'', s.mig_batch_id::bigint)'], + nk_local_exprs = ARRAY[c_local_corp_num, c_local_target_environment, 'l.account_id', 'l.mig_batch_id'], + nk_cols = ARRAY[c_corp_num, c_target_environment, 'account_id', 'mig_batch_id'], + nk_enforced = false, + fk_map = '{"mig_batch_id":"mig_batch"}'::jsonb + WHERE table_name = 'mig_corp_account'; + + UPDATE delta_ctl.table_config SET + pk_col = 'id', + nk_stage_exprs = ARRAY[c_stage_corp_num, c_stage_flow_name, c_stage_environment], + nk_local_exprs = ARRAY[c_local_corp_num, c_local_flow_name, c_local_environment], + nk_cols = ARRAY[c_corp_num, c_flow_name, c_environment], + nk_enforced = true, + has_last_modified = true, + fk_map = '{"mig_batch_id":"mig_batch","corp_num":"external:corporation.corp_num","last_processed_event_id":"external:event.event_id","failed_event_id":"external:event.event_id"}'::jsonb + WHERE table_name = 'corp_processing'; + + UPDATE delta_ctl.table_config SET + pk_col = 'id', + nk_stage_exprs = ARRAY[c_stage_corp_num, c_stage_flow_name, c_stage_environment], + nk_local_exprs = ARRAY[c_local_corp_num, c_local_flow_name, c_local_environment], + nk_cols = ARRAY[c_corp_num, c_flow_name, c_environment], + nk_enforced = true, + has_last_modified = true, + fk_map = '{"mig_batch_id":"mig_batch","corp_num":"external:corporation.corp_num"}'::jsonb + WHERE table_name = 'colin_tracking'; + + UPDATE delta_ctl.table_config SET + pk_col = 'id', + nk_stage_exprs = ARRAY[c_stage_corp_num, c_stage_flow_name, c_stage_environment, 's.operation', 's.operation_scope', 's.attempt_key'], + nk_local_exprs = ARRAY[c_local_corp_num, c_local_flow_name, c_local_environment, 'l.operation', 'l.operation_scope', 'l.attempt_key'], + nk_cols = ARRAY[c_corp_num, c_flow_name, c_environment, 'operation', 'operation_scope', 'attempt_key'], + nk_enforced = true, + has_last_modified = true, + fk_map = '{"mig_batch_id":"mig_batch","corp_num":"external:corporation.corp_num"}'::jsonb + WHERE table_name = delta_ctl.auth_processing_table(); + + UPDATE delta_ctl.table_config SET + pk_col = 'id', + match_mode = delta_ctl.hash_parent_mode(), + compare_ignore_cols = ARRAY['id'], + fk_map = '{"auth_processing_id":"auth_processing"}'::jsonb + WHERE table_name = delta_ctl.auth_component_op_table(); +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.create_class_table(p_table text) +RETURNS void +LANGUAGE plpgsql +AS $$ +BEGIN + PERFORM delta_ctl.assert_table_name(p_table); + EXECUTE format('DROP TABLE IF EXISTS delta_diff.%I', delta_ctl.class_table_name(p_table)); + EXECUTE format($fmt$ + CREATE TABLE delta_diff.%I ( + _delta_row_id bigint PRIMARY KEY, + staged_pk bigint NULL, + local_pk bigint NULL, + local_ctid tid NULL, + class text NOT NULL, + changed_cols text[], + block_reason text, + parent_pending boolean NOT NULL DEFAULT false, + selected boolean NOT NULL DEFAULT false + ) + $fmt$, delta_ctl.class_table_name(p_table)); +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.parent_pending_expr(p_table text, p_fk_alias text DEFAULT NULL) +RETURNS text +LANGUAGE plpgsql +AS $$ +DECLARE + r record; + parts text[] := '{}'; +BEGIN + IF p_fk_alias IS NOT NULL THEN + PERFORM delta_ctl.assert_identifier(p_fk_alias, delta_ctl.fk_alias_kind()); + END IF; + + FOR r IN + SELECT key AS fk_col, value AS parent_table + FROM delta_ctl.table_config c, jsonb_each_text(c.fk_map) + WHERE c.table_name = p_table AND value NOT LIKE delta_ctl.external_fk_pattern() + LOOP + IF p_fk_alias IS NOT NULL THEN + parts := parts || format('COALESCE(%I.%I, false)', p_fk_alias, delta_ctl.parent_pending_col(r.fk_col)); + ELSE + parts := parts || format('delta_ctl.parent_pending(%L, s.%I::bigint)', r.parent_table, r.fk_col); + END IF; + END LOOP; + + IF array_length(parts, 1) IS NULL THEN + RETURN delta_ctl.false_expr(); + END IF; + RETURN array_to_string(parts, ' OR '); +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.create_stage_fk_map(p_table text) +RETURNS text +LANGUAGE plpgsql +AS $$ +DECLARE + r record; + v_temp_name text; + v_temp_rel text; + v_selects text[] := ARRAY['s._delta_row_id']; + v_joins text[] := '{}'; + v_join text; + v_alias text; + v_idx int := 0; +BEGIN + PERFORM delta_ctl.assert_table_name(p_table); + IF NOT delta_ctl.stage_table_exists(p_table) THEN + RETURN NULL; + END IF; + + FOR r IN + SELECT key AS fk_col, value AS parent_table, + to_regclass(delta_ctl.map_rel(value)) AS map_rel + FROM delta_ctl.table_config c, jsonb_each_text(c.fk_map) + WHERE c.table_name = p_table AND value NOT LIKE delta_ctl.external_fk_pattern() + ORDER BY key + LOOP + PERFORM delta_ctl.assert_identifier(r.fk_col, 'FK column'); + PERFORM delta_ctl.assert_identifier(r.parent_table, delta_ctl.parent_table_kind()); + v_idx := v_idx + 1; + v_alias := 'm' || v_idx; + IF r.map_rel IS NOT NULL THEN + v_joins := v_joins || format('LEFT JOIN %s %I ON %I.staged_id = s.%I', r.map_rel, v_alias, v_alias, r.fk_col); + v_selects := v_selects || format('CASE WHEN s.%I IS NULL THEN NULL::bigint ELSE %I.local_id END AS %I', r.fk_col, v_alias, delta_ctl.local_id_col(r.fk_col)); + v_selects := v_selects || format('CASE WHEN s.%I IS NULL THEN false ELSE COALESCE(%I.disposition LIKE ''NEW%%'', false) END AS %I', r.fk_col, v_alias, delta_ctl.parent_pending_col(r.fk_col)); + ELSIF to_regclass(delta_ctl.public_rel(r.parent_table)) IS NOT NULL THEN + v_joins := v_joins || format('LEFT JOIN public.%I %I ON %I.id = s.%I', r.parent_table, v_alias, v_alias, r.fk_col); + v_selects := v_selects || format('CASE WHEN s.%I IS NULL THEN NULL::bigint ELSE %I.id::bigint END AS %I', r.fk_col, v_alias, delta_ctl.local_id_col(r.fk_col)); + v_selects := v_selects || format('false AS %I', delta_ctl.parent_pending_col(r.fk_col)); + ELSE + v_selects := v_selects || format('NULL::bigint AS %I', delta_ctl.local_id_col(r.fk_col)); + v_selects := v_selects || format('false AS %I', delta_ctl.parent_pending_col(r.fk_col)); + END IF; + END LOOP; + + IF v_idx = 0 THEN + RETURN NULL; + END IF; + + v_temp_name := 'delta_fk_' || p_table; + v_temp_rel := delta_ctl.temp_rel(v_temp_name); + v_join := array_to_string(v_joins, E'\n '); + + EXECUTE delta_ctl.drop_table_sql(v_temp_rel); + EXECUTE format($fmt$ + CREATE TEMP TABLE %I AS + SELECT %s + FROM delta_stage.%I s + %s + $fmt$, v_temp_name, array_to_string(v_selects, ', '), p_table, v_join); + EXECUTE format('CREATE INDEX ON %s (_delta_row_id)', v_temp_rel); + EXECUTE delta_ctl.analyze_sql(v_temp_rel); + RETURN v_temp_rel; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.detect_ambiguity(p_table text, p_fk_temp text DEFAULT NULL) +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + v_cfg delta_ctl.table_config%ROWTYPE; + v_stage_exprs text[]; + v_local_exprs text[]; + v_stage_selects text[] := '{}'; + v_local_selects text[] := '{}'; + v_stage_join text := ''; + v_stage_dup text; + v_local_dup text; + v_staged_pk text; + v_parent_pending text; + v_group_by text; + v_stage_dup_join text; + v_local_dup_join text; + v_i int; + v_sql text; +BEGIN + SELECT * INTO v_cfg FROM delta_ctl.table_config WHERE table_name = p_table; + IF NOT FOUND OR v_cfg.nk_enforced OR v_cfg.match_mode <> 'nk' THEN + RETURN; + END IF; + + v_stage_exprs := delta_ctl.effective_stage_nk_exprs(p_table, CASE WHEN p_fk_temp IS NULL THEN NULL ELSE 'fk' END); + v_local_exprs := v_cfg.nk_local_exprs; + IF COALESCE(array_length(v_stage_exprs, 1), 0) = 0 THEN + RETURN; + END IF; + + IF p_fk_temp IS NOT NULL THEN + v_stage_join := delta_ctl.stage_fk_join(p_fk_temp); + END IF; + + FOR v_i IN 1..array_length(v_stage_exprs, 1) LOOP + v_stage_selects := v_stage_selects || delta_ctl.aliased_expr(v_stage_exprs[v_i], 'k' || v_i); + v_local_selects := v_local_selects || delta_ctl.aliased_expr(v_local_exprs[v_i], 'k' || v_i); + END LOOP; + + v_stage_dup := delta_ctl.temp_rel(delta_ctl.duplicate_table_name(p_table, 'stage')); + v_local_dup := delta_ctl.temp_rel(delta_ctl.duplicate_table_name(p_table, 'local')); + v_staged_pk := CASE WHEN v_cfg.pk_col IS NULL THEN delta_ctl.null_bigint_expr() ELSE format('s.%I::bigint', v_cfg.pk_col) END; + v_parent_pending := delta_ctl.parent_pending_expr(p_table, CASE WHEN p_fk_temp IS NULL THEN NULL ELSE 'fk' END); + v_group_by := array_to_string(ARRAY(SELECT gs::text FROM generate_series(1, array_length(v_stage_exprs, 1)) AS gs), ', '); + v_stage_dup_join := array_to_string(ARRAY( + SELECT format('sd.%I IS NOT DISTINCT FROM (%s)', 'k' || i, v_stage_exprs[i]) + FROM generate_subscripts(v_stage_exprs, 1) AS i + ), delta_ctl.and_separator()); + v_stage_dup_join := format('sd.nk_hash = (%s) AND %s', delta_ctl.nk_hash_expr(v_stage_exprs), v_stage_dup_join); + v_local_dup_join := array_to_string(ARRAY( + SELECT format('ld.%I IS NOT DISTINCT FROM (%s)', 'k' || i, v_stage_exprs[i]) + FROM generate_subscripts(v_stage_exprs, 1) AS i + ), delta_ctl.and_separator()); + v_local_dup_join := format('ld.nk_hash = (%s) AND %s', delta_ctl.nk_hash_expr(v_stage_exprs), v_local_dup_join); + + EXECUTE delta_ctl.drop_table_sql(v_stage_dup); + EXECUTE delta_ctl.drop_table_sql(v_local_dup); + EXECUTE format($fmt$ + CREATE TEMP TABLE %I AS + SELECT %s, %s AS nk_hash, count(*)::bigint AS n + FROM delta_stage.%I s + %s + WHERE NOT (%s) + GROUP BY %s HAVING count(*) > 1 + $fmt$, delta_ctl.duplicate_table_name(p_table, 'stage'), array_to_string(v_stage_selects, ', '), + delta_ctl.nk_hash_expr(v_stage_exprs), p_table, v_stage_join, v_parent_pending, v_group_by); + EXECUTE format($fmt$ + CREATE TEMP TABLE %I AS + SELECT %s, %s AS nk_hash, count(*)::bigint AS n + FROM public.%I l + GROUP BY %s HAVING count(*) > 1 + $fmt$, delta_ctl.duplicate_table_name(p_table, 'local'), array_to_string(v_local_selects, ', '), + delta_ctl.nk_hash_expr(v_local_exprs), p_table, v_group_by); + EXECUTE delta_ctl.nk_hash_index_sql(v_stage_dup); + EXECUTE delta_ctl.nk_hash_index_sql(v_local_dup); + EXECUTE delta_ctl.analyze_sql(v_stage_dup); + EXECUTE delta_ctl.analyze_sql(v_local_dup); + + v_sql := format($fmt$ + WITH ambiguous AS ( + SELECT s._delta_row_id, + %s AS staged_pk, + CASE + WHEN sd.n IS NOT NULL AND ld.n IS NOT NULL THEN format('duplicate NK in staged dump (%%s rows) and local table (%%s rows)', sd.n, ld.n) + WHEN sd.n IS NOT NULL THEN format('duplicate NK in staged dump (%%s rows)', sd.n) + ELSE format('duplicate NK in local table (%%s rows)', ld.n) + END AS reason + FROM delta_stage.%I s + %s + LEFT JOIN %s sd ON %s + LEFT JOIN %s ld ON %s + WHERE NOT (%s) + AND (sd.n IS NOT NULL OR ld.n IS NOT NULL) + ) + INSERT INTO delta_diff.%I (_delta_row_id, staged_pk, class, block_reason) + SELECT _delta_row_id, staged_pk, 'AMBIGUOUS_NK', reason -- NOSONAR + FROM ambiguous + ON CONFLICT (_delta_row_id) DO UPDATE + SET class = EXCLUDED.class, block_reason = EXCLUDED.block_reason + $fmt$, v_staged_pk, p_table, v_stage_join, v_stage_dup, v_stage_dup_join, + v_local_dup, v_local_dup_join, v_parent_pending, delta_ctl.class_table_name(p_table)); + + EXECUTE v_sql; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.classify_nk_table(p_table text, p_fk_temp text DEFAULT NULL) +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + v_cfg delta_ctl.table_config%ROWTYPE; + v_join text; + v_join_residual text; + v_stage_nk_exprs text[]; + v_fk_join text := ''; + v_fk_alias text := NULL; + v_staged_pk text; + v_local_pk text; + v_compare_cols text[]; + v_stage_compare_exprs text[] := '{}'; + v_local_compare_exprs text[] := '{}'; + v_compare_equal text; + v_changed_array text; + v_parent_pending text; + v_col text; + v_sql text; + v_started timestamptz; +BEGIN + SELECT * INTO v_cfg FROM delta_ctl.table_config WHERE table_name = p_table; + IF NOT FOUND THEN + RAISE EXCEPTION 'missing table_config row for %', p_table; + END IF; + + IF p_fk_temp IS NOT NULL THEN + v_fk_alias := 'fk'; + v_fk_join := delta_ctl.stage_fk_join(p_fk_temp); + END IF; + + v_started := delta_ctl.phase_notice(p_table, 'create_class_table'); + PERFORM delta_ctl.create_class_table(p_table); + PERFORM delta_ctl.phase_notice(p_table, 'create_class_table', v_started); + + v_started := delta_ctl.phase_notice(p_table, 'detect_ambiguity'); + PERFORM delta_ctl.detect_ambiguity(p_table, p_fk_temp); + PERFORM delta_ctl.phase_notice(p_table, 'detect_ambiguity', v_started); + + v_stage_nk_exprs := delta_ctl.effective_stage_nk_exprs(p_table, v_fk_alias); + v_join_residual := delta_ctl.null_safe_join(v_cfg.nk_local_exprs, v_stage_nk_exprs); + v_join := format('(%s) = (%s) AND %s', + delta_ctl.nk_hash_expr(v_cfg.nk_local_exprs), + delta_ctl.nk_hash_expr(v_stage_nk_exprs), + v_join_residual); + v_staged_pk := CASE WHEN v_cfg.pk_col IS NULL THEN delta_ctl.null_bigint_expr() ELSE format('s.%I::bigint', v_cfg.pk_col) END; + v_local_pk := CASE WHEN v_cfg.pk_col IS NULL THEN delta_ctl.null_bigint_expr() ELSE format('l.%I::bigint', v_cfg.pk_col) END; + v_parent_pending := delta_ctl.parent_pending_expr(p_table, v_fk_alias); + v_compare_cols := delta_ctl.compare_columns(p_table); + + IF array_length(v_compare_cols, 1) IS NULL THEN + v_compare_equal := 'true'; + v_changed_array := 'NULL::text[]'; + ELSE + FOREACH v_col IN ARRAY v_compare_cols LOOP + v_stage_compare_exprs := v_stage_compare_exprs || delta_ctl.stage_compare_expr(p_table, v_col, v_fk_alias); + v_local_compare_exprs := v_local_compare_exprs || delta_ctl.local_compare_expr(v_col); + END LOOP; + v_compare_equal := format('%s IS NOT DISTINCT FROM %s', + delta_ctl.jsonb_array_expr(v_stage_compare_exprs), + delta_ctl.jsonb_array_expr(v_local_compare_exprs)); + v_changed_array := 'array_remove(ARRAY[' || array_to_string(ARRAY( + SELECT format('CASE WHEN %s IS DISTINCT FROM %s THEN %L END', + delta_ctl.stage_compare_expr(p_table, c, v_fk_alias), delta_ctl.local_compare_expr(c), c) + FROM unnest(v_compare_cols) AS c + ), ', ') || '], NULL)'; + END IF; + + v_sql := format($fmt$ + INSERT INTO delta_diff.%I (_delta_row_id, staged_pk, local_pk, local_ctid, class, changed_cols, parent_pending) + SELECT s._delta_row_id, + %s AS staged_pk, + %s AS local_pk, + l.ctid AS local_ctid, + CASE + WHEN (%s) THEN 'NEW' -- NOSONAR + WHEN l.ctid IS NULL THEN 'NEW' + WHEN %s THEN 'UNCHANGED' -- NOSONAR + WHEN %s THEN 'CHANGED_LOCAL_NEWER' -- NOSONAR + ELSE 'CHANGED' -- NOSONAR + END AS class, + CASE WHEN (%s) OR l.ctid IS NULL OR %s THEN NULL::text[] ELSE %s END AS changed_cols, + (%s) AS parent_pending + FROM delta_stage.%I s + %s + LEFT JOIN public.%I l ON %s + WHERE NOT EXISTS ( + SELECT 1 FROM delta_diff.%I d WHERE d._delta_row_id = s._delta_row_id + ) + $fmt$, delta_ctl.class_table_name(p_table), v_staged_pk, v_local_pk, v_parent_pending, + v_compare_equal, + CASE WHEN v_cfg.has_last_modified THEN 'l.last_modified > s.last_modified' ELSE delta_ctl.false_expr() END, + v_parent_pending, v_compare_equal, v_changed_array, v_parent_pending, + p_table, v_fk_join, p_table, v_join, delta_ctl.class_table_name(p_table)); + + v_started := delta_ctl.phase_notice(p_table, 'main_classify_insert'); + EXECUTE v_sql; + PERFORM delta_ctl.phase_notice(p_table, 'main_classify_insert', v_started); +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.auth_component_hash_expr(p_alias text, p_parent_expr text) +RETURNS text +LANGUAGE plpgsql +AS $$ +DECLARE + v_exprs text[] := ARRAY[p_parent_expr]; + v_col text; +BEGIN + FOR v_col IN + SELECT a.attname + FROM pg_attribute a + WHERE a.attrelid = 'public.auth_component_operation'::regclass + AND a.attnum > 0 + AND NOT a.attisdropped + AND a.attname NOT IN ('id', 'auth_processing_id') + ORDER BY a.attnum + LOOP + v_exprs := v_exprs || format('%s.%I', p_alias, v_col); + END LOOP; + + RETURN 'md5(' || delta_ctl.jsonb_array_expr(v_exprs) || '::text)'; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.classify_auth_component_operation() +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + c_new_pattern CONSTANT text := 'NEW%'; + v_stage_hash text; + v_local_hash text; + v_sql text; +BEGIN + PERFORM delta_ctl.create_class_table(delta_ctl.auth_component_op_table()); + + IF to_regclass('delta_map.auth_processing_id_map') IS NULL THEN + EXECUTE $sql$ + INSERT INTO delta_diff.auth_component_operation_class (_delta_row_id, staged_pk, class, block_reason) + SELECT s._delta_row_id, s.id::bigint, 'BLOCKED_FK', 'auth_processing map not available' -- NOSONAR + FROM delta_stage.auth_component_operation s + $sql$; + RETURN; + END IF; + + v_stage_hash := delta_ctl.auth_component_hash_expr('s', 'apm.local_id'); + v_local_hash := delta_ctl.auth_component_hash_expr('l', 'l.auth_processing_id'); + + v_sql := format($fmt$ + WITH staged AS ( + SELECT s._delta_row_id, + s.id::bigint AS staged_pk, + apm.local_id AS parent_local_id, + apm.disposition AS parent_disposition, + %s AS content_hash + FROM delta_stage.auth_component_operation s + LEFT JOIN delta_map.auth_processing_id_map apm + ON apm.staged_id = s.auth_processing_id + ), relevant_parent_ids AS ( + SELECT DISTINCT parent_local_id + FROM staged + WHERE parent_disposition = 'MATCHED' -- NOSONAR + AND parent_local_id IS NOT NULL + ), local_hashes AS ( + SELECT min(local_pk)::bigint AS local_pk, content_hash + FROM ( + SELECT l.id::bigint AS local_pk, + %s AS content_hash + FROM public.auth_component_operation l + JOIN relevant_parent_ids rp ON rp.parent_local_id = l.auth_processing_id + ) h + GROUP BY content_hash + ) + INSERT INTO delta_diff.auth_component_operation_class + (_delta_row_id, staged_pk, local_pk, class, parent_pending, block_reason) + SELECT st._delta_row_id, + st.staged_pk, + lh.local_pk, + CASE + WHEN st.parent_disposition IS NULL THEN 'BLOCKED_FK' + WHEN st.parent_disposition LIKE %L THEN 'NEW' + WHEN st.parent_disposition <> 'MATCHED' THEN 'BLOCKED_PARENT' -- NOSONAR + WHEN lh.local_pk IS NULL THEN 'NEW' + ELSE 'UNCHANGED' + END AS class, + COALESCE(st.parent_disposition LIKE %L, false) AS parent_pending, + CASE + WHEN st.parent_disposition IS NULL THEN 'auth_processing parent not present' + WHEN st.parent_disposition <> 'MATCHED' AND st.parent_disposition NOT LIKE %L + THEN 'auth_processing parent blocked: ' || st.parent_disposition + END AS block_reason + FROM staged st + LEFT JOIN local_hashes lh + ON lh.content_hash = st.content_hash +$fmt$, v_stage_hash, v_local_hash, c_new_pattern, c_new_pattern, c_new_pattern); + + EXECUTE v_sql; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.build_id_map(p_table text) +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + v_cfg delta_ctl.table_config%ROWTYPE; + v_map_name text; + v_dup_count bigint; + v_sql text; +BEGIN + SELECT * INTO v_cfg FROM delta_ctl.table_config WHERE table_name = p_table; + IF NOT FOUND OR v_cfg.pk_col IS NULL OR NOT delta_ctl.stage_table_exists(p_table) THEN + RETURN; + END IF; + + v_map_name := delta_ctl.map_table_name(p_table); + + EXECUTE format('SELECT count(*) FROM (SELECT %I FROM delta_stage.%I WHERE %I IS NOT NULL GROUP BY 1 HAVING count(*) > 1) x', + v_cfg.pk_col, p_table, v_cfg.pk_col) + INTO v_dup_count; + IF v_dup_count > 0 THEN + RAISE EXCEPTION 'duplicate staged primary key values in %.%', 'delta_stage', p_table; + END IF; + + EXECUTE format('DROP TABLE IF EXISTS delta_map.%I', v_map_name); + EXECUTE format('CREATE TABLE delta_map.%I (staged_id bigint PRIMARY KEY, local_id bigint NULL, disposition text NOT NULL)', v_map_name); + + v_sql := format($fmt$ + INSERT INTO delta_map.%I (staged_id, local_id, disposition) + SELECT s.%I::bigint AS staged_id, + CASE + WHEN d.class IN ('BLOCKED_FK', 'AMBIGUOUS_NK') THEN NULL::bigint + WHEN d.local_pk IS NOT NULL THEN d.local_pk + WHEN d.class = 'NEW' AND NOT EXISTS (SELECT 1 FROM public.%I l2 WHERE l2.%I = s.%I) THEN s.%I::bigint + ELSE NULL::bigint + END AS local_id, + CASE + WHEN d.class IN ('BLOCKED_FK', 'AMBIGUOUS_NK') THEN d.class + WHEN d.local_pk IS NOT NULL THEN 'MATCHED' + WHEN d.class = 'NEW' AND NOT EXISTS (SELECT 1 FROM public.%I l2 WHERE l2.%I = s.%I) THEN 'NEW_PRESERVED' + WHEN d.class = 'NEW' THEN 'NEW_REALLOCATED' + ELSE d.class + END AS disposition + FROM delta_stage.%I s + JOIN delta_diff.%I d ON d._delta_row_id = s._delta_row_id + WHERE s.%I IS NOT NULL + AND d.class IN ('NEW', 'CHANGED', 'CHANGED_LOCAL_NEWER', 'UNCHANGED', 'BLOCKED_FK') + $fmt$, v_map_name, v_cfg.pk_col, p_table, v_cfg.pk_col, v_cfg.pk_col, v_cfg.pk_col, + p_table, v_cfg.pk_col, v_cfg.pk_col, p_table, delta_ctl.class_table_name(p_table), v_cfg.pk_col); + + EXECUTE v_sql; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.detect_external_fk_blocking(p_table text) +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + r record; + v_target text; + v_target_table text; + v_target_col text; + v_sql text; +BEGIN + FOR r IN + SELECT key AS fk_col, value AS fk_target + FROM delta_ctl.table_config c, jsonb_each_text(c.fk_map) + WHERE c.table_name = p_table AND value LIKE delta_ctl.external_fk_pattern() + LOOP + v_target := substring(r.fk_target FROM 10); + v_target_table := split_part(v_target, '.', 1); + v_target_col := split_part(v_target, '.', 2); + PERFORM delta_ctl.assert_identifier(v_target_table, 'external table'); + PERFORM delta_ctl.assert_identifier(v_target_col, 'external column'); + + v_sql := format($fmt$ + UPDATE delta_diff.%I d + SET class = 'BLOCKED_FK', + block_reason = concat_ws('; ', d.block_reason, format('%I %%s not in local extract', s.%I)) + FROM delta_stage.%I s + WHERE d._delta_row_id = s._delta_row_id + AND d.class IN ('NEW', 'CHANGED', 'CHANGED_LOCAL_NEWER') + AND s.%I IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM public.%I ext WHERE ext.%I = s.%I + ) + $fmt$, delta_ctl.class_table_name(p_table), r.fk_col, r.fk_col, p_table, r.fk_col, v_target_table, v_target_col, r.fk_col); + + EXECUTE v_sql; + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.detect_preserved_fk_blocking(p_table text, p_fk_temp text DEFAULT NULL) +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + r record; + v_sql text; + v_fk_temp text; +BEGIN + v_fk_temp := p_fk_temp; + IF v_fk_temp IS NULL THEN + v_fk_temp := delta_ctl.create_stage_fk_map(p_table); + END IF; + IF v_fk_temp IS NULL THEN + RETURN; + END IF; + + FOR r IN + SELECT key AS fk_col, value AS parent_table + FROM delta_ctl.table_config c, jsonb_each_text(c.fk_map) + WHERE c.table_name = p_table AND value NOT LIKE delta_ctl.external_fk_pattern() + LOOP + PERFORM delta_ctl.assert_identifier(r.parent_table, delta_ctl.parent_table_kind()); + + v_sql := format($fmt$ + UPDATE delta_diff.%I d + SET class = 'BLOCKED_FK', + changed_cols = NULL, + block_reason = concat_ws('; ', d.block_reason, format('%I %%s has no safe mapped/local %s parent', s.%I)) + FROM delta_stage.%I s + JOIN %s fk ON fk._delta_row_id = s._delta_row_id + WHERE d._delta_row_id = s._delta_row_id + AND d.class IN ('NEW', 'CHANGED', 'CHANGED_LOCAL_NEWER', 'UNCHANGED') + AND s.%I IS NOT NULL + AND NOT COALESCE(fk.%I, false) + AND fk.%I IS NULL + $fmt$, delta_ctl.class_table_name(p_table), r.fk_col, r.parent_table, r.fk_col, + p_table, v_fk_temp, r.fk_col, delta_ctl.parent_pending_col(r.fk_col), delta_ctl.local_id_col(r.fk_col)); + + EXECUTE v_sql; + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.record_class_counts(p_table text) +RETURNS void +LANGUAGE plpgsql +AS $$ +BEGIN + IF NOT delta_ctl.stage_table_exists(p_table) THEN + RETURN; + END IF; + + EXECUTE format($fmt$ + INSERT INTO delta_ctl.run_counts(table_name, count_name, row_count) + SELECT %L, class, count(*) + FROM delta_diff.%I + GROUP BY class + ON CONFLICT (table_name, count_name) DO UPDATE + SET row_count = EXCLUDED.row_count, created_at = now() + $fmt$, p_table, delta_ctl.class_table_name(p_table)); +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.record_local_only_count(p_table text, p_fk_temp text DEFAULT NULL) +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + v_cfg delta_ctl.table_config%ROWTYPE; + v_join text; + v_sql text; + v_local_hash text; + v_stage_hash text; + v_stage_exprs text[]; + v_stage_selects text[] := '{}'; + v_stage_keys text; + v_stage_join text := ''; + v_fk_alias text := NULL; + v_i int; +BEGIN + SELECT * INTO v_cfg FROM delta_ctl.table_config WHERE table_name = p_table; + IF NOT FOUND THEN + RETURN; + END IF; + + IF NOT delta_ctl.stage_table_exists(p_table) THEN + RETURN; + END IF; + + IF v_cfg.match_mode = delta_ctl.hash_parent_mode() THEN + IF p_table <> delta_ctl.auth_component_op_table() THEN + RETURN; + END IF; + IF to_regclass('delta_map.auth_processing_id_map') IS NULL THEN + INSERT INTO delta_ctl.run_counts(table_name, count_name, row_count) + VALUES (p_table, 'LOCAL_ONLY', 0) -- NOSONAR + ON CONFLICT (table_name, count_name) DO UPDATE + SET row_count = EXCLUDED.row_count, created_at = now(); + RETURN; + END IF; + + v_local_hash := delta_ctl.auth_component_hash_expr('l', 'l.auth_processing_id'); + v_stage_hash := delta_ctl.auth_component_hash_expr('s', 'apm.local_id'); + v_sql := format($fmt$ + INSERT INTO delta_ctl.run_counts(table_name, count_name, row_count) + WITH relevant_parent_ids AS ( + SELECT DISTINCT local_id AS parent_local_id + FROM delta_map.auth_processing_id_map + WHERE disposition = 'MATCHED' + AND local_id IS NOT NULL + ), stage_hashes AS ( + SELECT DISTINCT %s AS content_hash + FROM delta_stage.auth_component_operation s + JOIN delta_map.auth_processing_id_map apm + ON apm.staged_id = s.auth_processing_id + WHERE apm.disposition = 'MATCHED' + AND apm.local_id IS NOT NULL + ) + SELECT %L, 'LOCAL_ONLY', count(*) + FROM public.auth_component_operation l + JOIN relevant_parent_ids rp ON rp.parent_local_id = l.auth_processing_id + WHERE NOT EXISTS ( + SELECT 1 FROM stage_hashes sh + WHERE sh.content_hash = %s + ) + ON CONFLICT (table_name, count_name) DO UPDATE + SET row_count = EXCLUDED.row_count, created_at = now() + $fmt$, v_stage_hash, p_table, v_local_hash); + EXECUTE v_sql; + RETURN; + END IF; + + IF p_fk_temp IS NOT NULL THEN + v_fk_alias := 'fk'; + v_stage_join := delta_ctl.stage_fk_join(p_fk_temp); + END IF; + + v_stage_exprs := delta_ctl.effective_stage_nk_exprs(p_table, v_fk_alias); + IF COALESCE(array_length(v_stage_exprs, 1), 0) = 0 THEN + RAISE NOTICE 'delta_restore table=% phase=record_local_only_count skipped reason=no_nk_exprs', p_table; + RETURN; + END IF; + + FOR v_i IN 1..array_length(v_stage_exprs, 1) LOOP + v_stage_selects := v_stage_selects || delta_ctl.aliased_expr(v_stage_exprs[v_i], 'k' || v_i); + END LOOP; + + v_stage_keys := delta_ctl.temp_rel('delta_stage_keys_' || p_table); + v_stage_hash := delta_ctl.nk_hash_expr(v_stage_exprs); + v_local_hash := delta_ctl.nk_hash_expr(v_cfg.nk_local_exprs); + v_join := array_to_string(ARRAY( + SELECT format('sk.%I IS NOT DISTINCT FROM (%s)', 'k' || i, v_cfg.nk_local_exprs[i]) + FROM generate_subscripts(v_stage_exprs, 1) AS i + ), delta_ctl.and_separator()); + v_join := format('sk.nk_hash = (%s) AND %s', v_local_hash, v_join); + + EXECUTE delta_ctl.drop_table_sql(v_stage_keys); + EXECUTE format($fmt$ + CREATE TEMP TABLE %I AS + SELECT DISTINCT %s, %s AS nk_hash + FROM delta_stage.%I s + %s + $fmt$, 'delta_stage_keys_' || p_table, array_to_string(v_stage_selects, ', '), v_stage_hash, p_table, v_stage_join); + EXECUTE delta_ctl.nk_hash_index_sql(v_stage_keys); + EXECUTE delta_ctl.analyze_sql(v_stage_keys); + + v_sql := format($fmt$ + INSERT INTO delta_ctl.run_counts(table_name, count_name, row_count) + SELECT %L, 'LOCAL_ONLY', count(*) + FROM public.%I l + WHERE NOT EXISTS ( + SELECT 1 FROM %s sk WHERE %s + ) + ON CONFLICT (table_name, count_name) DO UPDATE + SET row_count = EXCLUDED.row_count, created_at = now() + $fmt$, p_table, p_table, v_stage_keys, v_join); + EXECUTE v_sql; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.classify_table(p_table text) +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + v_cfg delta_ctl.table_config%ROWTYPE; + v_fk_temp text; + v_started timestamptz; + v_table_started timestamptz; +BEGIN + SELECT * INTO v_cfg FROM delta_ctl.table_config WHERE table_name = p_table; + IF NOT FOUND THEN + RAISE EXCEPTION 'missing table_config row for %', p_table; + END IF; + + IF NOT delta_ctl.stage_table_exists(p_table) THEN + RETURN; + END IF; + + v_table_started := delta_ctl.phase_notice(p_table, 'classify_table'); + + v_started := delta_ctl.phase_notice(p_table, 'prepare_fk_map'); + v_fk_temp := delta_ctl.create_stage_fk_map(p_table); + PERFORM delta_ctl.phase_notice(p_table, 'prepare_fk_map', v_started); + + IF v_cfg.match_mode = delta_ctl.hash_parent_mode() THEN + IF p_table <> delta_ctl.auth_component_op_table() THEN + RAISE EXCEPTION 'unsupported hash_parent table: %', p_table; + END IF; + v_started := delta_ctl.phase_notice(p_table, 'classify_auth_component_operation'); + PERFORM delta_ctl.classify_auth_component_operation(); + PERFORM delta_ctl.phase_notice(p_table, 'classify_auth_component_operation', v_started); + ELSE + v_started := delta_ctl.phase_notice(p_table, 'classify_nk_table'); + PERFORM delta_ctl.classify_nk_table(p_table, v_fk_temp); + PERFORM delta_ctl.phase_notice(p_table, 'classify_nk_table', v_started); + END IF; + + v_started := delta_ctl.phase_notice(p_table, 'detect_external_fk_blocking'); + PERFORM delta_ctl.detect_external_fk_blocking(p_table); + PERFORM delta_ctl.phase_notice(p_table, 'detect_external_fk_blocking', v_started); + + v_started := delta_ctl.phase_notice(p_table, 'detect_preserved_fk_blocking'); + PERFORM delta_ctl.detect_preserved_fk_blocking(p_table, v_fk_temp); + PERFORM delta_ctl.phase_notice(p_table, 'detect_preserved_fk_blocking', v_started); + + v_started := delta_ctl.phase_notice(p_table, 'build_id_map'); + PERFORM delta_ctl.build_id_map(p_table); + PERFORM delta_ctl.phase_notice(p_table, 'build_id_map', v_started); + + v_started := delta_ctl.phase_notice(p_table, 'record_class_counts'); + PERFORM delta_ctl.record_class_counts(p_table); + PERFORM delta_ctl.phase_notice(p_table, 'record_class_counts', v_started); + + v_started := delta_ctl.phase_notice(p_table, 'record_local_only_count'); + PERFORM delta_ctl.record_local_only_count(p_table, v_fk_temp); + PERFORM delta_ctl.phase_notice(p_table, 'record_local_only_count', v_started); + + PERFORM delta_ctl.phase_notice(p_table, 'classify_table', v_table_started); +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.run_preview_classification() +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + r record; + v_started timestamptz; +BEGIN + v_started := delta_ctl.phase_notice(NULL, 'run_preview_classification'); + + DELETE FROM delta_ctl.run_counts + WHERE count_name NOT IN ('STAGED', 'SKIPPED_ABSENT'); -- NOSONAR + TRUNCATE delta_ctl.apply_counts; + TRUNCATE delta_ctl.touched_tables; + TRUNCATE delta_ctl.dependency_violations; + + PERFORM delta_ctl.complete_table_config(); + + FOR r IN + SELECT table_name + FROM delta_ctl.table_config + ORDER BY load_phase, table_name + LOOP + PERFORM delta_ctl.classify_table(r.table_name); + END LOOP; + + PERFORM delta_ctl.phase_notice(NULL, 'run_preview_classification', v_started); +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.fn_counts() +RETURNS TABLE(table_name text, count_name text, row_count bigint, load_phase int) +LANGUAGE sql +AS $$ +SELECT rc.table_name, rc.count_name, rc.row_count, tc.load_phase +FROM delta_ctl.run_counts rc +LEFT JOIN delta_ctl.table_config tc ON tc.table_name = rc.table_name +ORDER BY tc.load_phase ASC NULLS LAST, rc.table_name ASC, rc.count_name ASC; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.tsv_escape(p_value text, p_max_len int DEFAULT 8192) +RETURNS text +LANGUAGE sql +IMMUTABLE PARALLEL SAFE +AS $$ +SELECT CASE + WHEN p_value IS NULL THEN E'\\N' + ELSE regexp_replace( + replace(replace(replace(replace(replace( + CASE WHEN length(p_value) > GREATEST(p_max_len, 1) + THEN left(p_value, GREATEST(p_max_len, 1)) || '…[truncated]' + ELSE p_value END, + E'\\', E'\\\\'), E'\t', E'\\t'), E'\n', E'\\n'), E'\r', E'\\r'), chr(27), '␛'), + '[[:cntrl:]]', '�', 'g') +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.display_text(p_value text, p_max_len int DEFAULT 60) +RETURNS text +LANGUAGE plpgsql +IMMUTABLE PARALLEL SAFE +AS $$ +DECLARE + v_value text; +BEGIN + IF p_value IS NULL THEN + RETURN 'NULL'; + END IF; + v_value := replace(replace(replace(replace( + p_value, E'\n', '␤'), E'\t', '␉'), E'\r', '␍'), chr(27), '␛'); + v_value := regexp_replace(v_value, '[[:cntrl:]]', '�', 'g'); + IF length(v_value) > GREATEST(p_max_len, 1) THEN + v_value := left(v_value, GREATEST(p_max_len, 1)) || '…'; + END IF; + RETURN v_value; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.display_value(p_row jsonb, p_col text, p_max_len int DEFAULT 60) +RETURNS text +LANGUAGE plpgsql +IMMUTABLE PARALLEL SAFE +AS $$ +BEGIN + IF p_row IS NULL OR NOT (p_row ? p_col) OR p_row -> p_col = 'null'::jsonb THEN + RETURN 'NULL'; + END IF; + RETURN delta_ctl.display_text(COALESCE(p_row ->> p_col, ''), p_max_len); +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.display_columns(p_table text) +RETURNS text[] +LANGUAGE plpgsql +AS $$ +DECLARE + v_cfg delta_ctl.table_config%ROWTYPE; + v_cols text[]; + v_expr text; + v_match text[]; +BEGIN + SELECT * INTO v_cfg FROM delta_ctl.table_config WHERE table_name = p_table; + IF NOT FOUND THEN + RAISE EXCEPTION 'missing table_config row for %', p_table; + END IF; + IF array_length(v_cfg.nk_cols, 1) IS NOT NULL THEN + RETURN v_cfg.nk_cols; + END IF; + FOREACH v_expr IN ARRAY v_cfg.nk_stage_exprs LOOP + v_match := regexp_match(v_expr, '^s\\.([a-z_][a-z0-9_]*)$'); + IF v_match IS NOT NULL AND NOT (v_match[1] = ANY(COALESCE(v_cols, '{}'))) THEN + v_cols := COALESCE(v_cols, '{}') || v_match[1]; + END IF; + END LOOP; + IF array_length(v_cols, 1) IS NOT NULL THEN + RETURN v_cols; + END IF; + SELECT COALESCE(array_agg(attname ORDER BY attnum), '{}') INTO v_cols + FROM ( + SELECT a.attname, a.attnum + FROM pg_attribute a + WHERE a.attrelid = delta_ctl.public_rel(p_table)::regclass + AND a.attnum > 0 AND NOT a.attisdropped + AND (v_cfg.pk_col IS NULL OR a.attname <> v_cfg.pk_col) + ORDER BY a.attnum + LIMIT 3 + ) q; + RETURN v_cols; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.render_samples(p_table text, p_class text, p_limit int DEFAULT 20) +RETURNS SETOF text +LANGUAGE plpgsql +AS $$ +DECLARE + r record; + v_cfg delta_ctl.table_config%ROWTYPE; + v_sql text; + v_local_join text; + v_col text; + v_cols text[]; + v_values text; + v_line text; + v_changed_count int; + v_limit int := GREATEST(COALESCE(p_limit, 20), 0); +BEGIN + PERFORM delta_ctl.assert_table_name(p_table); + SELECT * INTO v_cfg FROM delta_ctl.table_config WHERE table_name = p_table; + IF v_limit = 0 OR NOT FOUND + OR to_regclass(format('delta_diff.%I', delta_ctl.class_table_name(p_table))) IS NULL THEN -- NOSONAR + RETURN; + END IF; + + v_cols := delta_ctl.display_columns(p_table); + v_local_join := CASE + WHEN v_cfg.pk_col IS NULL THEN 'LEFT JOIN public.' || quote_ident(p_table) || ' l ON l.ctid = d.local_ctid' + ELSE format('LEFT JOIN public.%I l ON l.%I = d.local_pk', p_table, v_cfg.pk_col) + END; + v_sql := format($fmt$ + SELECT d._delta_row_id, d.staged_pk, d.local_pk, d.changed_cols, + d.block_reason, d.parent_pending, to_jsonb(s) AS staged_json, + to_jsonb(l) AS local_json + FROM delta_diff.%I d + JOIN delta_stage.%I s ON s._delta_row_id = d._delta_row_id + %s + WHERE d.class = $1 + ORDER BY d.staged_pk NULLS LAST, d._delta_row_id + LIMIT $2 + $fmt$, delta_ctl.class_table_name(p_table), p_table, v_local_join); + + FOR r IN EXECUTE v_sql USING p_class, v_limit LOOP + v_values := ''; + FOREACH v_col IN ARRAY v_cols LOOP + v_values := v_values || format(' · %s=%s', v_col, + delta_ctl.display_value(r.staged_json, v_col, 40)); + END LOOP; + v_line := format('▸ %s%srow %s%s%s', + CASE WHEN v_cfg.pk_col IS NOT NULL THEN format('id %s · ', COALESCE(r.staged_pk::text, '—')) ELSE '' END, + '', + r._delta_row_id, + v_values, + CASE WHEN r.parent_pending THEN ' · parent pending' ELSE '' END + || CASE WHEN r.block_reason IS NOT NULL + THEN ' · ' || delta_ctl.display_text(r.block_reason, 80) ELSE '' END); + IF length(v_line) > 220 THEN + v_line := left(v_line, 219) || '…'; + END IF; + RETURN NEXT v_line; + IF r.changed_cols IS NOT NULL THEN + v_changed_count := 0; + FOREACH v_col IN ARRAY r.changed_cols LOOP + EXIT WHEN v_changed_count >= 5; + RETURN NEXT format(' changed %s: %s → %s', v_col, + delta_ctl.display_value(r.local_json, v_col, 60), + delta_ctl.display_value(r.staged_json, v_col, 60)); + v_changed_count := v_changed_count + 1; + END LOOP; + END IF; + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.render_new_rows_tsv(p_table text, p_limit int DEFAULT 10000) +RETURNS SETOF text +LANGUAGE plpgsql +AS $$ +DECLARE + v_cfg delta_ctl.table_config%ROWTYPE; + v_cols text[]; + v_col text; + v_sql text; + v_line text; + r record; + v_limit int := GREATEST(COALESCE(p_limit, 10000), 0); +BEGIN + PERFORM delta_ctl.assert_table_name(p_table); + SELECT * INTO v_cfg FROM delta_ctl.table_config WHERE table_name = p_table; + IF NOT FOUND OR NOT delta_ctl.stage_table_exists(p_table) THEN RETURN; END IF; + SELECT array_agg(a.attname ORDER BY a.attnum) INTO v_cols + FROM pg_attribute a + WHERE a.attrelid = delta_ctl.public_rel(p_table)::regclass + AND a.attnum > 0 AND NOT a.attisdropped; + RETURN NEXT 'selector_id' || E'\t' || 'delta_row_id' || E'\t' || array_to_string(v_cols, E'\t'); -- NOSONAR + v_sql := format($fmt$ + SELECT d.staged_pk, d._delta_row_id, to_jsonb(s) AS staged_json + FROM delta_diff.%I d + JOIN delta_stage.%I s ON s._delta_row_id = d._delta_row_id + WHERE d.class = 'NEW' + ORDER BY d.staged_pk NULLS LAST, d._delta_row_id + LIMIT $1 + $fmt$, delta_ctl.class_table_name(p_table), p_table); + FOR r IN EXECUTE v_sql USING v_limit LOOP + v_line := delta_ctl.tsv_escape(COALESCE(r.staged_pk, r._delta_row_id)::text) + || E'\t' || r._delta_row_id::text; + FOREACH v_col IN ARRAY v_cols LOOP + v_line := v_line || E'\t' || delta_ctl.tsv_escape(r.staged_json ->> v_col); + END LOOP; + RETURN NEXT v_line; + END LOOP; + IF (SELECT row_count FROM delta_ctl.run_counts + WHERE table_name = p_table AND count_name = 'NEW') > v_limit THEN + RETURN NEXT delta_ctl.truncated_rows_marker(v_limit); + END IF; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.render_changed_rows_tsv(p_table text, p_limit int DEFAULT 10000) +RETURNS SETOF text +LANGUAGE plpgsql +AS $$ +DECLARE + v_cfg delta_ctl.table_config%ROWTYPE; + v_local_join text; + v_sql text; + v_col text; + r record; + v_limit int := GREATEST(COALESCE(p_limit, 10000), 0); +BEGIN + PERFORM delta_ctl.assert_table_name(p_table); + SELECT * INTO v_cfg FROM delta_ctl.table_config WHERE table_name = p_table; + IF NOT FOUND OR NOT delta_ctl.stage_table_exists(p_table) THEN RETURN; END IF; + RETURN NEXT 'selector_id' || E'\t' || 'delta_row_id' || E'\t' || 'local_pk' -- NOSONAR + || E'\t' || delta_ctl.class_header() || E'\t' || 'column' || E'\t' || 'local_value' || E'\t' || 'dump_value'; + v_local_join := CASE + WHEN v_cfg.pk_col IS NULL THEN format('JOIN public.%I l ON l.ctid = d.local_ctid', p_table) + ELSE format('JOIN public.%I l ON l.%I = d.local_pk', p_table, v_cfg.pk_col) + END; + v_sql := format($fmt$ + WITH ranked AS ( + SELECT d.staged_pk, d._delta_row_id, d.local_pk, d.class, d.changed_cols, + to_jsonb(s) AS staged_json, to_jsonb(l) AS local_json, + row_number() OVER ( + PARTITION BY d.class ORDER BY d.staged_pk NULLS LAST, d._delta_row_id + ) AS class_row_number + FROM delta_diff.%I d + JOIN delta_stage.%I s ON s._delta_row_id = d._delta_row_id + %s + WHERE d.class IN ('CHANGED', 'CHANGED_LOCAL_NEWER') + ) + SELECT staged_pk, _delta_row_id, local_pk, class, changed_cols, staged_json, local_json + FROM ranked + WHERE class_row_number <= $1 + ORDER BY staged_pk NULLS LAST, _delta_row_id + $fmt$, delta_ctl.class_table_name(p_table), p_table, v_local_join); + FOR r IN EXECUTE v_sql USING v_limit LOOP + FOREACH v_col IN ARRAY COALESCE(r.changed_cols, '{}') LOOP + RETURN NEXT delta_ctl.tsv_escape(COALESCE(r.staged_pk, r._delta_row_id)::text) + || E'\t' || r._delta_row_id::text + || E'\t' || delta_ctl.tsv_escape(r.local_pk::text) + || E'\t' || r.class + || E'\t' || delta_ctl.tsv_escape(v_col) + || E'\t' || delta_ctl.tsv_escape(r.local_json ->> v_col) + || E'\t' || delta_ctl.tsv_escape(r.staged_json ->> v_col); + END LOOP; + END LOOP; + IF EXISTS (SELECT 1 FROM delta_ctl.run_counts + WHERE table_name = p_table AND count_name IN ('CHANGED', 'CHANGED_LOCAL_NEWER') + AND row_count > v_limit) THEN + RETURN NEXT delta_ctl.truncated_rows_marker(v_limit); + END IF; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.render_blocked_rows_tsv(p_table text, p_limit int DEFAULT 10000) +RETURNS SETOF text +LANGUAGE plpgsql +AS $$ +DECLARE + v_cols text[]; + v_col text; + v_sql text; + v_line text; + r record; + v_limit int := GREATEST(COALESCE(p_limit, 10000), 0); +BEGIN + PERFORM delta_ctl.assert_table_name(p_table); + SELECT array_agg(a.attname ORDER BY a.attnum) INTO v_cols + FROM pg_attribute a + WHERE a.attrelid = delta_ctl.public_rel(p_table)::regclass + AND a.attnum > 0 AND NOT a.attisdropped; + RETURN NEXT 'selector_id' || E'\t' || 'delta_row_id' || E'\t' || delta_ctl.class_header() + || E'\t' || array_to_string(v_cols, E'\t') || E'\t' || 'block_reason'; + v_sql := format($fmt$ + WITH ranked AS ( + SELECT d.staged_pk, d._delta_row_id, d.class, d.block_reason, + to_jsonb(s) AS staged_json, + row_number() OVER ( + PARTITION BY d.class ORDER BY d.staged_pk NULLS LAST, d._delta_row_id + ) AS class_row_number + FROM delta_diff.%I d + JOIN delta_stage.%I s ON s._delta_row_id = d._delta_row_id + WHERE d.class IN ('BLOCKED_FK', 'AMBIGUOUS_NK') + ) + SELECT staged_pk, _delta_row_id, class, block_reason, staged_json + FROM ranked + WHERE class_row_number <= $1 + ORDER BY staged_pk NULLS LAST, _delta_row_id + $fmt$, delta_ctl.class_table_name(p_table), p_table); + FOR r IN EXECUTE v_sql USING v_limit LOOP + v_line := delta_ctl.tsv_escape(COALESCE(r.staged_pk, r._delta_row_id)::text) + || E'\t' || r._delta_row_id::text || E'\t' || r.class; + FOREACH v_col IN ARRAY v_cols LOOP + v_line := v_line || E'\t' || delta_ctl.tsv_escape(r.staged_json ->> v_col); + END LOOP; + RETURN NEXT v_line || E'\t' || delta_ctl.tsv_escape(r.block_reason); + END LOOP; + IF EXISTS (SELECT 1 FROM delta_ctl.run_counts + WHERE table_name = p_table AND count_name IN ('BLOCKED_FK', 'AMBIGUOUS_NK') + AND row_count > v_limit) THEN + RETURN NEXT delta_ctl.truncated_rows_marker(v_limit); + END IF; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.render_preview_lines(p_sample_size int DEFAULT 20) +RETURNS SETOF text +LANGUAGE plpgsql +AS $$ +DECLARE + c_table_hdr CONSTANT text := delta_ctl.table_kind(); + t record; + c record; + sample text; + summary text; +BEGIN + RETURN NEXT 'Class counts'; + RETURN NEXT '------------'; + RETURN NEXT format('%-36s %10s %10s %10s %18s %12s %13s %12s %12s', + c_table_hdr, 'NEW', 'CHANGED', 'UNCHANGED', 'CHANGED_LOCAL_NEWER', 'BLOCKED_FK', 'AMBIGUOUS_NK', 'LOCAL_ONLY', 'SKIPPED'); + + FOR t IN + SELECT tc.table_name, tc.load_phase, + COALESCE(max(rc.row_count) FILTER (WHERE rc.count_name = 'NEW'), 0) AS new_n, + COALESCE(max(rc.row_count) FILTER (WHERE rc.count_name = 'CHANGED'), 0) AS changed_n, + COALESCE(max(rc.row_count) FILTER (WHERE rc.count_name = 'UNCHANGED'), 0) AS unchanged_n, + COALESCE(max(rc.row_count) FILTER (WHERE rc.count_name = 'CHANGED_LOCAL_NEWER'), 0) AS local_newer_n, + COALESCE(max(rc.row_count) FILTER (WHERE rc.count_name = 'BLOCKED_FK'), 0) AS blocked_fk_n, + COALESCE(max(rc.row_count) FILTER (WHERE rc.count_name = 'AMBIGUOUS_NK'), 0) AS ambiguous_n, + COALESCE(max(rc.row_count) FILTER (WHERE rc.count_name = 'LOCAL_ONLY'), 0) AS local_only_n, + COALESCE(max(rc.row_count) FILTER (WHERE rc.count_name = 'SKIPPED_ABSENT'), 0) AS skipped_n + FROM delta_ctl.table_config tc + LEFT JOIN delta_ctl.run_counts rc ON rc.table_name = tc.table_name + GROUP BY tc.table_name, tc.load_phase + ORDER BY tc.load_phase, tc.table_name + LOOP + RETURN NEXT format('%-36s %10s %10s %10s %18s %12s %13s %12s %12s', + t.table_name, t.new_n, t.changed_n, t.unchanged_n, t.local_newer_n, t.blocked_fk_n, t.ambiguous_n, t.local_only_n, t.skipped_n); + END LOOP; + + RETURN NEXT ''; + RETURN NEXT 'Samples'; + RETURN NEXT '-------'; + + FOR t IN + SELECT table_name, load_phase + FROM delta_ctl.table_config + WHERE delta_ctl.stage_table_exists(table_name) + ORDER BY load_phase, table_name + LOOP + summary := NULL; + SELECT string_agg(format('%s %s', count_name, row_count), ' · ' ORDER BY count_name) + INTO summary + FROM delta_ctl.run_counts + WHERE table_name = t.table_name + AND count_name <> 'STAGED'; + + RETURN NEXT ''; + RETURN NEXT format('── %s ──', t.table_name); + RETURN NEXT COALESCE(summary, 'no class counts'); + + FOR c IN + SELECT count_name, row_count + FROM delta_ctl.run_counts + WHERE table_name = t.table_name + AND count_name IN ('NEW', 'CHANGED', 'CHANGED_LOCAL_NEWER', 'BLOCKED_FK', 'AMBIGUOUS_NK') + AND row_count > 0 + ORDER BY CASE count_name + WHEN 'CHANGED' THEN 1 + WHEN 'CHANGED_LOCAL_NEWER' THEN 2 + WHEN 'NEW' THEN 3 + WHEN 'BLOCKED_FK' THEN 4 + WHEN 'AMBIGUOUS_NK' THEN 5 + ELSE 99 END + LOOP + RETURN NEXT format('%s — showing up to %s of %s', c.count_name, p_sample_size, c.row_count); + FOR sample IN SELECT * FROM delta_ctl.render_samples(t.table_name, c.count_name, p_sample_size) LOOP + RETURN NEXT sample; + END LOOP; + END LOOP; + END LOOP; + + RETURN NEXT ''; + RETURN NEXT 'Selection: copy and edit selection.conf, check it with --mode validate, then run --mode apply.'; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.render_selection_manifest() +RETURNS SETOF text +LANGUAGE plpgsql +AS $$ +DECLARE + t record; + c record; + v_sha text; + v_staged text; + v_manifest_default text; + v_selector_limit_raw text; + v_selector_limit_normalized text; + v_selector_limit_present boolean; + v_selector_suggestion_limit integer := 50; + v_selector_suggestion_limit_max constant integer := 100000; + v_default_include text; + v_default_new bigint := 0; + v_default_changed bigint := 0; + v_default_total bigint := 0; + v_default_tables bigint := 0; + v_range_tokens text[]; + v_range_sizes bigint[]; + v_singleton_tokens text[]; + v_singleton_lines text[]; + v_total_rows bigint; + v_singleton_count bigint; + v_range_count bigint; + v_id_col text; + v_primary_kind text; + v_example_values text; + v_invalid_ids boolean; + v_class_width integer; + v_selector_line_max constant integer := 120; + v_selector_prefix text; + v_selector_line text; + v_range_annotation text; + v_token text; + v_count_token text; + v_token_index integer; + v_singleton_line_index integer; +BEGIN + SELECT value INTO v_sha FROM delta_ctl.run_metadata WHERE key = 'dump_sha256'; + SELECT COALESCE( + (SELECT value FROM delta_ctl.run_metadata WHERE key = 'manifest_default'), + 'new,changed') -- NOSONAR + INTO v_manifest_default; + IF v_manifest_default NOT IN ('new,changed', 'none') THEN + RAISE EXCEPTION 'unsupported manifest_default metadata value: %', v_manifest_default; + END IF; + + SELECT EXISTS ( + SELECT 1 FROM delta_ctl.run_metadata WHERE key = 'selector_suggestion_limit'), -- NOSONAR + (SELECT value FROM delta_ctl.run_metadata WHERE key = 'selector_suggestion_limit') + INTO v_selector_limit_present, v_selector_limit_raw; + IF v_selector_limit_present THEN + IF v_selector_limit_raw IS NULL OR v_selector_limit_raw !~ '^[0-9]+$' THEN + RAISE EXCEPTION + 'invalid selector_suggestion_limit metadata value: % (allowed range: 0..100000)', -- NOSONAR + COALESCE(v_selector_limit_raw, ''); + END IF; + v_selector_limit_normalized := COALESCE( + NULLIF(regexp_replace(v_selector_limit_raw, '^0+', ''), ''), '0'); + IF length(v_selector_limit_normalized) > length(v_selector_suggestion_limit_max::text) THEN + RAISE EXCEPTION + 'invalid selector_suggestion_limit metadata value: % (allowed range: 0..100000)', -- NOSONAR + v_selector_limit_raw; + END IF; + v_selector_suggestion_limit := v_selector_limit_normalized::integer; + IF v_selector_suggestion_limit > v_selector_suggestion_limit_max THEN + RAISE EXCEPTION + 'invalid selector_suggestion_limit metadata value: % (allowed range: 0..100000)', -- NOSONAR + v_selector_limit_raw; + END IF; + END IF; + + v_default_include := CASE WHEN v_manifest_default = 'none' THEN '' ELSE 'new,changed' END; + + SELECT string_agg(format('%s=%s', tc.table_name, COALESCE(rc.row_count, 0)), ' ' ORDER BY tc.load_phase, tc.table_name) + INTO v_staged + FROM delta_ctl.table_config tc + LEFT JOIN delta_ctl.run_counts rc + ON rc.table_name = tc.table_name AND rc.count_name = 'STAGED'; + + SELECT GREATEST(3, COALESCE(max(length(tc.table_name) + 2), 3)) + 2 + INTO v_class_width + FROM delta_ctl.table_config tc; + + IF v_manifest_default = 'new,changed' THEN + SELECT COALESCE(sum(counts.new_n), 0), + COALESCE(sum(counts.changed_n), 0), + count(*) FILTER (WHERE counts.new_n + counts.changed_n > 0) + INTO v_default_new, v_default_changed, v_default_tables + FROM ( + SELECT tc.table_name, + COALESCE(max(rc.row_count) FILTER (WHERE rc.count_name = 'NEW'), 0) AS new_n, + COALESCE(max(rc.row_count) FILTER (WHERE rc.count_name = 'CHANGED'), 0) AS changed_n + FROM delta_ctl.table_config tc + LEFT JOIN delta_ctl.run_counts rc ON rc.table_name = tc.table_name + GROUP BY tc.table_name + ) counts; + v_default_total := v_default_new + v_default_changed; + END IF; + + RETURN NEXT '# delta-selection v2'; + RETURN NEXT '# dump_sha256=' || COALESCE(v_sha, ''); + RETURN NEXT '# staged ' || COALESCE(v_staged, ''); + RETURN NEXT '#'; + RETURN NEXT '# The two binding headers above tie row selectors to this reviewed preview.'; + RETURN NEXT '# Keep them unchanged when uncommenting any .rows selector.'; + RETURN NEXT '# Class-only selection files remain backward compatible and do not require bindings.'; + RETURN NEXT '#'; + RETURN NEXT '# ---------------------------------------------------------------------------'; -- NOSONAR + RETURN NEXT '# ACTIVE SELECTION'; + RETURN NEXT '# Edit these lines to choose which classes are eligible for apply.'; + RETURN NEXT '# ---------------------------------------------------------------------------'; -- NOSONAR + RETURN NEXT '#'; + RETURN NEXT rpad('[*]', v_class_width, ' ') || 'include=' || v_default_include; + RETURN NEXT format( + '# Default selection ([*] %s) currently matches %s rows across %s tables (new=%s changed=%s).', + v_manifest_default, v_default_total, v_default_tables, v_default_new, v_default_changed); + RETURN NEXT '#'; + + FOR t IN + SELECT tc.table_name, tc.load_phase, + COALESCE(max(rc.row_count) FILTER (WHERE rc.count_name = 'NEW'), 0) AS new_n, + COALESCE(max(rc.row_count) FILTER (WHERE rc.count_name = 'CHANGED'), 0) AS changed_n, + COALESCE(max(rc.row_count) FILTER (WHERE rc.count_name = 'CHANGED_LOCAL_NEWER'), 0) AS local_newer_n, + COALESCE(( + SELECT string_agg(DISTINCT fk.parent_name, ',' ORDER BY fk.parent_name) + FROM jsonb_each_text(tc.fk_map) AS fk(child_col, parent_name) + WHERE fk.parent_name NOT LIKE 'external:%' + ), '') AS parents + FROM delta_ctl.table_config tc + LEFT JOIN delta_ctl.run_counts rc ON rc.table_name = tc.table_name + GROUP BY tc.table_name, tc.load_phase, tc.fk_map + ORDER BY tc.load_phase, tc.table_name + LOOP + RETURN NEXT rpad(format('[%s]', t.table_name), v_class_width, ' ') + || format('include=%s # new=%s changed=%s changed_local_newer=%s%s', + v_default_include, t.new_n, t.changed_n, t.local_newer_n, + CASE WHEN t.parents = '' THEN '' ELSE ' parents=' || t.parents END); + END LOOP; + + RETURN NEXT '#'; + FOR c IN + SELECT class_name, selector_name + FROM (VALUES ('NEW', 'new'), ('CHANGED', 'changed')) AS classes(class_name, selector_name) + LOOP + RETURN NEXT format( + '# Small %s sets receive ready-to-uncomment suggestions:', c.class_name); + FOR t IN + SELECT tc.table_name, tc.load_phase, tc.pk_col + FROM delta_ctl.table_config tc + LEFT JOIN delta_ctl.run_counts rc ON rc.table_name = tc.table_name + GROUP BY tc.table_name, tc.load_phase, tc.pk_col + HAVING COALESCE(max(rc.row_count) FILTER (WHERE rc.count_name = c.class_name), 0) + BETWEEN 1 AND v_selector_suggestion_limit + ORDER BY tc.load_phase, tc.table_name + LOOP + IF delta_ctl.stage_table_exists(t.table_name) THEN + v_id_col := CASE WHEN t.pk_col IS NULL THEN '_delta_row_id' ELSE t.pk_col END; + EXECUTE format($fmt$ + WITH source AS ( + SELECT s.%I::numeric AS n + FROM delta_stage.%I s + JOIN delta_diff.%I d ON d._delta_row_id = s._delta_row_id + WHERE d.class = $1 + ), ordered AS ( + SELECT n, n - row_number() OVER (ORDER BY n)::numeric AS grp + FROM source + ), runs AS ( + SELECT min(n) AS lo, max(n) AS hi, count(*)::bigint AS row_count + FROM ordered + GROUP BY grp + ) + SELECT + COALESCE( + array_agg((lo::text || '-' || hi::text) ORDER BY lo) + FILTER (WHERE row_count > 1), + '{}'::text[]), + COALESCE( + array_agg(row_count ORDER BY lo) + FILTER (WHERE row_count > 1), + '{}'::bigint[]), + COALESCE( + array_agg(lo::text ORDER BY lo) + FILTER (WHERE row_count = 1), + '{}'::text[]), + COALESCE(sum(row_count), 0)::bigint, + count(*) FILTER (WHERE row_count = 1)::bigint, + count(*) FILTER (WHERE row_count > 1)::bigint, + EXISTS (SELECT 1 FROM source WHERE n < 0) + FROM runs + $fmt$, v_id_col, t.table_name, delta_ctl.class_table_name(t.table_name)) + INTO v_range_tokens, v_range_sizes, v_singleton_tokens, + v_total_rows, v_singleton_count, v_range_count, v_invalid_ids + USING c.class_name; + + -- An actual blank line separates each table block, including the first + -- block from its section heading. + RETURN NEXT ''; + RETURN NEXT format( + '# [%s] Exact %s set: rows=%s singletons=%s ranges=%s', + t.table_name, c.class_name, v_total_rows, v_singleton_count, v_range_count); + + IF v_invalid_ids THEN + RETURN NEXT format( + '# [%s] Exact %s selector unavailable for this small set.', + t.table_name, c.class_name); + RETURN NEXT format( + '# Review details/%s.%s.tsv; negative id: values are outside the selector grammar.', + t.table_name, c.selector_name); + ELSE + v_selector_prefix := format( + '# [%s] %s.rows include=%s:', + t.table_name, c.selector_name, + CASE WHEN t.pk_col IS NULL THEN 'row' ELSE 'id' END); + + IF cardinality(v_range_tokens) > 0 THEN + v_selector_line := v_selector_prefix; + v_range_annotation := '# Range counts:'; -- NOSONAR + FOR v_token_index IN 1..cardinality(v_range_tokens) LOOP + v_token := v_range_tokens[v_token_index]; + v_count_token := format( + '%s (%s rows)', v_token, v_range_sizes[v_token_index]); + IF v_selector_line <> v_selector_prefix + AND (length(v_selector_line) + 1 + length(v_token) > v_selector_line_max + OR length(v_range_annotation) + 2 + length(v_count_token) + > v_selector_line_max) THEN + RETURN NEXT v_selector_line; + RETURN NEXT v_range_annotation; + v_selector_line := v_selector_prefix; + v_range_annotation := '# Range counts:'; -- NOSONAR + END IF; + v_selector_line := v_selector_line + || CASE WHEN v_selector_line = v_selector_prefix THEN '' ELSE ',' END + || v_token; + v_range_annotation := v_range_annotation + || CASE WHEN v_range_annotation = '# Range counts:' THEN ' ' ELSE ', ' END + || v_count_token; + END LOOP; + RETURN NEXT v_selector_line; + RETURN NEXT v_range_annotation; + END IF; + + IF cardinality(v_singleton_tokens) > 0 THEN + -- Keep the range/count pair adjacent, then separate singleton IDs + -- with one actual blank line when both categories exist. + IF cardinality(v_range_tokens) > 0 THEN + RETURN NEXT ''; + END IF; + + -- Build singleton lines before emitting them so long lists can be + -- split into visual paragraphs after every four selector lines. + v_singleton_lines := '{}'::text[]; + v_selector_line := v_selector_prefix; + FOREACH v_token IN ARRAY v_singleton_tokens LOOP + IF v_selector_line <> v_selector_prefix + AND length(v_selector_line) + 1 + length(v_token) > v_selector_line_max THEN + v_singleton_lines := array_append(v_singleton_lines, v_selector_line); + v_selector_line := v_selector_prefix; + END IF; + v_selector_line := v_selector_line + || CASE WHEN v_selector_line = v_selector_prefix THEN '' ELSE ',' END + || v_token; + END LOOP; + v_singleton_lines := array_append(v_singleton_lines, v_selector_line); + + FOR v_singleton_line_index IN 1..cardinality(v_singleton_lines) LOOP + IF cardinality(v_singleton_lines) > 8 + AND v_singleton_line_index > 1 + AND mod(v_singleton_line_index - 1, 4) = 0 THEN + RETURN NEXT ''; + END IF; + RETURN NEXT v_singleton_lines[v_singleton_line_index]; + END LOOP; + END IF; + END IF; + END IF; + END LOOP; + RETURN NEXT '#'; + END LOOP; + + FOR c IN + SELECT class_name, selector_name + FROM (VALUES ('NEW', 'new'), ('CHANGED', 'changed')) AS classes(class_name, selector_name) + LOOP + RETURN NEXT format( + '# Large %s sets receive a useful pointer rather than no guidance:', c.class_name); + FOR t IN + SELECT tc.table_name, tc.load_phase, tc.pk_col, + COALESCE(max(rc.row_count) + FILTER (WHERE rc.count_name = c.class_name), 0) AS class_n + FROM delta_ctl.table_config tc + LEFT JOIN delta_ctl.run_counts rc ON rc.table_name = tc.table_name + GROUP BY tc.table_name, tc.load_phase, tc.pk_col + HAVING COALESCE(max(rc.row_count) + FILTER (WHERE rc.count_name = c.class_name), 0) > v_selector_suggestion_limit + ORDER BY tc.load_phase, tc.table_name + LOOP + v_primary_kind := CASE WHEN t.pk_col IS NULL THEN 'row' ELSE 'id' END; + v_example_values := CASE WHEN t.pk_col IS NULL THEN '4,10-15' ELSE '100,200-210' END; + RETURN NEXT format( + '# %s has %s %s rows; choose selector_id%s from:', + t.table_name, t.class_n, c.class_name, + CASE + WHEN t.table_name = delta_ctl.auth_component_op_table() THEN '' + WHEN delta_ctl.corp_col_expr(t.table_name) IS NOT NULL + THEN ' or corp values' + ELSE '' + END); + RETURN NEXT format('# details/%s.%s.tsv', t.table_name, c.selector_name); + RETURN NEXT '# Example syntax:'; + RETURN NEXT format('# [%s] %s.rows include=%s:%s', + t.table_name, c.selector_name, v_primary_kind, v_example_values); + IF delta_ctl.corp_col_expr(t.table_name) IS NOT NULL THEN + IF t.table_name = delta_ctl.auth_component_op_table() THEN + RETURN NEXT '# corp: selectors are also supported via the staged auth_processing parent:'; + END IF; + RETURN NEXT format('# [%s] %s.rows include=corp:BC0000001,BC0000002', + t.table_name, c.selector_name); + END IF; + RETURN NEXT '#'; + END LOOP; + END LOOP; + + RETURN NEXT '# ---------------------------------------------------------------------------'; -- NOSONAR + RETURN NEXT '# Full selector grammar and 13 worked examples: selection_cookbook.txt (this run dir)'; + RETURN NEXT '# ---------------------------------------------------------------------------'; -- NOSONAR +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.render_selection_cookbook() +RETURNS SETOF text +LANGUAGE plpgsql +AS $$ +BEGIN + RETURN NEXT '# Classes: new | changed | changed_local_newer (other classes are never applyable).'; + RETURN NEXT '# Selector kinds: id: for PK tables; row: for PK-less tables; corp: only where supported.'; + RETURN NEXT '# Multiple includes union their matches; excludes subtract; --only-corps is a final intersection.'; + RETURN NEXT '#'; + RETURN NEXT '# ---------------------------------------------------------------------------'; -- NOSONAR + RETURN NEXT '# OPERATOR COOKBOOK'; + RETURN NEXT '# Everything below is commented documentation. Copy or uncomment only the'; + RETURN NEXT '# examples you intend to use. Row selectors never enable a class by themselves.'; + RETURN NEXT '# ---------------------------------------------------------------------------'; -- NOSONAR + RETURN NEXT '#'; + RETURN NEXT '# 1. Apply all NEW and CHANGED rows for a table:'; + RETURN NEXT '#'; + RETURN NEXT '# [mig_corp_batch] include=new,changed'; + RETURN NEXT '#'; + RETURN NEXT '# 2. Apply only NEW rows for a table:'; + RETURN NEXT '#'; + RETURN NEXT '# [mig_corp_batch] include=new'; + RETURN NEXT '#'; + RETURN NEXT '# 3. Apply only CHANGED rows for a table:'; + RETURN NEXT '#'; + RETURN NEXT '# [corp_processing] include=changed'; + RETURN NEXT '#'; + RETURN NEXT '# 4. Disable a table entirely:'; + RETURN NEXT '#'; + RETURN NEXT '# [mig_corp_batch] include='; + RETURN NEXT '#'; + RETURN NEXT '# 5. Apply specific NEW rows by staged primary key.'; + RETURN NEXT '# Lists and inclusive ranges may be mixed:'; + RETURN NEXT '#'; + RETURN NEXT '# [mig_corp_batch] include=new,changed'; + RETURN NEXT '# [mig_corp_batch] new.rows include=id:3067,7922,8241-8242'; + RETURN NEXT '#'; + RETURN NEXT '# 6. Apply every NEW row except specific IDs:'; + RETURN NEXT '#'; + RETURN NEXT '# [mig_corp_account] include=new,changed'; + RETURN NEXT '# [mig_corp_account] new.rows exclude=id:23,100-105'; + RETURN NEXT '#'; + RETURN NEXT '# 7. Combine multiple includes, then subtract exclusions.'; + RETURN NEXT '# Include lines are unioned; exclude lines are applied afterward:'; + RETURN NEXT '#'; + RETURN NEXT '# [corp_processing] include=new,changed'; -- NOSONAR + RETURN NEXT '# [corp_processing] new.rows include=id:53946-53959'; + RETURN NEXT '# [corp_processing] new.rows include=corp:BC0000001,BC0000002'; + RETURN NEXT '# [corp_processing] new.rows exclude=id:53947,53952'; + RETURN NEXT '#'; + RETURN NEXT '# 8. Apply specific CHANGED rows:'; + RETURN NEXT '#'; + RETURN NEXT '# [corp_processing] include=new,changed'; -- NOSONAR + RETURN NEXT '# [corp_processing] changed.rows include=id:881,900-905'; + RETURN NEXT '#'; + RETURN NEXT '# 9. Apply CHANGED rows for selected corporations:'; + RETURN NEXT '#'; + RETURN NEXT '# [corp_processing] include=new,changed'; -- NOSONAR + RETURN NEXT '# [corp_processing] changed.rows include=corp:BC0000001,BC0000002'; + RETURN NEXT '#'; + RETURN NEXT '# 10. Explicitly opt into locally newer rows.'; + RETURN NEXT '# CAUTION: these may overwrite more recently modified local values:'; + RETURN NEXT '#'; + RETURN NEXT '# [corp_processing] include=new,changed,changed_local_newer'; + RETURN NEXT '# [corp_processing] changed_local_newer.rows include=id:901'; + RETURN NEXT '#'; + RETURN NEXT '# 11. Select rows from a PK-less table using the staged row ordinal:'; + RETURN NEXT '#'; + RETURN NEXT '# [email_domain_groups] include=new'; + RETURN NEXT '# [email_domain_groups] new.rows include=row:4,10-15'; + RETURN NEXT '#'; + RETURN NEXT '# 12. Parent dependency reminder:'; + RETURN NEXT '# A selected NEW child whose preserved parent is also NEW requires that'; + RETURN NEXT '# parent row to remain selected.'; + RETURN NEXT '#'; + RETURN NEXT '# [mig_batch] include=new'; + RETURN NEXT '# [mig_batch] new.rows include=id:152'; + RETURN NEXT '# [mig_corp_account] include=new'; + RETURN NEXT '# [mig_corp_account] new.rows include=id:23643-23658'; + RETURN NEXT '#'; + RETURN NEXT '# 13. --only-corps is an additional global intersection:'; + RETURN NEXT '#'; + RETURN NEXT '# ./delta_restore_extract.sh ... ' || chr(92); + RETURN NEXT '# --selection-file selection.conf ' || chr(92); + RETURN NEXT '# --only-corps selected_corps.txt'; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.corp_col_expr(p_table text) +RETURNS text +LANGUAGE plpgsql +AS $$ +BEGIN + PERFORM delta_ctl.assert_table_name(p_table); + CASE p_table + WHEN delta_ctl.auth_component_op_table() THEN + IF delta_ctl.stage_table_exists(delta_ctl.auth_processing_table()) THEN + RETURN '(SELECT ap.corp_num::text FROM delta_stage.auth_processing ap WHERE ap.id = s.auth_processing_id LIMIT 1)'; + END IF; + RETURN NULL; + WHEN 'corp_processing', 'colin_tracking', delta_ctl.auth_processing_table(), + 'mig_corp_batch', 'mig_corp_account', 'exclude_corps', 'corps_with_third_party' + THEN RETURN 's.corp_num::text'; + WHEN 'bar_corps' THEN RETURN 's.identifier::text'; + ELSE RETURN NULL; + END CASE; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.selection_filter_expr(p_table text) +RETURNS text +LANGUAGE plpgsql +AS $$ +DECLARE + v_corp text; +BEGIN + PERFORM 1 FROM delta_ctl.only_corps LIMIT 1; + IF NOT FOUND THEN RETURN 'true'; END IF; + v_corp := delta_ctl.corp_col_expr(p_table); + IF v_corp IS NULL THEN RETURN 'true'; END IF; + RETURN format('EXISTS (SELECT 1 FROM delta_ctl.only_corps oc WHERE oc.corp_num = (%s))', v_corp); +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.verify_row_selection() +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + r record; + v_cfg delta_ctl.table_config%ROWTYPE; + v_condition text; + v_corp text; + v_filter text; + v_matched bigint; + v_problem text; + v_dups boolean; +BEGIN + TRUNCATE delta_ctl.selection_diagnostics; + FOR r IN + SELECT *, CASE WHEN kind = 'corp' THEN corp_num + WHEN is_range THEN value_from || '-' || value_to ELSE value_from::text END AS selector + FROM delta_ctl.row_selection + ORDER BY source_line, table_name, class, mode, kind, value_from, corp_num + LOOP + SELECT * INTO v_cfg FROM delta_ctl.table_config WHERE table_name = r.table_name; + v_problem := NULL; + v_matched := NULL; + IF NOT EXISTS (SELECT 1 FROM delta_ctl.selection s + WHERE s.table_name = r.table_name AND s.class = r.class) THEN + v_problem := 'CLASS_NOT_INCLUDED'; + ELSIF NOT delta_ctl.stage_table_exists(r.table_name) + OR to_regclass(format('delta_diff.%I', delta_ctl.class_table_name(r.table_name))) IS NULL THEN + v_matched := 0; + v_problem := 'NO_MATCH'; + ELSIF r.kind = 'id' AND v_cfg.pk_col IS NULL THEN + v_problem := 'KIND_UNSUPPORTED'; + ELSIF r.kind = 'corp' AND delta_ctl.corp_col_expr(r.table_name) IS NULL THEN + v_problem := 'KIND_UNSUPPORTED'; + END IF; + + IF v_problem IS NULL AND r.kind = 'id' THEN + EXECUTE format('SELECT EXISTS (SELECT 1 FROM delta_stage.%I GROUP BY %I HAVING count(*) > 1)', + r.table_name, v_cfg.pk_col) INTO v_dups; + IF v_dups THEN v_problem := 'DUP_STAGED_PK'; END IF; + END IF; + + IF v_problem IS NULL THEN + IF r.kind = 'id' THEN + v_condition := format('s.%I::bigint BETWEEN $2 AND $3', v_cfg.pk_col); + ELSIF r.kind = 'row' THEN + v_condition := 's._delta_row_id BETWEEN $2 AND $3'; + ELSE + v_corp := delta_ctl.corp_col_expr(r.table_name); + v_condition := format('(%s) = $4', v_corp); + END IF; + v_filter := delta_ctl.selection_filter_expr(r.table_name); + EXECUTE format($fmt$ + SELECT count(*) + FROM delta_diff.%I d + JOIN delta_stage.%I s ON s._delta_row_id = d._delta_row_id + WHERE d.class = $1 AND (%s) AND (%s) + $fmt$, delta_ctl.class_table_name(r.table_name), r.table_name, v_condition, v_filter) + INTO v_matched USING r.class, r.value_from, r.value_to, r.corp_num; + IF v_matched = 0 THEN + v_problem := 'NO_MATCH'; + ELSIF r.kind IN ('id', 'row') AND NOT r.is_range AND v_matched > 1 THEN + v_problem := 'SCALAR_MULTI_MATCH'; + END IF; + END IF; + + INSERT INTO delta_ctl.selection_diagnostics( + table_name, class, mode, kind, selector, source_line, matched, problem) + VALUES (r.table_name, r.class, r.mode, r.kind, r.selector, + r.source_line, v_matched, v_problem); + END LOOP; + +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.stamp_selection() +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + r record; + v_filter text; + v_corp text; + v_match text; +BEGIN + PERFORM delta_ctl.verify_row_selection(); + IF EXISTS (SELECT 1 FROM delta_ctl.selection_diagnostics WHERE problem IS NOT NULL) THEN + RAISE EXCEPTION 'SELECTION_INVALID: row selectors failed validation; query delta_ctl.selection_diagnostics'; + END IF; + FOR r IN + SELECT table_name, pk_col + FROM delta_ctl.table_config + WHERE delta_ctl.stage_table_exists(table_name) + ORDER BY load_phase, table_name + LOOP + EXECUTE format('UPDATE delta_diff.%I SET selected = false', delta_ctl.class_table_name(r.table_name)); + v_filter := delta_ctl.selection_filter_expr(r.table_name); + EXECUTE format($fmt$ + UPDATE delta_diff.%I d + SET selected = true + FROM delta_stage.%I s + WHERE d._delta_row_id = s._delta_row_id + AND d.class IN ('NEW', 'CHANGED', 'CHANGED_LOCAL_NEWER') + AND EXISTS ( + SELECT 1 FROM delta_ctl.selection sel + WHERE sel.table_name = %L AND sel.class = d.class + ) + AND (%s) + $fmt$, delta_ctl.class_table_name(r.table_name), r.table_name, r.table_name, v_filter); + + IF EXISTS (SELECT 1 FROM delta_ctl.row_selection rs WHERE rs.table_name = r.table_name) THEN + v_corp := delta_ctl.corp_col_expr(r.table_name); + v_match := '(' + || CASE WHEN r.pk_col IS NULL THEN 'false' + ELSE format('(rs.kind = ''id'' AND s.%I::bigint BETWEEN rs.value_from AND rs.value_to)', r.pk_col) END + || ' OR (rs.kind = ''row'' AND s._delta_row_id BETWEEN rs.value_from AND rs.value_to)' + || CASE WHEN v_corp IS NULL THEN '' + ELSE format(' OR (rs.kind = ''corp'' AND (%s) = rs.corp_num)', v_corp) END + || ')'; + EXECUTE format($fmt$ + UPDATE delta_diff.%I d + SET selected = false + FROM delta_stage.%I s + WHERE d._delta_row_id = s._delta_row_id + AND d.selected + AND ( + ( + EXISTS ( + SELECT 1 FROM delta_ctl.row_selection rs + WHERE rs.table_name = %L AND rs.class = d.class AND rs.mode = 'include' + ) + AND NOT EXISTS ( + SELECT 1 FROM delta_ctl.row_selection rs + WHERE rs.table_name = %L AND rs.class = d.class AND rs.mode = 'include' + AND %s + ) + ) + OR EXISTS ( + SELECT 1 FROM delta_ctl.row_selection rs + WHERE rs.table_name = %L AND rs.class = d.class AND rs.mode = 'exclude' + AND %s + ) + ) + $fmt$, delta_ctl.class_table_name(r.table_name), r.table_name, + r.table_name, r.table_name, v_match, r.table_name, v_match); + END IF; + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.selected_counts() +RETURNS TABLE(table_name text, class text, row_count bigint, load_phase int) +LANGUAGE plpgsql +AS $$ +DECLARE + r record; +BEGIN + FOR r IN + SELECT tc.table_name, tc.load_phase + FROM delta_ctl.table_config tc + WHERE delta_ctl.stage_table_exists(tc.table_name) + ORDER BY tc.load_phase, tc.table_name + LOOP + RETURN QUERY EXECUTE format( + 'SELECT %L::text, class::text, count(*)::bigint, %s::int FROM delta_diff.%I WHERE selected GROUP BY class', + r.table_name, r.load_phase, delta_ctl.class_table_name(r.table_name)); + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.validate_dependencies() +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + c record; + p record; + v_parent_pk text; + v_count bigint; + v_sample_ids text; +BEGIN + TRUNCATE delta_ctl.dependency_violations; + + FOR c IN + SELECT table_name, fk_map + FROM delta_ctl.table_config + WHERE delta_ctl.stage_table_exists(table_name) + LOOP + FOR p IN + SELECT key AS fk_col, value AS parent_table + FROM jsonb_each_text(c.fk_map) + WHERE value NOT LIKE delta_ctl.external_fk_pattern() + LOOP + SELECT pk_col INTO v_parent_pk FROM delta_ctl.table_config WHERE table_name = p.parent_table; + IF v_parent_pk IS NULL + OR NOT delta_ctl.stage_table_exists(p.parent_table) + OR to_regclass(format('delta_diff.%I', delta_ctl.class_table_name(p.parent_table))) IS NULL THEN + CONTINUE; + END IF; + + EXECUTE format($fmt$ + WITH violations AS ( + SELECT COALESCE(cd.staged_pk::text, cd._delta_row_id::text) AS sample_id + FROM delta_diff.%I cd + JOIN delta_stage.%I cs ON cs._delta_row_id = cd._delta_row_id + LEFT JOIN delta_stage.%I ps ON ps.%I = cs.%I + LEFT JOIN delta_diff.%I pd ON pd._delta_row_id = ps._delta_row_id + WHERE cd.selected + AND cs.%I IS NOT NULL + AND ( + pd._delta_row_id IS NULL + OR pd.class IN ('BLOCKED_FK', 'AMBIGUOUS_NK', 'BLOCKED_PARENT') + OR (pd.class = 'NEW' AND NOT pd.selected) + ) + ) + SELECT count(*), + (SELECT string_agg(sample_id, ',' ORDER BY sample_id) + FROM (SELECT sample_id FROM violations ORDER BY sample_id LIMIT 10) q) + FROM violations + $fmt$, delta_ctl.class_table_name(c.table_name), c.table_name, p.parent_table, v_parent_pk, p.fk_col, + delta_ctl.class_table_name(p.parent_table), p.fk_col) + INTO v_count, v_sample_ids; + + IF v_count > 0 THEN + INSERT INTO delta_ctl.dependency_violations(child_table, parent_table, reason, row_count, sample_ids) + VALUES (c.table_name, p.parent_table, + format('selected child rows require selected/non-blocked parent via %I', p.fk_col), + v_count, v_sample_ids); + END IF; + END LOOP; + END LOOP; + +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.insert_expr(p_table text, p_col text) +RETURNS text +LANGUAGE plpgsql +AS $$ +DECLARE + v_cfg delta_ctl.table_config%ROWTYPE; + v_parent text; +BEGIN + SELECT * INTO v_cfg FROM delta_ctl.table_config WHERE table_name = p_table; + IF v_cfg.pk_col IS NOT NULL AND p_col = v_cfg.pk_col THEN + RETURN 'm.local_id'; + END IF; + + SELECT value INTO v_parent + FROM delta_ctl.table_config c, jsonb_each_text(c.fk_map) + WHERE c.table_name = p_table + AND key = p_col + AND value NOT LIKE delta_ctl.external_fk_pattern(); + + IF v_parent IS NOT NULL THEN + RETURN delta_ctl.map_fk_expr(v_parent, p_col); + END IF; + + RETURN format('s.%I', p_col); +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.update_columns(p_table text) +RETURNS text[] +LANGUAGE plpgsql +AS $$ +DECLARE + v_cfg delta_ctl.table_config%ROWTYPE; + v_cols text[]; +BEGIN + SELECT c.table_name, + c.load_phase, + c.pk_col, + c.nk_exprs, + c.nk_stage_exprs, + c.nk_local_exprs, + c.nk_cols, + c.nk_enforced, + c.fk_map, + c.has_last_modified, + c.match_mode, + c.classify_ignore_cols, + c.compare_ignore_cols + INTO v_cfg + FROM delta_ctl.table_config c + WHERE c.table_name = p_table; + + SELECT COALESCE(array_agg(a.attname ORDER BY a.attnum), '{}') + INTO v_cols + FROM pg_attribute a + WHERE a.attrelid = delta_ctl.public_rel(p_table)::regclass + AND a.attnum > 0 + AND NOT a.attisdropped + AND (v_cfg.pk_col IS NULL OR a.attname <> v_cfg.pk_col) + AND NOT (a.attname = ANY(v_cfg.nk_cols)); + + RETURN v_cols; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.allocate_ids(p_table text) +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + v_cfg delta_ctl.table_config%ROWTYPE; + v_seq text; + v_sql text; + v_missing bigint; +BEGIN + SELECT * INTO v_cfg FROM delta_ctl.table_config WHERE table_name = p_table; + IF NOT FOUND OR v_cfg.pk_col IS NULL OR NOT delta_ctl.stage_table_exists(p_table) THEN + RETURN; + END IF; + + v_seq := pg_get_serial_sequence(delta_ctl.public_rel(p_table), v_cfg.pk_col); + IF v_seq IS NULL THEN + RETURN; + END IF; + + v_sql := format($fmt$ + SELECT setval(%L, GREATEST( + (SELECT last_value FROM %s), + (SELECT COALESCE(max(%I), 0) FROM public.%I), + (SELECT COALESCE(max(local_id), 0) FROM delta_map.%I), + 1 + ), true) + $fmt$, v_seq, v_seq::regclass, v_cfg.pk_col, p_table, delta_ctl.map_table_name(p_table)); + EXECUTE v_sql; + + EXECUTE format($fmt$ + UPDATE delta_map.%I m + SET local_id = nextval(%L) + FROM delta_stage.%I s + JOIN delta_diff.%I d ON d._delta_row_id = s._delta_row_id + WHERE m.staged_id = s.%I + AND m.disposition = 'NEW_REALLOCATED' + AND d.class = 'NEW' + AND d.selected + $fmt$, delta_ctl.map_table_name(p_table), v_seq, p_table, delta_ctl.class_table_name(p_table), v_cfg.pk_col); + + EXECUTE format($fmt$ + SELECT count(*) + FROM delta_stage.%I s + JOIN delta_diff.%I d ON d._delta_row_id = s._delta_row_id + JOIN delta_map.%I m ON m.staged_id = s.%I + WHERE d.class = 'NEW' AND d.selected AND m.local_id IS NULL + $fmt$, p_table, delta_ctl.class_table_name(p_table), delta_ctl.map_table_name(p_table), v_cfg.pk_col) + INTO v_missing; + + IF v_missing > 0 THEN + RAISE EXCEPTION 'APPLY_VERIFICATION_FAILED: selected NEW rows in % lack allocated local IDs (%)', p_table, v_missing; + END IF; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.apply_table(p_table text) +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + v_cfg delta_ctl.table_config%ROWTYPE; + v_cols text[]; + v_insert_cols text; + v_insert_exprs text; + v_update_cols text[]; + v_set_list text; + v_join_map text := ''; + v_where_target text; + v_expected bigint; + v_affected bigint; + v_class text; +BEGIN + SELECT * INTO v_cfg FROM delta_ctl.table_config WHERE table_name = p_table; + IF NOT FOUND OR NOT delta_ctl.stage_table_exists(p_table) THEN + RETURN; + END IF; + + PERFORM delta_ctl.allocate_ids(p_table); + + SELECT COALESCE(array_agg(a.attname ORDER BY a.attnum), '{}') INTO v_cols + FROM pg_attribute a + WHERE a.attrelid = delta_ctl.public_rel(p_table)::regclass + AND a.attnum > 0 + AND NOT a.attisdropped; + + v_insert_cols := array_to_string(ARRAY(SELECT format('%I', c) FROM unnest(v_cols) AS c), ', '); + v_insert_exprs := array_to_string(ARRAY(SELECT delta_ctl.insert_expr(p_table, c) FROM unnest(v_cols) AS c), ', '); + IF v_cfg.pk_col IS NOT NULL THEN + v_join_map := format('JOIN delta_map.%I m ON m.staged_id = s.%I', delta_ctl.map_table_name(p_table), v_cfg.pk_col); + END IF; + + EXECUTE format('SELECT count(*) FROM delta_diff.%I WHERE class = ''NEW'' AND selected', delta_ctl.class_table_name(p_table)) INTO v_expected; + EXECUTE format($fmt$ + INSERT INTO public.%I (%s) + SELECT %s + FROM delta_stage.%I s + JOIN delta_diff.%I d ON d._delta_row_id = s._delta_row_id + %s + WHERE d.class = 'NEW' AND d.selected + $fmt$, p_table, v_insert_cols, v_insert_exprs, p_table, delta_ctl.class_table_name(p_table), v_join_map); + GET DIAGNOSTICS v_affected = ROW_COUNT; + INSERT INTO delta_ctl.apply_counts(table_name, class, action, expected_count, affected_count) + VALUES (p_table, 'NEW', 'INSERT', v_expected, v_affected) + ON CONFLICT (table_name, class, action) DO UPDATE + SET expected_count = EXCLUDED.expected_count, affected_count = EXCLUDED.affected_count; + IF v_expected <> v_affected THEN + RAISE EXCEPTION 'APPLY_VERIFICATION_FAILED: %.NEW insert expected %, affected %', p_table, v_expected, v_affected; + END IF; + + IF v_affected > 0 THEN + INSERT INTO delta_ctl.touched_tables(table_name) VALUES (p_table) ON CONFLICT DO NOTHING; + END IF; + + IF p_table = delta_ctl.auth_component_op_table() THEN + RETURN; + END IF; + + v_update_cols := delta_ctl.update_columns(p_table); + IF array_length(v_update_cols, 1) IS NULL THEN + RETURN; + END IF; + + v_set_list := array_to_string(ARRAY( + SELECT format('%I = %s', c, delta_ctl.insert_expr(p_table, c)) + FROM unnest(v_update_cols) AS c + ), ', '); + v_where_target := CASE WHEN v_cfg.pk_col IS NULL THEN 'l.ctid = d.local_ctid' ELSE format('l.%I = d.local_pk', v_cfg.pk_col) END; + + FOREACH v_class IN ARRAY ARRAY['CHANGED', 'CHANGED_LOCAL_NEWER'] LOOP + EXECUTE format('SELECT count(*) FROM delta_diff.%I WHERE class = %L AND selected', delta_ctl.class_table_name(p_table), v_class) INTO v_expected; + EXECUTE format($fmt$ + UPDATE public.%I l + SET %s + FROM delta_stage.%I s + JOIN delta_diff.%I d ON d._delta_row_id = s._delta_row_id + WHERE d.class = %L AND d.selected AND %s + $fmt$, p_table, v_set_list, p_table, delta_ctl.class_table_name(p_table), v_class, v_where_target); + GET DIAGNOSTICS v_affected = ROW_COUNT; + INSERT INTO delta_ctl.apply_counts(table_name, class, action, expected_count, affected_count) + VALUES (p_table, v_class, 'UPDATE', v_expected, v_affected) + ON CONFLICT (table_name, class, action) DO UPDATE + SET expected_count = EXCLUDED.expected_count, affected_count = EXCLUDED.affected_count; + IF v_expected <> v_affected THEN + RAISE EXCEPTION 'APPLY_VERIFICATION_FAILED: %.% update expected %, affected %', p_table, v_class, v_expected, v_affected; + END IF; + IF v_affected > 0 THEN + INSERT INTO delta_ctl.touched_tables(table_name) VALUES (p_table) ON CONFLICT DO NOTHING; + END IF; + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.run_apply() +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + r record; +BEGIN + PERFORM delta_ctl.run_preview_classification(); + PERFORM delta_ctl.stamp_selection(); + PERFORM delta_ctl.validate_dependencies(); + IF EXISTS (SELECT 1 FROM delta_ctl.dependency_violations) THEN + RAISE EXCEPTION 'SELECTION_INVALID: selected child rows have blocked, missing, or unselected NEW preserved parents. Query delta_ctl.dependency_violations for details.'; + END IF; + + TRUNCATE delta_ctl.apply_counts; + TRUNCATE delta_ctl.touched_tables; + + FOR r IN + SELECT table_name + FROM delta_ctl.table_config + WHERE delta_ctl.stage_table_exists(table_name) + ORDER BY load_phase, table_name + LOOP + PERFORM delta_ctl.apply_table(r.table_name); + END LOOP; +END; +$$; + +CREATE OR REPLACE FUNCTION delta_ctl.render_apply_summary_lines() +RETURNS SETOF text +LANGUAGE plpgsql +AS $$ +DECLARE + r record; + m record; + c_table_hdr CONSTANT text := delta_ctl.table_kind(); + c_summary_row_format CONSTANT text := '%-36s %-20s %10s'; +BEGIN + RETURN NEXT 'Apply summary'; + RETURN NEXT '============='; + RETURN NEXT ''; + + RETURN NEXT 'Selected counts'; + RETURN NEXT '---------------'; + RETURN NEXT format(c_summary_row_format, c_table_hdr, delta_ctl.class_header(), 'selected'); + FOR r IN + SELECT table_name, class, row_count, load_phase + FROM delta_ctl.selected_counts() + ORDER BY load_phase, table_name, class + LOOP + RETURN NEXT format(c_summary_row_format, r.table_name, r.class, r.row_count); + END LOOP; + + RETURN NEXT ''; + RETURN NEXT 'Affected counts'; + RETURN NEXT '---------------'; + RETURN NEXT format('%-36s %-20s %-8s %10s %10s', c_table_hdr, delta_ctl.class_header(), 'action', 'expected', 'affected'); + FOR r IN + SELECT table_name, class, action, expected_count, affected_count + FROM delta_ctl.apply_counts + ORDER BY table_name, class, action + LOOP + RETURN NEXT format('%-36s %-20s %-8s %10s %10s', r.table_name, r.class, r.action, r.expected_count, r.affected_count); + END LOOP; + + RETURN NEXT ''; + RETURN NEXT 'ID map dispositions'; + RETURN NEXT '-------------------'; + FOR r IN + SELECT table_name + FROM delta_ctl.table_config + WHERE delta_ctl.map_table_exists(table_name) + ORDER BY load_phase, table_name + LOOP + FOR m IN EXECUTE format( + 'SELECT disposition, count(*)::bigint AS n FROM delta_map.%I GROUP BY disposition ORDER BY disposition', + delta_ctl.map_table_name(r.table_name)) + LOOP + RETURN NEXT format(c_summary_row_format, r.table_name, m.disposition, m.n); + END LOOP; + END LOOP; + + RETURN NEXT ''; + RETURN NEXT 'Blocked/skipped counts'; + RETURN NEXT '----------------------'; + FOR r IN + SELECT table_name, count_name, row_count + FROM delta_ctl.run_counts + WHERE count_name IN ('BLOCKED_FK', 'BLOCKED_PARENT', 'AMBIGUOUS_NK', 'SKIPPED_ABSENT', 'LOCAL_ONLY') + AND row_count > 0 + ORDER BY table_name, count_name + LOOP + RETURN NEXT format(c_summary_row_format, r.table_name, r.count_name, r.row_count); + END LOOP; + + RETURN NEXT ''; + RETURN NEXT 'Touched tables'; + RETURN NEXT '--------------'; + FOR r IN SELECT table_name FROM delta_ctl.touched_tables ORDER BY table_name LOOP + RETURN NEXT '- ' || r.table_name; + END LOOP; +END; +$$; diff --git a/data-tool/scripts/restore/delta/20_session_begin.sql b/data-tool/scripts/restore/delta/20_session_begin.sql new file mode 100644 index 0000000000..c939975f32 --- /dev/null +++ b/data-tool/scripts/restore/delta/20_session_begin.sql @@ -0,0 +1,8 @@ +-- Delta restore staging/apply session settings. +-- These settings are scoped to the current psql session/transaction. + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET synchronous_commit = off; +SET work_mem = '512MB'; diff --git a/data-tool/scripts/restore/delta/21_session_end.sql b/data-tool/scripts/restore/delta/21_session_end.sql new file mode 100644 index 0000000000..272b8e512f --- /dev/null +++ b/data-tool/scripts/restore/delta/21_session_end.sql @@ -0,0 +1,7 @@ +-- Reset delta restore session settings. + +SET work_mem TO DEFAULT; +SET synchronous_commit TO DEFAULT; +SET statement_timeout TO DEFAULT; +SET lock_timeout TO DEFAULT; +SET idle_in_transaction_session_timeout TO DEFAULT; diff --git a/data-tool/scripts/restore/delta/align_details.awk b/data-tool/scripts/restore/delta/align_details.awk new file mode 100644 index 0000000000..5f2d8f30a3 --- /dev/null +++ b/data-tool/scripts/restore/delta/align_details.awk @@ -0,0 +1,75 @@ +# Produce a fixed-width companion for a canonical detail TSV. +# +# Invoke with the same non-empty input path twice: +# awk -v max_width=40 -f align_details.awk details.tsv details.tsv +# +# The first pass measures column widths; the second pass renders the output. +# Comment lines are not included in width calculations and pass through verbatim. + +BEGIN { + FS = "\t" + if (max_width == "" || max_width !~ /^[0-9]+$/ || max_width < 1) { + max_width = 40 + } +} + +NR == FNR { + if ($0 ~ /^#/) { + next + } + + if (NF > column_count) { + column_count = NF + } + for (i = 1; i <= NF; i++) { + cell_width = length($i) + if (cell_width > max_width) { + cell_width = max_width + } + if (cell_width > widths[i]) { + widths[i] = cell_width + } + } + next +} + +function clipped(value) { + if (length(value) <= max_width) { + return value + } + return substr(value, 1, max_width - 1) "…" +} + +function dashes(count, result, i) { + result = "" + for (i = 0; i < count; i++) { + result = result "-" + } + return result +} + +function render_fields(underline, result, value, i) { + result = "" + for (i = 1; i <= column_count; i++) { + value = underline ? dashes(widths[i]) : clipped(i <= NF ? $i : "") + if (i < column_count) { + result = result sprintf("%-" widths[i] "s", value) " " + } else { + result = result value + } + } + print result +} + +$0 ~ /^#/ { + print + next +} + +{ + render_fields(0) + if (!header_written) { + render_fields(1) + header_written = 1 + } +} diff --git a/data-tool/scripts/restore/delta/rewrite_copy_targets.awk b/data-tool/scripts/restore/delta/rewrite_copy_targets.awk new file mode 100644 index 0000000000..5ac6d66886 --- /dev/null +++ b/data-tool/scripts/restore/delta/rewrite_copy_targets.awk @@ -0,0 +1,129 @@ +# Rewrite pg_restore COPY targets from public.
to delta_stage.
. +# Also captures the dump column list as: tablecol1,col2,... +# +# Contract: +# - expected= contains one expected table name per line. +# - sidecar= is overwritten with captured columns. +# - COPY headers must look like: COPY public. (cols) FROM stdin; +# - Every expected table must be seen exactly once. +# - Client-version-only SET commands unsupported by older targets may be +# neutralized outside COPY data only. + +function fail(message) { + fatal = 1 + print "rewrite_copy_targets.awk: " message > "/dev/stderr" + print "SELECT 1/0 AS rewrite_copy_targets_failed;" + exit 2 +} + +function trim(value) { + gsub(/^[[:space:]]+/, "", value) + gsub(/[[:space:]]+$/, "", value) + return value +} + +BEGIN { + state = "OUTSIDE" + + if (expected == "") { + fail("missing -v expected=") + } + if (sidecar == "") { + fail("missing -v sidecar=") + } + + while ((getline table < expected) > 0) { + table = trim(table) + if (table == "" || table ~ /^#/) { + continue + } + if (table !~ /^[a-z_][a-z0-9_]*$/) { + fail("invalid expected table identifier: " table) + } + if (wanted[table]) { + fail("duplicate expected table: " table) + } + wanted[table] = 1 + expected_count++ + } + close(expected) + + # Truncate the sidecar before the first capture. + printf "" > sidecar + close(sidecar) +} + +{ + line = $0 + + if (state == "OUTSIDE") { + # pg_dump/pg_restore from newer PostgreSQL clients can emit this SET, but + # older target servers reject the parameter. It is only a timeout guard, and + # delta session SQL already disables relevant timeouts, so skip it outside + # COPY data while leaving payload rows untouched. + if (line ~ /^SET transaction_timeout = /) { + print "-- skipped unsupported client-only setting: " line + next + } + + if (line ~ /^COPY public\./) { + if (line !~ /^COPY public\.[a-z_][a-z0-9_]*[[:space:]]*\(.+\) FROM stdin;$/) { + fail("unexpected COPY header form: " line) + } + + table = line + sub(/^COPY public\./, "", table) + sub(/[[:space:]]*\(.*/, "", table) + + if (!wanted[table]) { + fail("COPY header table is not expected: " table) + } + if (seen[table]) { + fail("duplicate COPY stream for table: " table) + } + seen[table] = 1 + seen_count++ + + cols = line + sub(/^COPY public\.[a-z_][a-z0-9_]*[[:space:]]*\(/, "", cols) + sub(/\) FROM stdin;$/, "", cols) + gsub(/[[:space:]]*,[[:space:]]*/, ",", cols) + cols = trim(cols) + + print table "\t" cols >> sidecar + close(sidecar) + + sub(/^COPY public\./, "COPY delta_stage.", line) + print line + state = "INSIDE" + next + } + + print line + next + } + + print line + if (line == "\\.") { + state = "OUTSIDE" + } +} + +END { + if (fatal) { + exit 2 + } + if (state != "OUTSIDE") { + print "rewrite_copy_targets.awk: input ended inside COPY data" > "/dev/stderr" + exit 2 + } + if (seen_count != expected_count) { + for (table in wanted) { + if (!seen[table]) { + print "rewrite_copy_targets.awk: expected table was not seen: " table > "/dev/stderr" + } + } + print "SELECT 1/0 AS rewrite_copy_targets_missing_expected_table;" + exit 2 + } +} diff --git a/data-tool/scripts/restore/preserved_tables.conf b/data-tool/scripts/restore/preserved_tables.conf new file mode 100644 index 0000000000..bb49dcf734 --- /dev/null +++ b/data-tool/scripts/restore/preserved_tables.conf @@ -0,0 +1,18 @@ +# table_name load_phase +mig_group 20 +mig_batch 30 +mig_corp_batch 40 +mig_corp_account 40 +corp_processing 50 +colin_tracking 50 +auth_processing 50 +auth_component_operation 60 +email_domain_groups 10 +bad_emails 10 +excluded_emails 10 +excluded_email_domains 10 +excluded_email_domain_patterns 10 +exclude_corps 10 +corps_with_third_party 10 +bar_corps 10 +demigrated_filings 10 diff --git a/data-tool/scripts/restore/repair_sequences.sql b/data-tool/scripts/restore/repair_sequences.sql new file mode 100644 index 0000000000..b617190d37 --- /dev/null +++ b/data-tool/scripts/restore/repair_sequences.sql @@ -0,0 +1,43 @@ +DO $$ +DECLARE + r record; + max_val bigint; + sch text := current_schema(); +BEGIN + FOR r IN + -- For SERIAL/IDENTITY columns (auto-dependency from sequence to column) + SELECT seq.relname AS seq, + tbl.relname AS tbl, + col.attname AS col + FROM pg_class seq + JOIN pg_depend d ON d.objid = seq.oid AND d.deptype = 'a' + JOIN pg_class tbl ON tbl.oid = d.refobjid + JOIN pg_attribute col + ON col.attrelid = tbl.oid AND col.attnum = d.refobjsubid + WHERE seq.relkind = 'S' + AND tbl.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = sch) + UNION ALL + -- For columns with DEFAULT nextval('sequence') (normal dependency from column default to sequence) + SELECT + seq.relname as seq, + tbl.relname as tbl, + col.attname as col + FROM pg_depend d + JOIN pg_class seq ON seq.oid = d.refobjid AND seq.relkind = 'S' + JOIN pg_attrdef ad ON ad.oid = d.objid AND d.classid = 'pg_attrdef'::regclass + JOIN pg_attribute col ON col.attrelid = ad.adrelid AND col.attnum = ad.adnum + JOIN pg_class tbl ON tbl.oid = ad.adrelid + WHERE d.deptype = 'n' + AND tbl.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = sch) + LOOP + EXECUTE format('SELECT MAX(%I) FROM %I.%I', r.col, sch, r.tbl) INTO max_val; + IF max_val IS NULL THEN + -- Table is empty, reset sequence to start at 1. The next call to nextval() will return 1. + EXECUTE format('SELECT setval(%L, 1, false);', format('%I.%I', sch, r.seq)); + ELSE + -- Table has data, set sequence so nextval() returns max_val + 1. + EXECUTE format('SELECT setval(%L, %s);', format('%I.%I', sch, r.seq), max_val); + END IF; + END LOOP; +END; +$$; diff --git a/data-tool/scripts/subset/subset_pg_call_address_transpose.sql b/data-tool/scripts/subset/subset_pg_call_address_transpose.sql new file mode 100644 index 0000000000..8bb6b3edd2 --- /dev/null +++ b/data-tool/scripts/subset/subset_pg_call_address_transpose.sql @@ -0,0 +1,11 @@ +-- Invoke the address-transpose orchestrator once at the end of a subset +-- load/refresh while the shared advisory lock is still held. +-- +-- IMPORTANT (DbSchemaCLI compatibility): +-- - Plain statements only; no DO $$ blocks, dollar quoting, or client scripting. +-- - The phase procedures COMMIT internally, so CALL must be top-level. +-- - This terminal result-bearing CALL returns one JSON value with phase counts, their total, and elapsed seconds. + +SET search_path TO public; + +CALL public.colin_address_transpose(NULL); diff --git a/data-tool/scripts/transfer_cprd_corps.sql b/data-tool/scripts/transfer_cprd_corps.sql index 23537e36ae..596df08fd7 100644 --- a/data-tool/scripts/transfer_cprd_corps.sql +++ b/data-tool/scripts/transfer_cprd_corps.sql @@ -1536,6 +1536,11 @@ ALTER TABLE notification ENABLE TRIGGER ALL; ALTER TABLE notification_resend ENABLE TRIGGER ALL; ALTER TABLE party_notification ENABLE TRIGGER ALL; +-- Normalize active-corp party/office addresses while the subset-run advisory lock is held. +-- The result-bearing CALL returns one JSON value with phase counts, their total, and elapsed seconds. +SET search_path TO public; +CALL public.colin_address_transpose(NULL); + -- Release COLIN extract refresh advisory lock. SELECT pg_advisory_unlock( hashtext('lear:data-tool:colin_subset_extract'), diff --git a/data-tool/scripts/transfer_cprd_lib_only.sql b/data-tool/scripts/transfer_cprd_lib_only.sql new file mode 100644 index 0000000000..930b419543 --- /dev/null +++ b/data-tool/scripts/transfer_cprd_lib_only.sql @@ -0,0 +1,1525 @@ +vset cli.settings.ignore_errors=false +vset cli.settings.transfer_threads=4 +vset format.date=YYYY-MM-dd'T'hh:mm:ss'Z' +vset format.timestamp=YYYY-MM-dd'T'hh:mm:ss'Z' + +connect cprd_pg; + +-- Serialize COLIN extract refreshes on this target DB so subset/full runs cannot overlap. +SELECT pg_advisory_lock( + hashtext('lear:data-tool:colin_subset_extract'), + hashtext('subset_run') +); + +learn schema public; + +-- cutoff timestamp +insert into public.colin_extract_version (extracted_at) +values (current_timestamp); + + + +-- disable triggers +ALTER TABLE corporation DISABLE TRIGGER ALL; +ALTER TABLE corp_name DISABLE TRIGGER ALL; +ALTER TABLE corp_state DISABLE TRIGGER ALL; +ALTER TABLE event DISABLE TRIGGER ALL; +ALTER TABLE filing DISABLE TRIGGER ALL; +ALTER TABLE filing_user DISABLE TRIGGER ALL; +ALTER TABLE office DISABLE TRIGGER ALL; +ALTER TABLE address DISABLE TRIGGER ALL; +ALTER TABLE corp_comments DISABLE TRIGGER ALL; +ALTER TABLE ledger_text DISABLE TRIGGER ALL; +ALTER TABLE corp_party DISABLE TRIGGER ALL; +ALTER TABLE corp_party_relationship DISABLE TRIGGER ALL; +ALTER TABLE offices_held DISABLE TRIGGER ALL; +ALTER TABLE completing_party DISABLE TRIGGER ALL; +ALTER TABLE submitting_party DISABLE TRIGGER ALL; +ALTER TABLE corp_flag DISABLE TRIGGER ALL; +ALTER TABLE cont_out DISABLE TRIGGER ALL; +ALTER TABLE conv_event DISABLE TRIGGER ALL; +ALTER TABLE conv_ledger DISABLE TRIGGER ALL; +ALTER TABLE corp_involved_amalgamating DISABLE TRIGGER ALL; +ALTER TABLE corp_involved_cont_in DISABLE TRIGGER ALL; +ALTER TABLE corp_restriction DISABLE TRIGGER ALL; +ALTER TABLE correction DISABLE TRIGGER ALL; +ALTER TABLE jurisdiction DISABLE TRIGGER ALL; +ALTER TABLE resolution DISABLE TRIGGER ALL; +ALTER TABLE share_series DISABLE TRIGGER ALL; +ALTER TABLE share_struct DISABLE TRIGGER ALL; +ALTER TABLE share_struct_cls DISABLE TRIGGER ALL; +ALTER TABLE notification DISABLE TRIGGER ALL; +ALTER TABLE notification_resend DISABLE TRIGGER ALL; +ALTER TABLE party_notification DISABLE TRIGGER ALL; + + +-- alter tables +alter table corporation alter column send_ar_ind type integer using send_ar_ind::integer; +alter table filing + alter column arrangement_ind type integer using arrangement_ind::integer, + alter column court_appr_ind type integer using court_appr_ind::integer; +alter table share_struct_cls + alter column max_share_ind type integer using max_share_ind::integer, + alter column spec_rights_ind type integer using spec_rights_ind::integer, + alter column par_value_ind type integer using par_value_ind::integer; +alter table conv_event alter report_corp_ind type integer using report_corp_ind::integer; +alter table corp_involved_amalgamating alter adopted_corp_ind type integer using adopted_corp_ind::integer; +alter table corp_restriction alter restriction_ind type integer using restriction_ind::integer; +alter table share_series + alter column max_share_ind type integer using max_share_ind::integer, + alter column spec_right_ind type integer using spec_right_ind::integer; + + +-- corporation +transfer public.corporation from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end CORP_NUM, + CORP_FROZEN_TYP_CD as corp_frozen_type_cd, + case + when c.CORP_TYP_CD in ('QA', 'QB', 'QC', 'QD', 'QE') then 'BC' + else c.CORP_TYP_CD + end CORP_TYPE_CD, + CORP_PASSWORD, + RECOGNITION_DTS, + BN_9, + bn_15, + admin_email, + ACCESSION_NUM, + LAST_AR_FILED_DT, + case SEND_AR_IND + when 'N' then 0 + when 'Y' then 1 + else 1 + end SEND_AR_IND, + (select + to_number(to_char(max(date_1), 'YYYY')) + from eml_log e, rep_data r + where + e.corp_num=c.corp_num + and e.param_id=r.param_id + and e.corp_num=r.t20_1) as LAST_AR_REMINDER_YEAR +from corporation_cte c +order by corp_num; + + + +-- event +transfer public.event from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select e.event_id, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end CORP_NUM, + e.event_typ_cd as event_type_cd, + e.event_timestmp as event_timerstamp, + e.trigger_dts +from event e + , corporation_cte c +where c.corp_num = e.corp_num + and event_typ_cd not in ('BNUPD','ADDLEDGR') --not doing anything +order by e.event_id desc; + + + +-- corp_name +transfer public.corp_name from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end CORP_NUM, + cn.CORP_NAME_TYP_CD, + cn.start_event_id, + cn.end_event_id, + cn.CORP_NME as corp_name +from CORP_NAME cn + , corporation_cte c +where cn.corp_num = c.corp_num +order by c.corp_num, start_event_id; + + + +-- corp_state +transfer public.corp_state from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end CORP_NUM, + cs.STATE_TYP_CD as state_type_cd, + cos.op_state_typ_cd as op_state_type_cd, + cs.start_event_id, + cs.end_event_id +from CORP_STATE cs + , corporation_cte c + , corp_op_state cos +where cs.corp_num = c.corp_num + and cos.state_typ_cd = cs.state_typ_cd +order by c.corp_num, start_event_id; + + + +-- filing +transfer public.filing from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select e.event_id, + f.filing_typ_cd as filing_type_cd, + f.effective_dt, + f.withdrawn_event_id, + trim(f.ods_typ_cd) as ods_type_cd, + f.nr_num, + f.COURT_ORDER_NUM, + f.CHANGE_DT, + f.PERIOD_END_DT, + case f.ARRANGEMENT_IND + when 'N' then 0 + when 'Y' then 1 + else 1 + end ARRANGEMENT_IND, + f.AUTH_SIGN_DT, + case f.COURT_APPR_IND + when 'N' then 0 + when 'Y' then 1 + else 1 + end COURT_APPR_IND +from event e + , filing f + , corporation_cte c +where e.event_id = f.event_id + and c.corp_num = e.corp_num +order by e.event_id; + + + +-- filing_user +transfer public.filing_user from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select e.event_id, + u.user_id, + u.last_nme as last_name, + u.first_nme as first_name, + u.middle_nme as middle_name, + u.email_addr, + u.BCOL_ACCT_NUM, + u.ROLE_TYP_CD +from event e + , filing_user u + , corporation_cte c +where e.event_id = u.event_id + and c.corp_num = e.corp_num +order by e.event_id; + + + +-- office +transfer public.office from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end CORP_NUM, + o.office_typ_cd, + o.start_event_id, + o.end_event_id, + o.mailing_addr_id, + o.delivery_addr_id +from office o + , corporation_cte c +where o.corp_num = c.corp_num +order by c.corp_num, start_event_id; + + +vset cli.settings.transfer_threads=3 +-- address +transfer public.address from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select distinct ADDR_ID, + trim(PROVINCE) as PROVINCE, + trim(COUNTRY_TYP_CD) as COUNTRY_TYP_CD, + trim(replace(POSTAL_CD, CHR(0), '')) as POSTAL_CD, + trim(ADDR_LINE_1) as ADDR_LINE_1, + trim(replace(ADDR_LINE_2, CHR(0), '')) as ADDR_LINE_2, + trim(ADDR_LINE_3) as ADDR_LINE_3, + trim(CITY) as CITY +from (select a.* + from address a + , corp_party x + , corporation_cte c + where x.corp_num = c.corp_num + and (delivery_addr_id = a.addr_id or mailing_addr_id = a.addr_id) + UNION ALL + select a.* + from address a + , office x + , corporation_cte c + where x.corp_num = c.corp_num + and (delivery_addr_id = a.addr_id or mailing_addr_id = a.addr_id) + UNION ALL + select a.* + from address a + , completing_party x + , corporation_cte c + , event e + where x.event_id = e.event_id + and e.corp_num = c.corp_num + and mailing_addr_id = a.addr_id + UNION ALL + select a.* + from address a + , notification x + , corporation_cte c + , event e + where x.event_id = e.event_id + and e.corp_num = c.corp_num + and mailing_addr_id = a.addr_id + ) +order by addr_id; + + +vset cli.settings.transfer_threads=4 +-- corp_comments +transfer public.corp_comments from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select cc.comment_dts, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end CORP_NUM, + cc.comments, + cc.USER_ID, + cc.FIRST_NME, + cc.LAST_NME, + cc.MIDDLE_NME, + cc.ACCESSION_COMMENTS +from corp_comments cc + , corporation_cte c +where c.corp_num = cc.corp_num +order by c.corp_num, comment_dts; + + + +-- ledger_text +transfer public.ledger_text from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select e.event_id, + l.notation, + l.USER_ID, + l.LEDGER_TEXT_DTS +from ledger_text l + , event e + , corporation_cte c +where e.event_id = l.event_id + and c.corp_num = e.corp_num +order by e.event_id; + + + +-- corp_party +transfer public.corp_party from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select p.corp_party_id, + p.mailing_addr_id, + p.delivery_addr_id, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end CORP_NUM, + nvl(p.party_typ_cd, ' ') as party_typ_cd, + p.start_event_id, + p.end_event_id, + p.prev_party_id, + p.appointment_dt, + p.cessation_dt, + nvl(p.LAST_NME, ' ') as last_name, + nvl(p.MIDDLE_NME, ' ') as middle_name, + nvl(p.FIRST_NME, ' ') as first_name, + nvl(p.BUSINESS_NME, ' ') as business_name, + p.BUS_COMPANY_NUM, + p.CORR_TYP_CD, + p.OFFICE_NOTIFICATION_DT +from corp_party p + , corporation_cte c +where p.corp_num = c.corp_num +order by c.corp_num, corp_party_id; + + + +-- corp_party_relationship +transfer public.CORP_PARTY_RELATIONSHIP from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select cpr.CORP_PARTY_ID, cpr.RELATIONSHIP_TYP_CD +from CORP_PARTY_RELATIONSHIP cpr, + corp_party p + , + corporation_cte c +where p.corp_num = c.corp_num + and cpr.corp_party_id = p.corp_party_id +order by c.corp_num, corp_party_id; + + + +-- offices_held +transfer public.OFFICES_HELD from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select oh.CORP_PARTY_ID, oh.OFFICER_TYP_CD +from OFFICES_HELD oh, + corp_party p + , + corporation_cte c +where p.corp_num = c.corp_num + and oh.corp_party_id = p.corp_party_id +order by c.corp_num, corp_party_id; + + + +-- completing_party +transfer public.completing_party from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select e.event_id, + cp.MAILING_ADDR_ID, + cp.FIRST_NME, + cp.LAST_NME, + cp.MIDDLE_NME, + cp.EMAIL_REQ_ADDRESS +from event e + , completing_party cp + , corporation_cte c +where e.event_id = cp.event_id + and c.corp_num = e.corp_num +order by e.event_id; + + + +-- submitting_party +-- transfer public.SUBMITTING_PARTY from cprd using +-- with corporation_cte as ( +-- select c.* +-- from corporation c +-- where c.CORP_NUM not in ( +-- select c.corp_num +-- from corporation c +-- , event e +-- , filing f +-- , filing_user u +-- where e.CORP_NUM = c.CORP_NUM +-- and f.event_id = e.event_id +-- and u.event_id = e.event_id +-- and u.user_id = 'BCOMPS' +-- and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') +-- ) +-- and c.CORP_TYP_CD in ('LIB') + +-- -- altered from BC to BEN then BEN to BC before directed launch +-- and c.CORP_NUM not in ('0460007', '1255957', '1186381') +-- ) +-- select e.event_id, +-- sp.MAILING_ADDR_ID, +-- sp.NOTIFY_ADDR_ID, +-- sp.METHOD_TYP_CD, +-- sp.FIRST_NME, +-- sp.LAST_NME, +-- sp.MIDDLE_NME, +-- sp.EMAIL_REQ_ADDRESS, +-- sp.PICKUP_BY, +-- sp.BUSINESS_NME, +-- sp.NOTIFY_FIRST_NME, +-- sp.NOTIFY_LAST_NME, +-- sp.NOTIFY_MIDDLE_NME, +-- sp.PHONE_NUMBER +-- from event e +-- , SUBMITTING_PARTY sp +-- , corporation_cte c +-- where e.event_id = sp.event_id +-- and c.corp_num = e.corp_num +-- order by e.event_id; + + + +-- corp_flag +transfer public.corp_flag from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end CORP_NUM, + cf.CORP_FLAG_TYPE_CD, + cf.start_event_id, + cf.end_event_id +from corp_flag cf + , corporation_cte c +where cf.corp_num = c.corp_num +order by c.corp_num, cf.start_event_id; + + + +-- cont_out +-- transfer public.CONT_OUT from cprd using +-- with corporation_cte as ( +-- select c.* +-- from corporation c +-- where c.CORP_NUM not in ( +-- select c.corp_num +-- from corporation c +-- , event e +-- , filing f +-- , filing_user u +-- where e.CORP_NUM = c.CORP_NUM +-- and f.event_id = e.event_id +-- and u.event_id = e.event_id +-- and u.user_id = 'BCOMPS' +-- and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') +-- ) +-- and c.CORP_TYP_CD in ('LIB') + +-- -- altered from BC to BEN then BEN to BC before directed launch +-- and c.CORP_NUM not in ('0460007', '1255957', '1186381') +-- ) +-- select case +-- when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM +-- else c.CORP_NUM +-- end CORP_NUM, +-- co.CAN_JUR_TYP_CD, +-- co.CONT_OUT_DT, +-- co.OTHR_JURI_DESC, +-- co.HOME_COMPANY_NME, +-- co.start_event_id, +-- co.end_event_id +-- from CONT_OUT co +-- , corporation_cte c +-- where co.corp_num = c.corp_num +-- order by c.corp_num, co.start_event_id; + + + +-- conv_event +transfer public.CONV_EVENT from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select e.event_id, + ce.effective_dt, + case ce.REPORT_CORP_IND + when 'N' then 0 + when 'Y' then 1 + else 1 + end REPORT_CORP_IND, + ce.ACTIVITY_USER_ID, + ce.ACTIVITY_DT, + ce.ANNUAL_FILE_DT, + ce.ACCESSION_NUM, + ce.REMARKS +from event e + , CONV_EVENT ce + , corporation_cte c +where e.event_id = ce.event_id + and c.corp_num = e.corp_num +order by e.event_id; + + + +-- conv_ledger +transfer public.CONV_LEDGER from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select e.event_id, + cl.LEDGER_TITLE_TXT, + cl.LEDGER_DESC, + cl.cars_docmnt_id +from event e + , CONV_LEDGER cl + , corporation_cte c +where e.event_id = cl.event_id + and c.corp_num = e.corp_num +order by e.event_id; + +transfer public.carsfile from cprd using +select + documtid, + filedate, + regiracf +from carsfile; + +transfer public.carsbox from cprd using +select + documtid, + accesnum, + batchnum, + boxrracf +from carsbox; + +transfer public.carsrept from cprd using +select + documtid, + docutype, + compnumb +from carsrept; + +transfer public.carindiv from cprd using +select + documtid, + replace(surname, CHR(0), '') as surname, + replace(firname, CHR(0), '') as firname, + replace(dircpoco, CHR(0), '') as dircpoco, + replace(dircflag, CHR(0), '') as dircflag, + replace(offiflag, CHR(0), '') as offiflag, + replace(chgreasn, CHR(0), '') as chgreasn, + replace(pfirname, CHR(0), '') as pfirname, + replace(psurname, CHR(0), '') as psurname, + replace(offtitle, CHR(0), '') as offtitle, + replace(dircaddr01, CHR(0), '') as dircaddr01, + replace(dircaddr02, CHR(0), '') as dircaddr02, + replace(dircaddr03, CHR(0), '') as dircaddr03, + replace(dircaddr04, CHR(0), '') as dircaddr04 +from carindiv; + +-- corp_involved - amalgamaTING_businesses +transfer public.corp_involved_amalgamating from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select * +from (select e.event_id, -- SELECT BY EVENT + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end TED_CORP_NUM, + case + when c2.corp_typ_cd in ('BC', 'ULC', 'CC') then 'BC' || c2.corp_num + else c2.corp_num + end TING_CORP_NUM, + ci.CORP_INVOLVE_ID, + ci.CAN_JUR_TYP_CD, + case ci.ADOPTED_CORP_IND + when 'N' then 0 + when 'Y' then 1 + else 0 + end ADOPTED_CORP_IND, + ci.HOME_JURI_NUM, + ci.OTHR_JURI_DESC, + ci.FOREIGN_NME + from event e + , CORP_INVOLVED ci + , corporation_cte c + , corporation c2 + where e.event_id = ci.event_id + and c.corp_num = e.corp_num + and event_typ_cd = 'CONVAMAL' + and c2.corp_num = ci.corp_num + UNION ALL + select e.event_id, -- SELECT BY FILING + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end TED_CORP_NUM, + case + when c2.corp_typ_cd in ('BC', 'ULC', 'CC') then 'BC' || c2.corp_num + else c2.corp_num + end TING_CORP_NUM, + ci.CORP_INVOLVE_ID, + ci.CAN_JUR_TYP_CD, + case ci.ADOPTED_CORP_IND + when 'N' then 0 + when 'Y' then 1 + else 0 + end ADOPTED_CORP_IND, + ci.HOME_JURI_NUM, + ci.OTHR_JURI_DESC, + ci.FOREIGN_NME + from event e + , CORP_INVOLVED ci + , corporation_cte c + , corporation c2 + , filing f + where e.event_id = ci.event_id + and c.corp_num = e.corp_num + and c2.corp_num = ci.corp_num + and e.event_id = f.event_id + and filing_typ_cd in ('AMALH', 'AMALV', 'AMALR', 'AMLHU', 'AMLVU', 'AMLRU', 'AMLHC', 'AMLVC', 'AMLRC') + ) +order by event_id; + + + +-- corp_involved - continue_in_historical_xpro +transfer public.corp_involved_cont_in from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select e.event_id, + case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end CORP_NUM +from event e + , CORP_INVOLVED ci + , corporation_cte c + , filing f +where e.event_id = ci.event_id + and c.corp_num = e.corp_num + and e.event_id = f.event_id + and filing_typ_cd in ('CONTI', 'CONTU', 'CONTC') +order by e.event_id; + + + +-- corp_restriction +-- transfer public.CORP_RESTRICTION from cprd using +-- with corporation_cte as ( +-- select c.* +-- from corporation c +-- where c.CORP_NUM not in ( +-- select c.corp_num +-- from corporation c +-- , event e +-- , filing f +-- , filing_user u +-- where e.CORP_NUM = c.CORP_NUM +-- and f.event_id = e.event_id +-- and u.event_id = e.event_id +-- and u.user_id = 'BCOMPS' +-- and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') +-- ) +-- and c.CORP_TYP_CD in ('LIB') + +-- -- altered from BC to BEN then BEN to BC before directed launch +-- and c.CORP_NUM not in ('0460007', '1255957', '1186381') +-- ) +-- select case +-- when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM +-- else c.CORP_NUM +-- end CORP_NUM, +-- case cr.RESTRICTION_IND +-- when 'N' then 0 +-- when 'Y' then 1 +-- else 0 +-- end RESTRICTION_IND, +-- cr.start_event_id, +-- cr.end_event_id +-- from CORP_RESTRICTION cr +-- , corporation_cte c +-- where cr.corp_num = c.corp_num +-- order by c.corp_num, cr.start_event_id; + + + +-- correction +-- transfer public.CORRECTION from cprd using +-- with corporation_cte as ( +-- select c.* +-- from corporation c +-- where c.CORP_NUM not in ( +-- select c.corp_num +-- from corporation c +-- , event e +-- , filing f +-- , filing_user u +-- where e.CORP_NUM = c.CORP_NUM +-- and f.event_id = e.event_id +-- and u.event_id = e.event_id +-- and u.user_id = 'BCOMPS' +-- and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') +-- ) +-- and c.CORP_TYP_CD in ('LIB') + +-- -- altered from BC to BEN then BEN to BC before directed launch +-- and c.CORP_NUM not in ('0460007', '1255957', '1186381') +-- ) +-- select e.event_id, +-- case +-- when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM +-- else c.CORP_NUM +-- end CORP_NUM, +-- corr.ASSOCIATED_DOC_DESC +-- from event e +-- , CORRECTION corr +-- , corporation_cte c +-- where e.event_id = corr.event_id +-- and c.corp_num = e.corp_num +-- order by e.event_id; + + + +-- continued_in_from_jurisdiction +transfer public.jurisdiction from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select case + when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM + else c.CORP_NUM + end CORP_NUM, + j.CAN_JUR_TYP_CD, + j.XPRO_TYP_CD, + j.HOME_RECOGN_DT, + j.OTHR_JURIS_DESC, + j.HOME_JURIS_NUM, + j.BC_XPRO_NUM, + j.HOME_COMPANY_NME, + j.start_event_id +from JURISDICTION j + , corporation_cte c +where j.corp_num = c.corp_num +order by c.corp_num, j.start_event_id; + + + +-- resolution +-- transfer public.RESOLUTION from cprd using +-- with corporation_cte as ( +-- select c.* +-- from corporation c +-- where c.CORP_NUM not in ( +-- select c.corp_num +-- from corporation c +-- , event e +-- , filing f +-- , filing_user u +-- where e.CORP_NUM = c.CORP_NUM +-- and f.event_id = e.event_id +-- and u.event_id = e.event_id +-- and u.user_id = 'BCOMPS' +-- and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') +-- ) +-- and c.CORP_TYP_CD in ('LIB') + +-- -- altered from BC to BEN then BEN to BC before directed launch +-- and c.CORP_NUM not in ('0460007', '1255957', '1186381') +-- ) +-- select case +-- when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM +-- else c.CORP_NUM +-- end CORP_NUM, +-- r.RESOLUTION_DT, +-- r.RESOLUTION_TYPE_CODE, +-- r.start_event_id, +-- r.end_event_id +-- from RESOLUTION r +-- , corporation_cte c +-- where r.corp_num = c.corp_num +-- order by c.corp_num, r.start_event_id; + + + +-- share_struct +-- transfer public.SHARE_STRUCT from cprd using +-- with corporation_cte as ( +-- select c.* +-- from corporation c +-- where c.CORP_NUM not in ( +-- select c.corp_num +-- from corporation c +-- , event e +-- , filing f +-- , filing_user u +-- where e.CORP_NUM = c.CORP_NUM +-- and f.event_id = e.event_id +-- and u.event_id = e.event_id +-- and u.user_id = 'BCOMPS' +-- and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') +-- ) +-- and c.CORP_TYP_CD in ('LIB') + +-- -- altered from BC to BEN then BEN to BC before directed launch +-- and c.CORP_NUM not in ('0460007', '1255957', '1186381') +-- ) +-- select case +-- when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM +-- else c.CORP_NUM +-- end CORP_NUM, +-- ss.start_event_id, +-- ss.end_event_id +-- from SHARE_STRUCT ss +-- , corporation_cte c +-- where ss.corp_num = c.corp_num +-- order by c.corp_num, ss.start_event_id; + + + +--share_struct_cls +-- transfer public.SHARE_STRUCT_CLS from cprd using +-- with corporation_cte as ( +-- select c.* +-- from corporation c +-- where c.CORP_NUM not in ( +-- select c.corp_num +-- from corporation c +-- , event e +-- , filing f +-- , filing_user u +-- where e.CORP_NUM = c.CORP_NUM +-- and f.event_id = e.event_id +-- and u.event_id = e.event_id +-- and u.user_id = 'BCOMPS' +-- and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') +-- ) +-- and c.CORP_TYP_CD in ('LIB') + +-- -- altered from BC to BEN then BEN to BC before directed launch +-- and c.CORP_NUM not in ('0460007', '1255957', '1186381') +-- ) +-- select case +-- when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM +-- else c.CORP_NUM +-- end CORP_NUM, +-- ssc.SHARE_CLASS_ID, +-- replace(ssc.CLASS_NME, CHR(0), '') as CLASS_NME, +-- ssc.CURRENCY_TYP_CD, +-- case ssc.MAX_SHARE_IND +-- when 'N' then 0 +-- when 'Y' then 1 +-- else 0 +-- end MAX_SHARE_IND, +-- ssc.SHARE_QUANTITY, +-- case ssc.SPEC_RIGHTS_IND +-- when 'N' then 0 +-- when 'Y' then 1 +-- else 0 +-- end SPEC_RIGHTS_IND, +-- case ssc.PAR_VALUE_IND +-- when 'N' then 0 +-- when 'Y' then 1 +-- else 0 +-- end PAR_VALUE_IND, +-- ssc.PAR_VALUE_AMT + 0 AS PAR_VALUE_AMT, +-- ssc.OTHER_CURRENCY, +-- ssc.start_event_id +-- from SHARE_STRUCT_CLS ssc +-- , corporation_cte c +-- where ssc.corp_num = c.corp_num +-- order by c.corp_num, ssc.start_event_id; + + + +--share_series +-- transfer public.SHARE_SERIES from cprd using +-- with corporation_cte as ( +-- select c.* +-- from corporation c +-- where c.CORP_NUM not in ( +-- select c.corp_num +-- from corporation c +-- , event e +-- , filing f +-- , filing_user u +-- where e.CORP_NUM = c.CORP_NUM +-- and f.event_id = e.event_id +-- and u.event_id = e.event_id +-- and u.user_id = 'BCOMPS' +-- and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') +-- ) +-- and c.CORP_TYP_CD in ('LIB') + +-- -- altered from BC to BEN then BEN to BC before directed launch +-- and c.CORP_NUM not in ('0460007', '1255957', '1186381') +-- ) +-- select case +-- when c.CORP_TYP_CD in ('BC', 'ULC', 'CC') then 'BC' || c.CORP_NUM +-- else c.CORP_NUM +-- end CORP_NUM, +-- ss.SHARE_CLASS_ID, +-- ss.SERIES_ID, +-- case ss.MAX_SHARE_IND +-- when 'N' then 0 +-- when 'Y' then 1 +-- else 0 +-- end MAX_SHARE_IND, +-- ss.SHARE_QUANTITY, +-- case ss.SPEC_RIGHT_IND +-- when 'N' then 0 +-- when 'Y' then 1 +-- else 0 +-- end SPEC_RIGHT_IND, +-- ss.SERIES_NME, +-- ss.start_event_id +-- from SHARE_SERIES ss +-- , corporation_cte c +-- where ss.corp_num = c.corp_num +-- order by c.corp_num, ss.start_event_id; + + + +-- notification +transfer public.NOTIFICATION from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select e.event_id, + n.METHOD_TYP_CD, + n.mailing_addr_id, + n.FIRST_NME, + n.LAST_NME, + n.MIDDLE_NME, + n.PICKUP_BY, + n.EMAIL_ADDRESS, + n.PHONE_NUMBER +from event e + , NOTIFICATION n + , corporation_cte c +where e.event_id = n.event_id + and c.corp_num = e.corp_num +order by e.event_id; + + + +-- notification_resend +transfer public.NOTIFICATION_RESEND from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select e.event_id, + nr.METHOD_TYP_CD, + nr.mailing_addr_id, + nr.FIRST_NME, + nr.LAST_NME, + nr.MIDDLE_NME, + nr.PICKUP_BY, + nr.EMAIL_ADDRESS, + nr.PHONE_NUMBER +from event e + , NOTIFICATION_RESEND nr + , corporation_cte c +where e.event_id = nr.event_id + and c.corp_num = e.corp_num +order by e.event_id; + + + +-- party_notification +-- transfer public.PARTY_NOTIFICATION from cprd using +-- with corporation_cte as ( +-- select c.* +-- from corporation c +-- where c.CORP_NUM not in ( +-- select c.corp_num +-- from corporation c +-- , event e +-- , filing f +-- , filing_user u +-- where e.CORP_NUM = c.CORP_NUM +-- and f.event_id = e.event_id +-- and u.event_id = e.event_id +-- and u.user_id = 'BCOMPS' +-- and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') +-- ) +-- and c.CORP_TYP_CD in ('LIB') + +-- -- altered from BC to BEN then BEN to BC before directed launch +-- and c.CORP_NUM not in ('0460007', '1255957', '1186381') +-- ) +-- select pn.PARTY_ID, +-- pn.METHOD_TYP_CD, +-- pn.mailing_addr_id, +-- pn.FIRST_NME, +-- pn.LAST_NME, +-- pn.MIDDLE_NME, +-- pn.BUSINESS_NME, +-- pn.PICKUP_BY, +-- pn.EMAIL_ADDRESS, +-- pn.PHONE_NUMBER +-- from corp_party cp +-- , PARTY_NOTIFICATION pn +-- , corporation_cte c +-- where cp.CORP_PARTY_ID = pn.party_id +-- and c.corp_num = cp.corp_num +-- order by c.corp_num; + + + +-- payment +transfer public.payment from cprd using +with corporation_cte as ( + select c.* + from corporation c + where c.CORP_NUM not in ( + select c.corp_num + from corporation c + , event e + , filing f + , filing_user u + where e.CORP_NUM = c.CORP_NUM + and f.event_id = e.event_id + and u.event_id = e.event_id + and u.user_id = 'BCOMPS' + and f.filing_typ_cd in ('BEINC', 'ICORP', 'ICORU', 'ICORC', 'CONTB', 'CONTI', 'CONTU', 'CONTC') + ) + and c.CORP_TYP_CD in ('LIB') + + -- altered from BC to BEN then BEN to BC before directed launch + and c.CORP_NUM not in ('0460007', '1255957', '1186381') +) +select p.event_id, + p.payment_typ_cd, + p.cc_holder_nme +from payment p + , event e + , corporation_cte c +where p.event_id = e.event_id +and e.corp_num = c.corp_num +order by e.event_id; + + + +-- alter tables +alter table corporation alter column send_ar_ind type boolean using send_ar_ind::boolean; +alter table filing + alter column arrangement_ind type boolean using arrangement_ind::boolean, + alter column court_appr_ind type boolean using court_appr_ind::boolean; +alter table share_struct_cls + alter column max_share_ind type boolean using max_share_ind::boolean, + alter column spec_rights_ind type boolean using spec_rights_ind::boolean, + alter column par_value_ind type boolean using par_value_ind::boolean; +alter table conv_event alter report_corp_ind type boolean using report_corp_ind::boolean; +alter table corp_involved_amalgamating alter adopted_corp_ind type boolean using adopted_corp_ind::boolean; +alter table corp_restriction alter restriction_ind type boolean using restriction_ind::boolean; +alter table share_series + alter column max_share_ind type boolean using max_share_ind::boolean, + alter column spec_right_ind type boolean using spec_right_ind::boolean; + + +-- enable triggers +ALTER TABLE corporation ENABLE TRIGGER ALL; +ALTER TABLE corp_name ENABLE TRIGGER ALL; +ALTER TABLE corp_state ENABLE TRIGGER ALL; +ALTER TABLE event ENABLE TRIGGER ALL; +ALTER TABLE filing ENABLE TRIGGER ALL; +ALTER TABLE filing_user ENABLE TRIGGER ALL; +ALTER TABLE office ENABLE TRIGGER ALL; +ALTER TABLE address ENABLE TRIGGER ALL; +ALTER TABLE corp_comments ENABLE TRIGGER ALL; +ALTER TABLE ledger_text ENABLE TRIGGER ALL; +ALTER TABLE corp_party ENABLE TRIGGER ALL; +ALTER TABLE corp_party_relationship ENABLE TRIGGER ALL; +ALTER TABLE offices_held ENABLE TRIGGER ALL; +ALTER TABLE completing_party ENABLE TRIGGER ALL; +ALTER TABLE submitting_party ENABLE TRIGGER ALL; +ALTER TABLE corp_flag ENABLE TRIGGER ALL; +ALTER TABLE cont_out ENABLE TRIGGER ALL; +ALTER TABLE conv_event ENABLE TRIGGER ALL; +ALTER TABLE conv_ledger ENABLE TRIGGER ALL; +ALTER TABLE corp_involved_amalgamating ENABLE TRIGGER ALL; +ALTER TABLE corp_involved_cont_in ENABLE TRIGGER ALL; +ALTER TABLE corp_restriction ENABLE TRIGGER ALL; +ALTER TABLE correction ENABLE TRIGGER ALL; +ALTER TABLE jurisdiction ENABLE TRIGGER ALL; +ALTER TABLE resolution ENABLE TRIGGER ALL; +ALTER TABLE share_series ENABLE TRIGGER ALL; +ALTER TABLE share_struct ENABLE TRIGGER ALL; +ALTER TABLE share_struct_cls ENABLE TRIGGER ALL; +ALTER TABLE notification ENABLE TRIGGER ALL; +ALTER TABLE notification_resend ENABLE TRIGGER ALL; +ALTER TABLE party_notification ENABLE TRIGGER ALL; + +-- Release COLIN extract refresh advisory lock. +SELECT pg_advisory_unlock( + hashtext('lear:data-tool:colin_subset_extract'), + hashtext('subset_run') +); diff --git a/data-tool/tests/delta_restore/expected/t01_preview_patterns.txt b/data-tool/tests/delta_restore/expected/t01_preview_patterns.txt new file mode 100644 index 0000000000..6b8ef29858 --- /dev/null +++ b/data-tool/tests/delta_restore/expected/t01_preview_patterns.txt @@ -0,0 +1,7 @@ +Delta restore preview +Class counts +Detail artifacts +Viewing tips +--mode validate +UNCHANGED +selection.conf diff --git a/data-tool/tests/delta_restore/expected/t08_blocked_fk_assert.sql b/data-tool/tests/delta_restore/expected/t08_blocked_fk_assert.sql new file mode 100644 index 0000000000..c9d345f073 --- /dev/null +++ b/data-tool/tests/delta_restore/expected/t08_blocked_fk_assert.sql @@ -0,0 +1,17 @@ +DO $$ +DECLARE + v_class text; + v_reason text; +BEGIN + SELECT class, block_reason INTO v_class, v_reason + FROM delta_diff.mig_batch_class + LIMIT 1; + + IF v_class <> 'BLOCKED_FK' THEN + RAISE EXCEPTION 'expected mig_batch BLOCKED_FK, got class=% reason=%', v_class, v_reason; + END IF; + + IF v_reason IS NULL OR v_reason NOT LIKE '%mig_group parent%' THEN + RAISE EXCEPTION 'expected preserved-parent block reason, got %', v_reason; + END IF; +END $$; diff --git a/data-tool/tests/delta_restore/expected/t09_unenforced_nk_ambiguity_assert.sql b/data-tool/tests/delta_restore/expected/t09_unenforced_nk_ambiguity_assert.sql new file mode 100644 index 0000000000..4bdf6edd62 --- /dev/null +++ b/data-tool/tests/delta_restore/expected/t09_unenforced_nk_ambiguity_assert.sql @@ -0,0 +1,12 @@ +DO $$ +DECLARE + v_ambiguous bigint; +BEGIN + SELECT count(*) INTO v_ambiguous + FROM delta_diff.bar_corps_class + WHERE class = 'AMBIGUOUS_NK'; + + IF v_ambiguous <> 2 THEN + RAISE EXCEPTION 'expected 2 bar_corps AMBIGUOUS_NK rows, got %', v_ambiguous; + END IF; +END $$; diff --git a/data-tool/tests/delta_restore/expected/t10_auth_component_local_only_assert.sql b/data-tool/tests/delta_restore/expected/t10_auth_component_local_only_assert.sql new file mode 100644 index 0000000000..b1e3f207b7 --- /dev/null +++ b/data-tool/tests/delta_restore/expected/t10_auth_component_local_only_assert.sql @@ -0,0 +1,13 @@ +DO $$ +DECLARE + v_local_only bigint; +BEGIN + SELECT row_count INTO v_local_only + FROM delta_ctl.run_counts + WHERE table_name = 'auth_component_operation' + AND count_name = 'LOCAL_ONLY'; + + IF COALESCE(v_local_only, -1) <> 1 THEN + RAISE EXCEPTION 'expected auth_component_operation LOCAL_ONLY=1, got %', v_local_only; + END IF; +END $$; diff --git a/data-tool/tests/delta_restore/expected/t11_nk_hash_join_perf_assert.sql b/data-tool/tests/delta_restore/expected/t11_nk_hash_join_perf_assert.sql new file mode 100644 index 0000000000..72aef58437 --- /dev/null +++ b/data-tool/tests/delta_restore/expected/t11_nk_hash_join_perf_assert.sql @@ -0,0 +1,103 @@ +\if :{?t11_nk_hash_join_perf_max_ms} +\else +\set t11_nk_hash_join_perf_max_ms 60000 +\endif + +CREATE TEMP TABLE t11_nk_hash_join_perf_settings(max_ms numeric NOT NULL); +INSERT INTO t11_nk_hash_join_perf_settings(max_ms) +VALUES (:t11_nk_hash_join_perf_max_ms); + +DO $$ +DECLARE + v_elapsed_ms numeric; + v_max_ms numeric; + v_unchanged bigint; + v_changed bigint; + v_new bigint; + v_local_only bigint; + v_null_class text; + v_null_local_ctid tid; + v_null_changed_cols text[]; + v_cross_type_hash_match boolean; +BEGIN + SELECT max_ms INTO v_max_ms + FROM t11_nk_hash_join_perf_settings + LIMIT 1; + + IF v_max_ms IS NULL OR v_max_ms <= 0 THEN + RAISE EXCEPTION 'expected positive t11 max runtime threshold, got %', v_max_ms; + END IF; + + SELECT elapsed_ms INTO v_elapsed_ms + FROM t11_nk_hash_join_perf_timing + LIMIT 1; + + IF v_elapsed_ms IS NULL THEN + RAISE EXCEPTION 'expected t11 elapsed timing row'; + END IF; + + -- Generous, configurable ceiling: hash-accelerated NK classification should be + -- comfortably below this for ~50k local + ~50k staged rows; nested-loop regressions + -- should fail without making slower CI hosts flaky. Override with psql variable + -- t11_nk_hash_join_perf_max_ms when needed. + IF v_elapsed_ms > v_max_ms THEN + RAISE EXCEPTION 'expected t11 NK hash smoke under % ms, got % ms', v_max_ms, v_elapsed_ms; + END IF; + + EXECUTE format('SELECT (%s) = (%s)', + delta_ctl.nk_hash_expr(ARRAY['5::integer']), + delta_ctl.nk_hash_expr(ARRAY['5::bigint'])) + INTO v_cross_type_hash_match; + + IF v_cross_type_hash_match IS DISTINCT FROM true THEN + RAISE EXCEPTION 'expected NK hash equality for integer and bigint jsonb rendering'; + END IF; + + SELECT count(*) INTO v_unchanged + FROM delta_diff.bar_corps_class + WHERE class = 'UNCHANGED'; + + SELECT count(*) INTO v_changed + FROM delta_diff.bar_corps_class + WHERE class = 'CHANGED'; + + SELECT count(*) INTO v_new + FROM delta_diff.bar_corps_class + WHERE class = 'NEW'; + + SELECT row_count INTO v_local_only + FROM delta_ctl.run_counts + WHERE table_name = 'bar_corps' + AND count_name = 'LOCAL_ONLY'; + + IF v_unchanged <> 25000 THEN + RAISE EXCEPTION 'expected bar_corps UNCHANGED=25000, got %', v_unchanged; + END IF; + + IF v_changed <> 15001 THEN + RAISE EXCEPTION 'expected bar_corps CHANGED=15001 including NULL-key match, got %', v_changed; + END IF; + + IF v_new <> 10000 THEN + RAISE EXCEPTION 'expected bar_corps NEW=10000, got %', v_new; + END IF; + + IF COALESCE(v_local_only, -1) <> 10000 THEN + RAISE EXCEPTION 'expected bar_corps LOCAL_ONLY=10000, got %', v_local_only; + END IF; + + SELECT d.class, d.local_ctid, d.changed_cols + INTO v_null_class, v_null_local_ctid, v_null_changed_cols + FROM delta_diff.bar_corps_class d + JOIN delta_stage.bar_corps s ON s._delta_row_id = d._delta_row_id + WHERE s.identifier IS NULL; + + IF v_null_class <> 'CHANGED' OR v_null_local_ctid IS NULL THEN + RAISE EXCEPTION 'expected staged NULL identifier to match local NULL as CHANGED, got class=% local_ctid=%', + v_null_class, v_null_local_ctid; + END IF; + + IF v_null_changed_cols IS DISTINCT FROM ARRAY['notes']::text[] THEN + RAISE EXCEPTION 'expected NULL-key match changed_cols={notes}, got %', v_null_changed_cols; + END IF; +END $$; diff --git a/data-tool/tests/delta_restore/expected/t12_row_selection_assert.sql b/data-tool/tests/delta_restore/expected/t12_row_selection_assert.sql new file mode 100644 index 0000000000..baa23b8faf --- /dev/null +++ b/data-tool/tests/delta_restore/expected/t12_row_selection_assert.sql @@ -0,0 +1,46 @@ +DO $$ +DECLARE + v_bad bigint; +BEGIN + IF (SELECT count(*) FROM public.mig_batch WHERE id = 152) <> 1 THEN + RAISE EXCEPTION 'expected selected mig_batch parent id 152'; + END IF; + IF (SELECT count(*) FROM public.mig_corp_account) <> 16 + OR EXISTS (SELECT 1 FROM public.mig_corp_account WHERE id = 23) THEN + RAISE EXCEPTION 'mig_corp_account row exclusion failed'; + END IF; + IF (SELECT count(*) FROM public.mig_corp_batch) <> 16 + OR EXISTS (SELECT 1 FROM public.mig_corp_batch WHERE id > 3016) THEN + RAISE EXCEPTION 'mig_corp_batch range inclusion failed'; + END IF; + IF (SELECT count(*) FROM public.corp_processing) <> 16 + OR EXISTS (SELECT 1 FROM public.corp_processing WHERE id > 4016) THEN + RAISE EXCEPTION 'corp_processing range inclusion failed'; + END IF; + + SELECT count(*) INTO v_bad + FROM delta_ctl.apply_counts + WHERE expected_count <> affected_count; + IF v_bad <> 0 THEN + RAISE EXCEPTION 'expected apply expected_count = affected_count'; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM delta_ctl.apply_counts + WHERE table_name = 'mig_batch' AND class = 'NEW' + AND expected_count = 1 AND affected_count = 1 + ) OR NOT EXISTS ( + SELECT 1 FROM delta_ctl.apply_counts + WHERE table_name = 'mig_corp_account' AND class = 'NEW' + AND expected_count = 16 AND affected_count = 16 + ) OR NOT EXISTS ( + SELECT 1 FROM delta_ctl.apply_counts + WHERE table_name = 'mig_corp_batch' AND class = 'NEW' + AND expected_count = 16 AND affected_count = 16 + ) OR NOT EXISTS ( + SELECT 1 FROM delta_ctl.apply_counts + WHERE table_name = 'corp_processing' AND class = 'NEW' + AND expected_count = 16 AND affected_count = 16 + ) THEN + RAISE EXCEPTION 'missing expected 1/16/16/16 apply counts'; + END IF; +END $$; diff --git a/data-tool/tests/delta_restore/expected/t13_selector_diagnostics_assert.sql b/data-tool/tests/delta_restore/expected/t13_selector_diagnostics_assert.sql new file mode 100644 index 0000000000..6a75034bb8 --- /dev/null +++ b/data-tool/tests/delta_restore/expected/t13_selector_diagnostics_assert.sql @@ -0,0 +1,39 @@ +DO $$ +BEGIN + IF (SELECT count(*) FROM delta_ctl.selection_diagnostics WHERE problem = 'KIND_UNSUPPORTED') <> 2 THEN + RAISE EXCEPTION 'expected two KIND_UNSUPPORTED diagnostics'; + END IF; + IF NOT EXISTS (SELECT 1 FROM delta_ctl.selection_diagnostics + WHERE source_line = 12 AND problem = 'CLASS_NOT_INCLUDED') THEN + RAISE EXCEPTION 'expected CLASS_NOT_INCLUDED diagnostic'; + END IF; + IF NOT EXISTS (SELECT 1 FROM delta_ctl.selection_diagnostics + WHERE source_line = 13 AND problem = 'NO_MATCH' AND matched = 0) THEN -- NOSONAR + RAISE EXCEPTION 'expected NO_MATCH diagnostic'; + END IF; + IF NOT EXISTS (SELECT 1 FROM delta_ctl.selection_diagnostics + WHERE source_line = 14 AND problem = 'NO_MATCH' AND matched = 0) THEN + RAISE EXCEPTION 'expected --only-corps to reduce selector to NO_MATCH'; + END IF; + IF NOT EXISTS (SELECT 1 FROM delta_ctl.selection_diagnostics + WHERE source_line = 15 AND problem IS NULL AND matched = 1) THEN + RAISE EXCEPTION 'expected parent-derived auth component corp selector match'; + END IF; + IF NOT EXISTS (SELECT 1 FROM delta_ctl.selection_diagnostics + WHERE source_line = 16 AND problem IS NULL AND matched = 1) THEN + RAISE EXCEPTION 'expected PK-less row selector match'; + END IF; + IF NOT EXISTS (SELECT 1 FROM delta_ctl.selection_diagnostics + WHERE source_line = 17 AND problem = 'NO_MATCH' AND matched = 0) THEN + RAISE EXCEPTION 'expected absent staged table to produce NO_MATCH'; + END IF; + + DELETE FROM delta_ctl.row_selection WHERE source_line NOT IN (15, 16); + PERFORM delta_ctl.stamp_selection(); + IF NOT EXISTS (SELECT 1 FROM delta_diff.auth_component_operation_class WHERE selected) THEN + RAISE EXCEPTION 'expected parent-derived auth component corp selector to stamp row'; + END IF; + IF NOT EXISTS (SELECT 1 FROM delta_diff.email_domain_groups_class WHERE selected) THEN + RAISE EXCEPTION 'expected PK-less row selector to stamp row'; + END IF; +END $$; diff --git a/data-tool/tests/delta_restore/expected/t14_parent_dependency_rows_assert.sql b/data-tool/tests/delta_restore/expected/t14_parent_dependency_rows_assert.sql new file mode 100644 index 0000000000..d89ecab894 --- /dev/null +++ b/data-tool/tests/delta_restore/expected/t14_parent_dependency_rows_assert.sql @@ -0,0 +1,17 @@ +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM delta_ctl.dependency_violations + WHERE child_table = 'mig_batch' AND parent_table = 'mig_group' + AND row_count = 1 AND sample_ids = '200' + ) THEN + RAISE EXCEPTION 'expected dependency violation with child sample id 200'; + END IF; + UPDATE delta_diff.mig_group_class + SET selected = true + WHERE staged_pk = 100; + PERFORM delta_ctl.validate_dependencies(); + IF EXISTS (SELECT 1 FROM delta_ctl.dependency_violations) THEN + RAISE EXCEPTION 'expected dependency validation to pass after selecting parent'; + END IF; +END $$; diff --git a/data-tool/tests/delta_restore/expected/t15_details_assert.sql b/data-tool/tests/delta_restore/expected/t15_details_assert.sql new file mode 100644 index 0000000000..06f5138c15 --- /dev/null +++ b/data-tool/tests/delta_restore/expected/t15_details_assert.sql @@ -0,0 +1,81 @@ +DO $$ +DECLARE + v_new text; + v_changed text; + v_blocked text; + v_samples text; + v_control text; +BEGIN + SELECT delta_ctl.display_value( + jsonb_build_object('value', E'line\nnext\tcell\rreturn' || chr(27) || 'escape' || chr(1) || 'control'), + 'value', 200) + INTO v_control; + IF position(chr(13) in v_control) > 0 + OR position(chr(27) in v_control) > 0 + OR position(chr(1) in v_control) > 0 + OR v_control NOT LIKE '%␤%' + OR v_control NOT LIKE '%␉%' + OR v_control NOT LIKE '%␍%' + OR v_control NOT LIKE '%␛%' + OR v_control NOT LIKE '%�%' THEN + RAISE EXCEPTION 'preview display value retained controls or omitted visible markers: %', v_control; + END IF; + + SELECT string_agg(line, E'\n') INTO v_new + FROM delta_ctl.render_new_rows_tsv('bad_emails', 1) line; -- NOSONAR + IF v_new NOT LIKE '%selector_id%' OR v_new NOT LIKE '%# TRUNCATED at 1 rows%' + OR position(E'tab\\tline\\nnext\\rreturn\\\\slash' in v_new) = 0 THEN + RAISE EXCEPTION 'NEW detail output missing header, escaping, or truncation: %', v_new; + END IF; + + SELECT string_agg(line, E'\n') INTO v_changed + FROM delta_ctl.render_changed_rows_tsv('bad_emails', 10) line; + IF v_changed NOT LIKE '%local old%' OR v_changed NOT LIKE '%dump new%' + OR v_changed NOT LIKE '%notes%' THEN + RAISE EXCEPTION 'CHANGED detail output missing old-to-new values: %', v_changed; + END IF; + + UPDATE delta_diff.bad_emails_class + SET class = 'BLOCKED_FK', + block_reason = E'missing\tparent\nrow\rreturn' || chr(27) || 'escape' || chr(1) || 'control' + WHERE staged_pk = 3; + SELECT string_agg(line, E'\n') INTO v_blocked + FROM delta_ctl.render_blocked_rows_tsv('bad_emails', 10) line; + IF v_blocked NOT LIKE '%newer@example.test%' + OR position(E'missing\\tparent\\nrow\\rreturn' in v_blocked) = 0 + OR position(chr(27) in v_blocked) > 0 + OR position(chr(1) in v_blocked) > 0 + OR v_blocked NOT LIKE '%␛%' + OR v_blocked NOT LIKE '%�%' THEN + RAISE EXCEPTION 'BLOCKED detail output missing escaping/sanitization: %', v_blocked; + END IF; + + SELECT string_agg(line, E'\n') INTO v_control + FROM delta_ctl.render_samples('bad_emails', 'BLOCKED_FK', 10) line; + IF position(chr(13) in v_control) > 0 + OR position(chr(27) in v_control) > 0 + OR position(chr(1) in v_control) > 0 + OR v_control NOT LIKE '%␍%' + OR v_control NOT LIKE '%␛%' + OR v_control NOT LIKE '%�%' THEN + RAISE EXCEPTION 'preview block reason retained controls or omitted visible markers: %', v_control; + END IF; + + SELECT string_agg(line, E'\n') INTO v_samples + FROM delta_ctl.render_samples('bad_emails', 'CHANGED', 10) line; -- NOSONAR + IF v_samples NOT LIKE '%email=changed@example.test%' + OR v_samples NOT LIKE '%changed notes: local old → dump new%' THEN + RAISE EXCEPTION 'enriched sample output missing values/diff: %', v_samples; + END IF; + + UPDATE delta_stage.bad_emails SET notes = repeat('x', 300) WHERE id = 1; + UPDATE delta_diff.bad_emails_class + SET changed_cols = ARRAY['email', 'notes', 'extra_1', 'extra_2', 'extra_3', 'extra_4'] + WHERE staged_pk = 1; + IF (SELECT count(*) FROM delta_ctl.render_samples('bad_emails', 'CHANGED', 10)) <> 6 THEN + RAISE EXCEPTION 'expected one sample line plus at most five changed-column lines'; + END IF; + IF (SELECT max(length(line)) FROM delta_ctl.render_samples('bad_emails', 'CHANGED', 10) line) > 220 THEN + RAISE EXCEPTION 'expected preview sample lines to be capped at 220 characters'; + END IF; +END $$; diff --git a/data-tool/tests/delta_restore/expected/t16_selection_manifest_assert.sql b/data-tool/tests/delta_restore/expected/t16_selection_manifest_assert.sql new file mode 100644 index 0000000000..cfa049081d --- /dev/null +++ b/data-tool/tests/delta_restore/expected/t16_selection_manifest_assert.sql @@ -0,0 +1,603 @@ +DO $$ +DECLARE + v_lines text[]; + v_none_lines text[]; + v_variant_lines text[]; + v_manifest text; + v_error text; + v_invalid_value text; + v_binding_headers integer; + v_active_count integer; + v_include_min integer; + v_include_max integer; + v_active integer; + v_wildcard integer; + v_small_new integer; + v_small_changed integer; + v_large_new integer; + v_large_changed integer; + v_pointer integer; + v_pos integer; + v_generic_count integer; + v_range_line_count integer; + v_singleton_line_count integer; + v_last_range integer; + v_first_singleton integer; + v_singleton_ordinals bigint[]; + v_singleton_ordinal_index integer; + v_exact_count bigint; + v_exact_distinct bigint; + v_missing_count bigint; + v_extra_count bigint; + c_range_counts_pattern constant text := '# Range counts:%'; + c_boundary_selector_pattern constant text := + '# [selector_limit_boundary] new.rows include=id:%'; +BEGIN + IF EXISTS ( + SELECT 1 FROM delta_ctl.run_metadata WHERE key = 'selector_suggestion_limit' -- NOSONAR + ) THEN + RAISE EXCEPTION 'selector limit fixture must begin without metadata to test fallback'; + END IF; + + SELECT array_agg(line ORDER BY ord), string_agg(line, E'\n' ORDER BY ord) + INTO v_lines, v_manifest + FROM delta_ctl.render_selection_manifest() WITH ORDINALITY AS manifest(line, ord); + + IF v_lines[1] <> '# delta-selection v2' + OR v_lines[2] <> '# dump_sha256=manifest-test-sha' + OR v_lines[3] <> '# staged bad_emails=5 selector_limit_boundary=51 mig_batch=5 mig_group=51 bar_corps=52 corp_processing=106 auth_component_operation=54' THEN + RAISE EXCEPTION 'v2 binding headers were not emitted first or exactly: %', v_manifest; + END IF; + + SELECT count(*) INTO v_binding_headers + FROM unnest(v_lines) line + WHERE line ~ '^# (dump_sha256=|staged )'; + IF v_binding_headers <> 2 THEN + RAISE EXCEPTION 'documentation was mistaken for a binding header: %', v_manifest; + END IF; + + v_active := array_position(v_lines, '# ACTIVE SELECTION'); + v_small_new := array_position( + v_lines, '# Small NEW sets receive ready-to-uncomment suggestions:'); -- NOSONAR + v_small_changed := array_position( + v_lines, '# Small CHANGED sets receive ready-to-uncomment suggestions:'); -- NOSONAR + v_large_new := array_position( + v_lines, '# Large NEW sets receive a useful pointer rather than no guidance:'); + v_large_changed := array_position( + v_lines, '# Large CHANGED sets receive a useful pointer rather than no guidance:'); + v_pointer := array_position( + v_lines, + '# Full selector grammar and 13 worked examples: selection_cookbook.txt (this run dir)'); + IF v_active IS NULL OR v_small_new IS NULL OR v_small_changed IS NULL + OR v_large_new IS NULL OR v_large_changed IS NULL OR v_pointer IS NULL + OR NOT (v_active < v_small_new + AND v_small_new < v_small_changed + AND v_small_changed < v_large_new + AND v_large_new < v_large_changed + AND v_large_changed < v_pointer) THEN + RAISE EXCEPTION 'manifest section order is wrong: %', v_manifest; + END IF; + IF v_lines[v_pointer - 1] <> '# ---------------------------------------------------------------------------' + OR v_lines[v_pointer + 1] <> '# ---------------------------------------------------------------------------' THEN + RAISE EXCEPTION 'cookbook pointer block does not match the approved shape: %', v_manifest; + END IF; + IF array_position(v_lines, '# OPERATOR COOKBOOK') IS NOT NULL + OR array_position(v_lines, '# 1. Apply all NEW and CHANGED rows for a table:') IS NOT NULL + OR array_position( + v_lines, + '# Classes: new | changed | changed_local_newer (other classes are never applyable).') + IS NOT NULL THEN + RAISE EXCEPTION 'manifest still embeds cookbook or grammar reference text: %', v_manifest; + END IF; + + SELECT count(*), min(strpos(line, 'include=')), max(strpos(line, 'include=')) + INTO v_active_count, v_include_min, v_include_max + FROM unnest(v_lines) line + WHERE line ~ '^\[[^]]+\][[:space:]]+include='; + IF v_active_count <> 8 OR v_include_min <> v_include_max THEN + RAISE EXCEPTION + 'wildcard/per-table active lines are missing or misaligned: count=% min=% max=% manifest=%', + v_active_count, v_include_min, v_include_max, v_manifest; + END IF; + + SELECT ord INTO v_wildcard + FROM unnest(v_lines) WITH ORDINALITY AS u(line, ord) + WHERE line LIKE '[*]%' LIMIT 1; + IF v_lines[v_wildcard] !~ '^\[\*\][[:space:]]+include=new,changed$' + OR v_lines[v_wildcard + 1] <> + '# Default selection ([*] new,changed) currently matches 323 rows across 7 tables (new=266 changed=57).' THEN + RAISE EXCEPTION 'default wildcard or blast-radius total changed: %', v_manifest; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM unnest(v_lines) line + WHERE line ~ '^\[mig_batch\][[:space:]]+include=new,changed # new=3 changed=2 changed_local_newer=0 parents=mig_group$' + ) THEN + RAISE EXCEPTION 'mig_batch active line lacks its preserved parent annotation: %', v_manifest; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM unnest(v_lines) line + WHERE line ~ '^\[corp_processing\][[:space:]]+include=new,changed # new=53 changed=53 changed_local_newer=0 parents=mig_batch$' + ) THEN + RAISE EXCEPTION 'corp_processing active line lacks its preserved parent annotation: %', v_manifest; + END IF; + IF EXISTS ( + SELECT 1 FROM unnest(v_lines) line + WHERE line LIKE '[mig_batch]%' AND line LIKE '%external:%' + ) THEN + RAISE EXCEPTION 'external FK leaked into parent annotations: %', v_manifest; + END IF; + + -- Every exact table block begins after an actual blank line. Mixed PK and + -- PK-less sets also keep range/count pairs adjacent and place one actual blank + -- line before their separate singleton selectors. + IF EXISTS ( + SELECT 1 + FROM unnest(v_lines) WITH ORDINALITY AS rendered(line, ord) + WHERE line ~ '^# \[[^]]+\] Exact (NEW|CHANGED) set:' + AND (ord < 3 + OR v_lines[(ord - 1)::integer] <> '' + OR v_lines[(ord - 2)::integer] = '') + ) THEN + RAISE EXCEPTION 'an exact table block lacks its leading blank line: %', v_manifest; + END IF; + + v_pos := array_position( + v_lines, '# [bad_emails] Exact NEW set: rows=3 singletons=1 ranges=1'); + IF v_pos IS NULL + OR v_lines[v_pos + 1] <> '# [bad_emails] new.rows include=id:2-3' + OR v_lines[v_pos + 2] <> '# Range counts: 2-3 (2 rows)' + OR v_lines[v_pos + 3] <> '' + OR v_lines[v_pos + 4] <> '# [bad_emails] new.rows include=id:5' THEN + RAISE EXCEPTION 'mixed PK NEW exact guidance has the wrong layout: %', v_manifest; + END IF; + v_pos := array_position( + v_lines, '# [mig_batch] Exact NEW set: rows=3 singletons=1 ranges=1'); + IF v_pos IS NULL + OR v_lines[v_pos + 1] <> '# [mig_batch] new.rows include=row:4-5' + OR v_lines[v_pos + 2] <> '# Range counts: 4-5 (2 rows)' + OR v_lines[v_pos + 3] <> '' + OR v_lines[v_pos + 4] <> '# [mig_batch] new.rows include=row:8' THEN + RAISE EXCEPTION 'mixed PK-less NEW exact guidance has the wrong layout: %', v_manifest; + END IF; + v_pos := array_position( + v_lines, '# [bad_emails] Exact CHANGED set: rows=2 singletons=2 ranges=0'); + IF v_pos IS NULL + OR v_lines[v_pos + 1] <> '# [bad_emails] changed.rows include=id:7,9' + OR v_lines[v_pos + 2] LIKE c_range_counts_pattern THEN + RAISE EXCEPTION 'singleton-only CHANGED exact guidance has the wrong layout: %', v_manifest; + END IF; + v_pos := array_position( + v_lines, '# [mig_batch] Exact CHANGED set: rows=2 singletons=0 ranges=1'); + IF v_pos IS NULL + OR v_lines[v_pos + 1] <> '# [mig_batch] changed.rows include=row:10-11' + OR v_lines[v_pos + 2] <> '# Range counts: 10-11 (2 rows)' THEN + RAISE EXCEPTION 'range-only PK-less CHANGED guidance has the wrong layout: %', v_manifest; + END IF; + + -- The inclusive default boundary remains exact and forces both categories to wrap. + IF array_position( + v_lines, + '# [selector_limit_boundary] Exact NEW set: rows=50 singletons=40 ranges=5') + IS NULL THEN + RAISE EXCEPTION '50-row exact summary is missing: %', v_manifest; + END IF; + SELECT + count(*) FILTER (WHERE line LIKE '%-%'), + count(*) FILTER (WHERE line NOT LIKE '%-%'), + max(ord) FILTER (WHERE line LIKE '%-%'), + min(ord) FILTER (WHERE line NOT LIKE '%-%') + INTO v_range_line_count, v_singleton_line_count, v_last_range, v_first_singleton + FROM unnest(v_lines) WITH ORDINALITY AS rendered(line, ord) + WHERE line LIKE c_boundary_selector_pattern; + IF v_range_line_count <= 1 OR v_singleton_line_count <= 8 + OR v_last_range >= v_first_singleton + OR v_lines[v_first_singleton - 1] <> '' + OR v_lines[v_first_singleton - 2] NOT LIKE c_range_counts_pattern THEN + RAISE EXCEPTION + 'boundary selectors did not wrap with all ranges before singletons: ranges=% singletons=% manifest=%', + v_range_line_count, v_singleton_line_count, v_manifest; + END IF; + + -- Long singleton lists form visual paragraphs of four selector lines. There + -- is exactly one actual blank line before selector lines 5, 9, 13, and so on. + SELECT array_agg(ord ORDER BY ord) + INTO v_singleton_ordinals + FROM unnest(v_lines) WITH ORDINALITY AS rendered(line, ord) + WHERE line LIKE c_boundary_selector_pattern + AND line NOT LIKE '%-%'; + FOR v_singleton_ordinal_index IN 2..cardinality(v_singleton_ordinals) LOOP + IF v_singleton_ordinals[v_singleton_ordinal_index] + - v_singleton_ordinals[v_singleton_ordinal_index - 1] + <> (CASE WHEN mod(v_singleton_ordinal_index - 1, 4) = 0 THEN 2 ELSE 1 END) THEN + RAISE EXCEPTION + 'long singleton selector paragraphs do not break every four lines: ordinals=% manifest=%', + v_singleton_ordinals, v_manifest; + END IF; + END LOOP; + + -- Every wrapped line is a complete selector. Categories never mix, and the soft + -- 120-character cap is exceeded only for one indivisible selector token. + IF EXISTS ( + SELECT 1 + FROM unnest(v_lines) line + WHERE line ~ '^# \[[^]]+\] (new|changed)\.rows include=(id|row):' + AND length(line) > 120 + AND cardinality(string_to_array( + regexp_replace(line, '^.*include=(id|row):', ''), ',')) <> 1 + ) OR EXISTS ( + SELECT 1 FROM unnest(v_lines) line + WHERE line LIKE c_range_counts_pattern AND length(line) > 120 + ) OR EXISTS ( + SELECT 1 + FROM unnest(v_lines) line + CROSS JOIN LATERAL unnest(string_to_array( + split_part(line, 'include=id:', 2), ',')) AS token -- NOSONAR + WHERE line LIKE c_boundary_selector_pattern + GROUP BY line + HAVING bool_or(position('-' in token) > 0) + AND bool_or(position('-' in token) = 0) + ) OR EXISTS ( + SELECT 1 FROM unnest(v_lines) line + WHERE right(line, 1) = E'\\' + OR line ~ '^#[[:space:]]+include=(id|row):' + ) THEN + RAISE EXCEPTION 'wrapped selector grammar, category separation, or line cap failed: %', + v_manifest; + END IF; + + -- Each wrapped range line is followed immediately by the annotation derived + -- from exactly that line's tokens, in the same order. + IF EXISTS ( + SELECT 1 + FROM unnest(v_lines) WITH ORDINALITY AS rendered(line, ord) + CROSS JOIN LATERAL ( + SELECT '# Range counts: ' || + string_agg( + token || ' (' || + (split_part(token, '-', 2)::bigint + - split_part(token, '-', 1)::bigint + 1)::text || + ' rows)', + ', ' ORDER BY token_ord) AS expected + FROM unnest(string_to_array( + split_part(rendered.line, 'include=id:', 2), ',')) + WITH ORDINALITY AS range_token(token, token_ord) + ) annotation + WHERE rendered.line LIKE '# [selector_limit_boundary] new.rows include=id:%-%' + AND v_lines[(rendered.ord + 1)::integer] <> annotation.expected + ) THEN + RAISE EXCEPTION 'a wrapped range annotation is absent or mismatched: %', v_manifest; + END IF; + + -- Expanding every repeated line must reconstruct the classified exact set once: + -- no omissions, extras, or duplicated tokens across chunks. + WITH selector_lines AS ( + SELECT split_part(line, 'include=id:', 2) AS payload + FROM unnest(v_lines) line + WHERE line LIKE c_boundary_selector_pattern + ), tokens AS ( + SELECT token + FROM selector_lines + CROSS JOIN LATERAL unnest(string_to_array(payload, ',')) AS parsed(token) + ), expanded AS ( + SELECT expanded_id AS id + FROM tokens + CROSS JOIN LATERAL generate_series( + split_part(token, '-', 1)::bigint, + CASE WHEN position('-' in token) > 0 + THEN split_part(token, '-', 2)::bigint + ELSE token::bigint + END + ) AS ids(expanded_id) + ), source AS ( + SELECT s.id + FROM delta_stage.selector_limit_boundary s + JOIN delta_diff.selector_limit_boundary_class d + ON d._delta_row_id = s._delta_row_id + WHERE d.class = 'NEW' + ) + SELECT + (SELECT count(*) FROM expanded), + (SELECT count(DISTINCT id) FROM expanded), + (SELECT count(*) FROM (SELECT id FROM source EXCEPT SELECT id FROM expanded) q), + (SELECT count(*) FROM (SELECT id FROM expanded EXCEPT SELECT id FROM source) q) + INTO v_exact_count, v_exact_distinct, v_missing_count, v_extra_count; + IF v_exact_count <> 50 OR v_exact_distinct <> 50 + OR v_missing_count <> 0 OR v_extra_count <> 0 THEN + RAISE EXCEPTION + 'repeated selector lines do not union to the original exact set: count=% distinct=% missing=% extra=% manifest=%', + v_exact_count, v_exact_distinct, v_missing_count, v_extra_count, v_manifest; + END IF; + + -- The paragraph threshold is strict: exactly eight singleton lines remain + -- contiguous, while nine lines break immediately before lines 5 and 9. + UPDATE delta_stage.selector_limit_boundary + SET id = 4000000000000000000::bigint + _delta_row_id * 2; + UPDATE delta_diff.selector_limit_boundary_class + SET class = CASE WHEN _delta_row_id <= 24 THEN 'NEW' ELSE 'UNCHANGED' END; -- NOSONAR + UPDATE delta_ctl.run_counts + SET row_count = 24 + WHERE table_name = 'selector_limit_boundary' AND count_name = 'NEW'; -- NOSONAR + SELECT array_agg(line ORDER BY ord) + INTO v_variant_lines + FROM delta_ctl.render_selection_manifest() WITH ORDINALITY AS manifest(line, ord); + SELECT array_agg(ord ORDER BY ord) + INTO v_singleton_ordinals + FROM unnest(v_variant_lines) WITH ORDINALITY AS rendered(line, ord) + WHERE line LIKE c_boundary_selector_pattern; + IF cardinality(v_singleton_ordinals) <> 8 OR EXISTS ( + SELECT 1 + FROM generate_subscripts(v_singleton_ordinals, 1) AS indexes(i) + WHERE i > 1 + AND v_singleton_ordinals[i] - v_singleton_ordinals[i - 1] <> 1 + ) THEN + RAISE EXCEPTION 'exactly eight singleton lines received paragraph spacing: ordinals=% manifest=%', + v_singleton_ordinals, array_to_string(v_variant_lines, E'\n'); + END IF; + + UPDATE delta_diff.selector_limit_boundary_class + SET class = CASE WHEN _delta_row_id <= 27 THEN 'NEW' ELSE 'UNCHANGED' END; + UPDATE delta_ctl.run_counts + SET row_count = 27 + WHERE table_name = 'selector_limit_boundary' AND count_name = 'NEW'; -- NOSONAR + SELECT array_agg(line ORDER BY ord) + INTO v_variant_lines + FROM delta_ctl.render_selection_manifest() WITH ORDINALITY AS manifest(line, ord); + SELECT array_agg(ord ORDER BY ord) + INTO v_singleton_ordinals + FROM unnest(v_variant_lines) WITH ORDINALITY AS rendered(line, ord) + WHERE line LIKE c_boundary_selector_pattern; + IF cardinality(v_singleton_ordinals) <> 9 OR EXISTS ( + SELECT 1 + FROM generate_subscripts(v_singleton_ordinals, 1) AS indexes(i) + WHERE i > 1 + AND v_singleton_ordinals[i] - v_singleton_ordinals[i - 1] + <> (CASE WHEN mod(i - 1, 4) = 0 THEN 2 ELSE 1 END) + ) THEN + RAISE EXCEPTION 'nine singleton lines lack paragraph spacing before lines 5 and 9: ordinals=% manifest=%', + v_singleton_ordinals, array_to_string(v_variant_lines, E'\n'); + END IF; + + -- Restore the inclusive 50-row fixture for all subsequent limit variants. + UPDATE delta_stage.selector_limit_boundary + SET id = CASE + WHEN _delta_row_id <= 10 THEN + 100000000000000::bigint + + ((_delta_row_id - 1) / 2) * 3 + + ((_delta_row_id - 1) % 2) + WHEN _delta_row_id <= 50 THEN + 2000000000000000000::bigint + (_delta_row_id - 11) * 2 + ELSE 3000000000000000000::bigint + END; + UPDATE delta_diff.selector_limit_boundary_class + SET class = CASE WHEN _delta_row_id <= 50 THEN 'NEW' ELSE 'UNCHANGED' END; + UPDATE delta_ctl.run_counts + SET row_count = 50 + WHERE table_name = 'selector_limit_boundary' AND count_name = 'NEW'; -- NOSONAR + + IF EXISTS ( + SELECT 1 FROM unnest(v_lines) line + WHERE line LIKE '# [%] changed_local_newer.rows include=%' + ) THEN + RAISE EXCEPTION 'CHANGED_LOCAL_NEWER received unsafe ready-to-uncomment guidance: %', v_manifest; + END IF; + + v_pos := array_position( + v_lines, '# corp_processing has 53 CHANGED rows; choose selector_id or corp values from:'); + IF v_pos IS NULL + OR v_lines[v_pos + 1] <> '# details/corp_processing.changed.tsv' + OR v_lines[v_pos + 2] <> '# Example syntax:' + OR v_lines[v_pos + 3] <> '# [corp_processing] changed.rows include=id:100,200-210' + OR v_lines[v_pos + 4] <> '# [corp_processing] changed.rows include=corp:BC0000001,BC0000002' + OR v_lines[v_pos + 5] <> '#' THEN + RAISE EXCEPTION 'large CHANGED guidance does not match the approved shape: %', v_manifest; + END IF; + IF NOT (v_large_changed < v_pos AND v_pos < v_pointer) THEN + RAISE EXCEPTION 'large CHANGED guidance is not in its grouped section: %', v_manifest; + END IF; + + v_pos := array_position( + v_lines, '# auth_component_operation has 54 NEW rows; choose selector_id from:'); + IF v_pos IS NULL + OR v_lines[v_pos + 1] <> '# details/auth_component_operation.new.tsv' + OR v_lines[v_pos + 3] <> '# [auth_component_operation] new.rows include=id:100,200-210' + OR v_lines[v_pos + 4] <> + '# corp: selectors are also supported via the staged auth_processing parent:' + OR v_lines[v_pos + 5] <> + '# [auth_component_operation] new.rows include=corp:BC0000001,BC0000002' THEN + RAISE EXCEPTION 'large NEW auth guidance lost its parent-derived corp caveat: %', v_manifest; + END IF; + + IF EXISTS ( + SELECT 1 FROM unnest(v_lines) line + WHERE line NOT LIKE '#%' AND line LIKE '%.rows %' + ) THEN + RAISE EXCEPTION 'rendered manifest contains an uncommented row selector: %', v_manifest; + END IF; + + -- Negative values still receive a summary but suppress every exact selector. + UPDATE delta_stage.bad_emails SET id = -2 WHERE _delta_row_id = 1; + SELECT array_agg(line ORDER BY ord) + INTO v_variant_lines + FROM delta_ctl.render_selection_manifest() WITH ORDINALITY AS manifest(line, ord); + IF array_position( + v_variant_lines, + '# [bad_emails] Exact NEW set: rows=3 singletons=3 ranges=0') IS NULL + OR array_position( + v_variant_lines, + '# [bad_emails] Exact NEW selector unavailable for this small set.') IS NULL + OR array_position( + v_variant_lines, + '# Review details/bad_emails.new.tsv; negative id: values are outside the selector grammar.') + IS NULL + OR EXISTS ( + SELECT 1 FROM unnest(v_variant_lines) line + WHERE line LIKE '# [bad_emails] new.rows include=id:%' + ) THEN + RAISE EXCEPTION 'negative selector fallback is incomplete or unsafe: %', + array_to_string(v_variant_lines, E'\n'); + END IF; + UPDATE delta_stage.bad_emails SET id = 2 WHERE _delta_row_id = 1; + + -- A singleton suggestion has no redundant range-count comment. + UPDATE delta_diff.bad_emails_class + SET class = 'UNCHANGED' + WHERE _delta_row_id = 5; + UPDATE delta_ctl.run_counts + SET row_count = 1 + WHERE table_name = 'bad_emails' AND count_name = 'CHANGED'; -- NOSONAR + SELECT array_agg(line ORDER BY ord) + INTO v_variant_lines + FROM delta_ctl.render_selection_manifest() WITH ORDINALITY AS manifest(line, ord); + v_pos := array_position( + v_variant_lines, + '# [bad_emails] Exact CHANGED set: rows=1 singletons=1 ranges=0'); + IF v_pos IS NULL + OR v_variant_lines[v_pos + 1] <> '# [bad_emails] changed.rows include=id:7' + OR v_variant_lines[v_pos + 2] LIKE c_range_counts_pattern THEN + RAISE EXCEPTION 'singleton suggestion received range-count guidance: %', + array_to_string(v_variant_lines, E'\n'); + END IF; + UPDATE delta_diff.bad_emails_class + SET class = 'CHANGED' + WHERE _delta_row_id = 5; + UPDATE delta_ctl.run_counts + SET row_count = 2 + WHERE table_name = 'bad_emails' AND count_name = 'CHANGED'; -- NOSONAR + + -- Missing selector-limit metadata falls back to 50: 50 is exact, 51 is generic. + UPDATE delta_diff.selector_limit_boundary_class + SET class = 'NEW' + WHERE _delta_row_id = 51; + UPDATE delta_ctl.run_counts + SET row_count = 51 + WHERE table_name = 'selector_limit_boundary' AND count_name = 'NEW'; -- NOSONAR + SELECT array_agg(line ORDER BY ord) + INTO v_variant_lines + FROM delta_ctl.render_selection_manifest() WITH ORDINALITY AS manifest(line, ord); + v_small_new := array_position( + v_variant_lines, '# Small NEW sets receive ready-to-uncomment suggestions:'); + v_small_changed := array_position( + v_variant_lines, '# Small CHANGED sets receive ready-to-uncomment suggestions:'); + IF EXISTS ( + SELECT 1 + FROM unnest(v_variant_lines) WITH ORDINALITY AS rendered(line, ord) + WHERE ord > v_small_new AND ord < v_small_changed + AND (line LIKE '# [selector_limit_boundary] Exact NEW set:%' + OR line LIKE c_boundary_selector_pattern) + ) + OR array_position( + v_variant_lines, + '# selector_limit_boundary has 51 NEW rows; choose selector_id from:') IS NULL THEN + RAISE EXCEPTION 'missing selector-limit metadata did not use the 50/51 boundary: %', + array_to_string(v_variant_lines, E'\n'); + END IF; + + -- A custom limit applies to both complementary renderer branches. + INSERT INTO delta_ctl.run_metadata(key, value) + VALUES ('selector_suggestion_limit', '2') + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value; + SELECT array_agg(line ORDER BY ord) + INTO v_variant_lines + FROM delta_ctl.render_selection_manifest() WITH ORDINALITY AS manifest(line, ord); + v_small_new := array_position( + v_variant_lines, '# Small NEW sets receive ready-to-uncomment suggestions:'); + v_small_changed := array_position( + v_variant_lines, '# Small CHANGED sets receive ready-to-uncomment suggestions:'); + IF array_position( + v_variant_lines, + '# [bad_emails] Exact CHANGED set: rows=2 singletons=2 ranges=0') IS NULL + OR array_position(v_variant_lines, '# [bad_emails] changed.rows include=id:7,9') IS NULL + OR EXISTS ( + SELECT 1 + FROM unnest(v_variant_lines) WITH ORDINALITY AS rendered(line, ord) + WHERE ord > v_small_new AND ord < v_small_changed + AND (line LIKE '# [bad_emails] Exact NEW set:%' + OR line LIKE '# [bad_emails] new.rows include=id:%') + ) + OR array_position( + v_variant_lines, '# bad_emails has 3 NEW rows; choose selector_id from:') IS NULL THEN + RAISE EXCEPTION 'custom selector limit did not preserve exact N and generic N+1: %', + array_to_string(v_variant_lines, E'\n'); + END IF; + + -- Zero disables every exact suggestion while retaining generic guidance. + UPDATE delta_ctl.run_metadata + SET value = '0' + WHERE key = 'selector_suggestion_limit'; + SELECT array_agg(line ORDER BY ord) + INTO v_variant_lines + FROM delta_ctl.render_selection_manifest() WITH ORDINALITY AS manifest(line, ord); + v_small_new := array_position( + v_variant_lines, '# Small NEW sets receive ready-to-uncomment suggestions:'); + v_large_new := array_position( + v_variant_lines, '# Large NEW sets receive a useful pointer rather than no guidance:'); + IF v_small_new IS NULL OR v_large_new IS NULL OR EXISTS ( + SELECT 1 + FROM unnest(v_variant_lines) WITH ORDINALITY AS rendered(line, ord) + WHERE ord > v_small_new AND ord < v_large_new + AND (line ~ '^# \[[^]]+\] (new|changed)\.rows include=(id|row):' + OR line ~ '^# \[[^]]+\] Exact (NEW|CHANGED) set:' + OR line ~ '^# \[[^]]+\] Exact (NEW|CHANGED) selector unavailable') + ) THEN + RAISE EXCEPTION 'zero selector limit still emitted exact selector assistance: %', + array_to_string(v_variant_lines, E'\n'); + END IF; + SELECT count(*) INTO v_generic_count + FROM unnest(v_variant_lines) line + WHERE line ~ '^# [a-z0-9_]+ has [0-9]+ (NEW|CHANGED) rows; choose selector_id'; + IF v_generic_count <> 10 THEN + RAISE EXCEPTION 'zero selector limit did not make every positive set generic: count=% manifest=%', + v_generic_count, array_to_string(v_variant_lines, E'\n'); + END IF; + + -- Present malformed metadata fails clearly; only absence receives the fallback. + FOREACH v_invalid_value IN ARRAY ARRAY['bogus', '-1', '100001', NULL]::text[] LOOP + UPDATE delta_ctl.run_metadata + SET value = v_invalid_value + WHERE key = 'selector_suggestion_limit'; + v_error := NULL; + BEGIN + PERFORM delta_ctl.render_selection_manifest(); + EXCEPTION WHEN OTHERS THEN + v_error := SQLERRM; + END; + IF v_error IS NULL + OR position('invalid selector_suggestion_limit metadata value:' in v_error) = 0 + OR position('allowed range: 0..100000' in v_error) = 0 THEN + RAISE EXCEPTION 'invalid selector metadata % did not fail clearly: %', + COALESCE(v_invalid_value, ''), COALESCE(v_error, ''); + END IF; + END LOOP; + DELETE FROM delta_ctl.run_metadata WHERE key = 'selector_suggestion_limit'; + + UPDATE delta_ctl.run_metadata SET value = 'none' WHERE key = 'manifest_default'; + SELECT array_agg(line ORDER BY ord) + INTO v_none_lines + FROM delta_ctl.render_selection_manifest() WITH ORDINALITY AS manifest(line, ord); + + SELECT ord INTO v_wildcard + FROM unnest(v_none_lines) WITH ORDINALITY AS u(line, ord) + WHERE line LIKE '[*]%' LIMIT 1; + IF v_none_lines[v_wildcard] !~ '^\[\*\][[:space:]]+include=$' + OR v_none_lines[v_wildcard + 1] <> + '# Default selection ([*] none) currently matches 0 rows across 0 tables (new=0 changed=0).' + OR EXISTS ( + SELECT 1 FROM unnest(v_none_lines) line + WHERE line NOT LIKE '#%' AND line ~ 'include=(new|changed)' + ) THEN + RAISE EXCEPTION '--manifest-default none did not produce an empty active selection: %', + array_to_string(v_none_lines, E'\n'); + END IF; + + DELETE FROM delta_ctl.run_metadata WHERE key = 'manifest_default'; + SELECT array_agg(line ORDER BY ord) + INTO v_none_lines + FROM delta_ctl.render_selection_manifest() WITH ORDINALITY AS manifest(line, ord); + IF NOT EXISTS ( + SELECT 1 FROM unnest(v_none_lines) line + WHERE line ~ '^\[\*\][[:space:]]+include=new,changed$' + ) THEN + RAISE EXCEPTION 'missing manifest_default metadata did not preserve the historical default'; + END IF; +END $$; diff --git a/data-tool/tests/delta_restore/expected/t17_selection_cookbook_assert.sql b/data-tool/tests/delta_restore/expected/t17_selection_cookbook_assert.sql new file mode 100644 index 0000000000..8d1d4d6476 --- /dev/null +++ b/data-tool/tests/delta_restore/expected/t17_selection_cookbook_assert.sql @@ -0,0 +1,105 @@ +DO $$ +DECLARE + v_actual text[]; + c_corp_processing_include constant text := '# [corp_processing] include=new,changed'; + v_expected text[] := ARRAY[ + '# Classes: new | changed | changed_local_newer (other classes are never applyable).', + '# Selector kinds: id: for PK tables; row: for PK-less tables; corp: only where supported.', + '# Multiple includes union their matches; excludes subtract; --only-corps is a final intersection.', + '#', + '# ---------------------------------------------------------------------------', + '# OPERATOR COOKBOOK', + '# Everything below is commented documentation. Copy or uncomment only the', + '# examples you intend to use. Row selectors never enable a class by themselves.', + '# ---------------------------------------------------------------------------', + '#', + '# 1. Apply all NEW and CHANGED rows for a table:', + '#', + '# [mig_corp_batch] include=new,changed', + '#', + '# 2. Apply only NEW rows for a table:', + '#', + '# [mig_corp_batch] include=new', + '#', + '# 3. Apply only CHANGED rows for a table:', + '#', + '# [corp_processing] include=changed', + '#', + '# 4. Disable a table entirely:', + '#', + '# [mig_corp_batch] include=', + '#', + '# 5. Apply specific NEW rows by staged primary key.', + '# Lists and inclusive ranges may be mixed:', + '#', + '# [mig_corp_batch] include=new,changed', + '# [mig_corp_batch] new.rows include=id:3067,7922,8241-8242', + '#', + '# 6. Apply every NEW row except specific IDs:', + '#', + '# [mig_corp_account] include=new,changed', + '# [mig_corp_account] new.rows exclude=id:23,100-105', + '#', + '# 7. Combine multiple includes, then subtract exclusions.', + '# Include lines are unioned; exclude lines are applied afterward:', + '#', + c_corp_processing_include, + '# [corp_processing] new.rows include=id:53946-53959', + '# [corp_processing] new.rows include=corp:BC0000001,BC0000002', + '# [corp_processing] new.rows exclude=id:53947,53952', + '#', + '# 8. Apply specific CHANGED rows:', + '#', + c_corp_processing_include, + '# [corp_processing] changed.rows include=id:881,900-905', + '#', + '# 9. Apply CHANGED rows for selected corporations:', + '#', + c_corp_processing_include, + '# [corp_processing] changed.rows include=corp:BC0000001,BC0000002', + '#', + '# 10. Explicitly opt into locally newer rows.', + '# CAUTION: these may overwrite more recently modified local values:', + '#', + '# [corp_processing] include=new,changed,changed_local_newer', + '# [corp_processing] changed_local_newer.rows include=id:901', + '#', + '# 11. Select rows from a PK-less table using the staged row ordinal:', + '#', + '# [email_domain_groups] include=new', + '# [email_domain_groups] new.rows include=row:4,10-15', + '#', + '# 12. Parent dependency reminder:', + '# A selected NEW child whose preserved parent is also NEW requires that', + '# parent row to remain selected.', + '#', + '# [mig_batch] include=new', + '# [mig_batch] new.rows include=id:152', + '# [mig_corp_account] include=new', + '# [mig_corp_account] new.rows include=id:23643-23658', + '#', + '# 13. --only-corps is an additional global intersection:', + '#', + '# ./delta_restore_extract.sh ... ' || chr(92), + '# --selection-file selection.conf ' || chr(92), + '# --only-corps selected_corps.txt' + ]; + v_line_no integer; +BEGIN + SELECT array_agg(line ORDER BY ord) + INTO v_actual + FROM delta_ctl.render_selection_cookbook() WITH ORDINALITY AS cookbook(line, ord); + + IF v_actual IS DISTINCT FROM v_expected THEN + FOR v_line_no IN 1..GREATEST( + COALESCE(array_length(v_expected, 1), 0), + COALESCE(array_length(v_actual, 1), 0)) + LOOP + IF v_expected[v_line_no] IS DISTINCT FROM v_actual[v_line_no] THEN + RAISE EXCEPTION 'selection cookbook differs at line %: expected=% actual=%', + v_line_no, v_expected[v_line_no], v_actual[v_line_no]; + END IF; + END LOOP; + RAISE EXCEPTION 'selection cookbook array metadata differs despite equal lines'; + END IF; +END $$; diff --git a/data-tool/tests/delta_restore/fixtures/minimal_schema.sql b/data-tool/tests/delta_restore/fixtures/minimal_schema.sql new file mode 100644 index 0000000000..2706d2b55a --- /dev/null +++ b/data-tool/tests/delta_restore/fixtures/minimal_schema.sql @@ -0,0 +1,188 @@ +-- Minimal preserved-table schema for delta restore harness smoke tests. +-- This is intentionally smaller than colin_corps_extract_postgres_ddl but keeps +-- the columns referenced by delta table_config, classification, and apply SQL. + +DROP TABLE IF EXISTS auth_component_operation CASCADE; +DROP TABLE IF EXISTS auth_processing CASCADE; +DROP TABLE IF EXISTS colin_tracking CASCADE; +DROP TABLE IF EXISTS corp_processing CASCADE; +DROP TABLE IF EXISTS mig_corp_account CASCADE; +DROP TABLE IF EXISTS mig_corp_batch CASCADE; +DROP TABLE IF EXISTS mig_batch CASCADE; +DROP TABLE IF EXISTS mig_group CASCADE; +DROP TABLE IF EXISTS bar_corps CASCADE; +DROP TABLE IF EXISTS corps_with_third_party CASCADE; +DROP TABLE IF EXISTS exclude_corps CASCADE; +DROP TABLE IF EXISTS excluded_email_domain_patterns CASCADE; +DROP TABLE IF EXISTS excluded_email_domains CASCADE; +DROP TABLE IF EXISTS excluded_emails CASCADE; +DROP TABLE IF EXISTS bad_emails CASCADE; +DROP TABLE IF EXISTS email_domain_groups CASCADE; +DROP TABLE IF EXISTS demigrated_filings CASCADE; +DROP TABLE IF EXISTS event CASCADE; +DROP TABLE IF EXISTS corporation CASCADE; + +CREATE TABLE corporation ( + corp_num text PRIMARY KEY +); + +CREATE TABLE event ( + event_id bigint PRIMARY KEY +); + +CREATE TABLE email_domain_groups ( + email_domain text PRIMARY KEY, + group_name text +); + +CREATE TABLE bad_emails ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + email text UNIQUE, + notes text +); + +CREATE TABLE excluded_emails ( + email text PRIMARY KEY, + notes text +); + +CREATE TABLE excluded_email_domains ( + email_domain text PRIMARY KEY, + notes text +); + +CREATE TABLE excluded_email_domain_patterns ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + email_domain text NOT NULL, + local_part_pattern text NOT NULL, + notes text, + UNIQUE (email_domain, local_part_pattern) +); + +CREATE TABLE exclude_corps ( + corp_num text PRIMARY KEY, + notes text +); + +CREATE TABLE corps_with_third_party ( + corp_num text, + vendor text, + vendor_count integer, + notes text +); + +CREATE TABLE bar_corps ( + identifier text, + notes text +); + +CREATE TABLE mig_group ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + name text, + target_environment text, + source_db text +); + +CREATE TABLE mig_batch ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + mig_group_id bigint, + name text, + target_environment text +); + +CREATE TABLE mig_corp_batch ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + mig_batch_id bigint, + corp_num text +); + +CREATE TABLE mig_corp_account ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + corp_num text, + target_environment text, + account_id text, + mig_batch_id bigint +); + +CREATE TABLE corp_processing ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + corp_num text, + flow_name text, + environment text, + mig_batch_id bigint, + last_processed_event_id bigint, + failed_event_id bigint, + processed_status text, + last_error text, + last_modified timestamptz +); + +CREATE TABLE colin_tracking ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + corp_num text, + flow_name text, + environment text, + mig_batch_id bigint, + processed_status text, + last_modified timestamptz +); + +CREATE TABLE auth_processing ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + corp_num text, + flow_name text, + environment text, + operation text, + operation_scope text, + attempt_key text, + mig_batch_id bigint, + processed_status text, + last_modified timestamptz +); + +CREATE TABLE auth_component_operation ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + auth_processing_id bigint, + component_name text, + operation text, + status text, + payload text +); + +CREATE TABLE demigrated_filings ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + filing_id integer UNIQUE, + corp_num text, + filing_date timestamptz, + filing_type text, + filing_json jsonb, + payment_id integer, + transaction_id integer, + business_id integer, + submitter_id integer, + status text, + payment_completion_date timestamptz, + paper_only boolean, + completion_date timestamptz, + effective_date timestamptz, + source text, + parent_filing_id integer, + payment_status_code text, + temp_reg text, + payment_account text, + tech_correction_json jsonb, + colin_only boolean, + deletion_locked boolean, + submitter_roles text, + meta_data jsonb, + filing_sub_type text, + approval_type text, + application_date timestamptz, + notice_date timestamptz, + resubmission_date timestamptz, + hide_in_ledger boolean NOT NULL DEFAULT false, + withdrawn_filing_id integer, + withdrawal_pending boolean NOT NULL DEFAULT false, + lear_only boolean, + notes text +); diff --git a/data-tool/tests/delta_restore/fixtures/t01_identical.sql b/data-tool/tests/delta_restore/fixtures/t01_identical.sql new file mode 100644 index 0000000000..fd38af90ce --- /dev/null +++ b/data-tool/tests/delta_restore/fixtures/t01_identical.sql @@ -0,0 +1,61 @@ +-- Seed used on both source and local DB for the optional T01 identical scenario. +\set corp_num1 '''BC0000001''' +\set corp_num2 '''BC0000002''' +\set mig_flow '''migrate-corps-flow''' +\set env '''dev''' +\set completed '''COMPLETED''' +\set ts '''2026-07-01T00:00:00+00''' + +INSERT INTO corporation(corp_num) VALUES (:corp_num1), (:corp_num2); +INSERT INTO event(event_id) VALUES (1001), (1002); + +INSERT INTO email_domain_groups(email_domain, group_name) +VALUES ('example.com', 'Example'); + +INSERT INTO bad_emails(id, email, notes) +VALUES (1, 'bad@example.com', 'seed'); + +INSERT INTO excluded_emails(email, notes) +VALUES ('skip@example.com', 'seed'); + +INSERT INTO excluded_email_domains(email_domain, notes) +VALUES ('blocked.example', 'seed'); + +INSERT INTO excluded_email_domain_patterns(id, email_domain, local_part_pattern, notes) +VALUES (1, 'pattern.example', 'test%', 'seed'); + +INSERT INTO exclude_corps(corp_num, notes) +VALUES (:corp_num2, 'seed'); + +INSERT INTO corps_with_third_party(corp_num, vendor, vendor_count, notes) +VALUES (:corp_num1, 'VendorA', 1, 'seed'); + +INSERT INTO bar_corps(identifier, notes) +VALUES (:corp_num1, 'seed'); + +INSERT INTO mig_group(id, name, target_environment, source_db) +VALUES (1, :mig_flow, :env, 'COLIN'); + +INSERT INTO mig_batch(id, mig_group_id, name, target_environment) +VALUES (10, 1, 'batch-1', :env); + +INSERT INTO mig_corp_batch(id, mig_batch_id, corp_num) +VALUES (20, 10, :corp_num1); + +INSERT INTO mig_corp_account(id, corp_num, target_environment, account_id, mig_batch_id) +VALUES (30, :corp_num1, :env, 'account-1', 10); + +INSERT INTO corp_processing(id, corp_num, flow_name, environment, mig_batch_id, last_processed_event_id, failed_event_id, processed_status, last_error, last_modified) +VALUES (40, :corp_num1, :mig_flow, :env, 10, 1001, NULL, :completed, NULL, :ts); + +INSERT INTO colin_tracking(id, corp_num, flow_name, environment, mig_batch_id, processed_status, last_modified) +VALUES (50, :corp_num1, :mig_flow, :env, 10, :completed, :ts); + +INSERT INTO auth_processing(id, corp_num, flow_name, environment, operation, operation_scope, attempt_key, mig_batch_id, processed_status, last_modified) +VALUES (60, :corp_num1, 'auth-flow', :env, 'CREATE', 'BUSINESS', 'attempt-1', 10, :completed, :ts); + +INSERT INTO auth_component_operation(id, auth_processing_id, component_name, operation, status, payload) +VALUES (70, 60, 'accounts', 'CREATE', :completed, '{"ok":true}'); + +INSERT INTO demigrated_filings(id, filing_id, notes) +VALUES (1, 90001, 'seed'); diff --git a/data-tool/tests/delta_restore/fixtures/t08_blocked_fk_smoke.sql b/data-tool/tests/delta_restore/fixtures/t08_blocked_fk_smoke.sql new file mode 100644 index 0000000000..e8121b78d8 --- /dev/null +++ b/data-tool/tests/delta_restore/fixtures/t08_blocked_fk_smoke.sql @@ -0,0 +1,20 @@ +-- Focused SQL smoke for preserved-parent FK blocking. +-- Assumes 00_install.sql, 10_functions.sql, and minimal_schema.sql have already run. + +CREATE UNLOGGED TABLE delta_stage.mig_group (LIKE public.mig_group INCLUDING DEFAULTS); +ALTER TABLE delta_stage.mig_group ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; + +CREATE UNLOGGED TABLE delta_stage.mig_batch (LIKE public.mig_batch INCLUDING DEFAULTS); +ALTER TABLE delta_stage.mig_batch ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; + +INSERT INTO delta_ctl.table_config(table_name, load_phase) +VALUES ('mig_group', 20), ('mig_batch', 30); + +SELECT delta_ctl.complete_table_config(); + +-- The child references a staged/local parent ID that does not exist and is not pending NEW. +INSERT INTO delta_stage.mig_batch(id, mig_group_id, name, target_environment) +VALUES (10, 999, 'batch-with-missing-parent', 'dev'); + +SELECT delta_ctl.classify_table('mig_group'); +SELECT delta_ctl.classify_table('mig_batch'); diff --git a/data-tool/tests/delta_restore/fixtures/t09_unenforced_nk_ambiguity.sql b/data-tool/tests/delta_restore/fixtures/t09_unenforced_nk_ambiguity.sql new file mode 100644 index 0000000000..a5e9e0d2ad --- /dev/null +++ b/data-tool/tests/delta_restore/fixtures/t09_unenforced_nk_ambiguity.sql @@ -0,0 +1,23 @@ +-- Focused SQL smoke for unenforced natural-key ambiguity detection. +-- Assumes 00_install.sql, 10_functions.sql, and minimal_schema.sql have already run. + +TRUNCATE delta_ctl.table_config, delta_ctl.run_counts, delta_ctl.apply_counts, + delta_ctl.touched_tables, delta_ctl.dependency_violations; +DROP TABLE IF EXISTS delta_stage.bar_corps; +DROP TABLE IF EXISTS delta_diff.bar_corps_class; +DROP TABLE IF EXISTS delta_map.bar_corps_id_map; +TRUNCATE public.bar_corps; + +CREATE UNLOGGED TABLE delta_stage.bar_corps (LIKE public.bar_corps INCLUDING DEFAULTS); +ALTER TABLE delta_stage.bar_corps ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; + +INSERT INTO delta_ctl.table_config(table_name, load_phase) +VALUES ('bar_corps', 10); + +SELECT delta_ctl.complete_table_config(); + +-- Duplicate staged unenforced NK rows must remain AMBIGUOUS_NK after typed-key rewrite. +INSERT INTO delta_stage.bar_corps(identifier, notes) +VALUES ('BC0000001', 'stage-a'), ('BC0000001', 'stage-b'); + +SELECT delta_ctl.classify_table('bar_corps'); diff --git a/data-tool/tests/delta_restore/fixtures/t10_auth_component_local_only.sql b/data-tool/tests/delta_restore/fixtures/t10_auth_component_local_only.sql new file mode 100644 index 0000000000..1630824bc7 --- /dev/null +++ b/data-tool/tests/delta_restore/fixtures/t10_auth_component_local_only.sql @@ -0,0 +1,58 @@ +-- Focused SQL smoke for auth_component_operation LOCAL_ONLY scoping. +-- Assumes 00_install.sql, 10_functions.sql, and minimal_schema.sql have already run. +\set corp_num '''BC0000001''' +\set auth_flow '''auth-flow''' +\set env '''dev''' +\set op '''CREATE''' +\set scope '''BUSINESS''' +\set completed '''COMPLETED''' +\set ts '''2026-07-01T00:00:00+00''' +\set ok_payload '''{\"ok\":true}''' +\set accounts '''accounts''' + +TRUNCATE delta_ctl.table_config, delta_ctl.run_counts, delta_ctl.apply_counts, + delta_ctl.touched_tables, delta_ctl.dependency_violations; +DROP TABLE IF EXISTS delta_stage.auth_component_operation; +DROP TABLE IF EXISTS delta_stage.auth_processing; +DROP TABLE IF EXISTS delta_diff.auth_component_operation_class; +DROP TABLE IF EXISTS delta_diff.auth_processing_class; +DROP TABLE IF EXISTS delta_map.auth_processing_id_map; +TRUNCATE public.auth_component_operation, public.auth_processing, public.corporation; + +CREATE UNLOGGED TABLE delta_stage.auth_processing (LIKE public.auth_processing INCLUDING DEFAULTS); +ALTER TABLE delta_stage.auth_processing ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; + +CREATE UNLOGGED TABLE delta_stage.auth_component_operation (LIKE public.auth_component_operation INCLUDING DEFAULTS); +ALTER TABLE delta_stage.auth_component_operation ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; + +INSERT INTO delta_ctl.table_config(table_name, load_phase) +VALUES ('auth_processing', 50), ('auth_component_operation', 60); + +SELECT delta_ctl.complete_table_config(); + +INSERT INTO public.corporation(corp_num) VALUES (:corp_num); + +-- One staged auth_processing row matches local id=100; local id=999 is unrelated scope. +INSERT INTO public.auth_processing(id, corp_num, flow_name, environment, operation, operation_scope, attempt_key, mig_batch_id, processed_status, last_modified) +VALUES + (100, :corp_num, :auth_flow, :env, :op, :scope, 'attempt-1', NULL, :completed, :ts), + (999, :corp_num, :auth_flow, :env, :op, :scope, 'unrelated', NULL, :completed, :ts); + +INSERT INTO delta_stage.auth_processing(id, corp_num, flow_name, environment, operation, operation_scope, attempt_key, mig_batch_id, processed_status, last_modified) +VALUES + (10, :corp_num, :auth_flow, :env, :op, :scope, 'attempt-1', NULL, :completed, :ts); + +-- Local row 200 matches staged row 20. Row 201 is local-only under the matched parent. +-- Row 299 is under an unrelated local parent and should not be counted by scoped LOCAL_ONLY. +INSERT INTO public.auth_component_operation(id, auth_processing_id, component_name, operation, status, payload) +VALUES + (200, 100, :accounts, :op, :completed, :ok_payload), + (201, 100, 'contacts', :op, :completed, :ok_payload), + (299, 999, :accounts, :op, :completed, '{"unrelated":true}'); + +INSERT INTO delta_stage.auth_component_operation(id, auth_processing_id, component_name, operation, status, payload) +VALUES + (20, 10, :accounts, :op, :completed, :ok_payload); + +SELECT delta_ctl.classify_table('auth_processing'); +SELECT delta_ctl.classify_table('auth_component_operation'); diff --git a/data-tool/tests/delta_restore/fixtures/t11_nk_hash_join_perf.sql b/data-tool/tests/delta_restore/fixtures/t11_nk_hash_join_perf.sql new file mode 100644 index 0000000000..1ec77f7457 --- /dev/null +++ b/data-tool/tests/delta_restore/fixtures/t11_nk_hash_join_perf.sql @@ -0,0 +1,59 @@ +-- Synthetic-volume SQL smoke for null-safe hash-accelerated natural-key matching. +-- Assumes 00_install.sql, 10_functions.sql, and minimal_schema.sql have already run. + +TRUNCATE delta_ctl.table_config, delta_ctl.run_counts, delta_ctl.apply_counts, + delta_ctl.touched_tables, delta_ctl.dependency_violations; +DROP TABLE IF EXISTS delta_stage.bar_corps; +DROP TABLE IF EXISTS delta_diff.bar_corps_class; +DROP TABLE IF EXISTS delta_map.bar_corps_id_map; +TRUNCATE public.bar_corps; + +CREATE UNLOGGED TABLE delta_stage.bar_corps (LIKE public.bar_corps INCLUDING DEFAULTS); +ALTER TABLE delta_stage.bar_corps ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; + +INSERT INTO delta_ctl.table_config(table_name, load_phase) +VALUES ('bar_corps', 10); + +SELECT delta_ctl.complete_table_config(); + +-- Local: 50k non-null keys plus one NULL key. The NULL key must match the +-- staged NULL key below via the null-safe NK residual, not become LOCAL_ONLY/NEW. +INSERT INTO public.bar_corps(identifier, notes) +SELECT 'BC' || lpad(g::text, 7, '0'), + CASE WHEN g <= 25000 THEN 'same-' || g ELSE 'local-' || g END +FROM generate_series(1, 50000) AS g; +INSERT INTO public.bar_corps(identifier, notes) +VALUES (NULL, 'local-null'); + +-- Staged: 25k unchanged, 15k changed, 10k new, plus one changed NULL-key match. +INSERT INTO delta_stage.bar_corps(identifier, notes) +SELECT 'BC' || lpad(g::text, 7, '0'), 'same-' || g +FROM generate_series(1, 25000) AS g; + +INSERT INTO delta_stage.bar_corps(identifier, notes) +SELECT 'BC' || lpad(g::text, 7, '0'), 'stage-' || g +FROM generate_series(25001, 40000) AS g; + +INSERT INTO delta_stage.bar_corps(identifier, notes) +SELECT 'BC' || lpad(g::text, 7, '0'), 'new-' || g +FROM generate_series(50001, 60000) AS g; + +INSERT INTO delta_stage.bar_corps(identifier, notes) +VALUES (NULL, 'stage-null'); + +ANALYZE public.bar_corps; +ANALYZE delta_stage.bar_corps; + +CREATE TEMP TABLE t11_nk_hash_join_perf_timing(elapsed_ms numeric NOT NULL); + +DO $$ +DECLARE + c_table_name CONSTANT text := 'bar_corps'; + v_started timestamptz := clock_timestamp(); +BEGIN + PERFORM delta_ctl.classify_nk_table(c_table_name); + PERFORM delta_ctl.record_class_counts(c_table_name); + PERFORM delta_ctl.record_local_only_count(c_table_name); + INSERT INTO t11_nk_hash_join_perf_timing(elapsed_ms) + VALUES (EXTRACT(epoch FROM clock_timestamp() - v_started) * 1000.0); +END $$; diff --git a/data-tool/tests/delta_restore/fixtures/t12_row_selection.sql b/data-tool/tests/delta_restore/fixtures/t12_row_selection.sql new file mode 100644 index 0000000000..2ce1d2210a --- /dev/null +++ b/data-tool/tests/delta_restore/fixtures/t12_row_selection.sql @@ -0,0 +1,71 @@ +-- End-to-end row selection apply: 1 parent plus 16/16/16 selected children. +TRUNCATE delta_ctl.table_config, delta_ctl.run_counts, delta_ctl.selection, + delta_ctl.row_selection, delta_ctl.selection_diagnostics, delta_ctl.apply_counts, + delta_ctl.touched_tables, delta_ctl.dependency_violations; +DROP SCHEMA delta_stage CASCADE; +DROP SCHEMA delta_map CASCADE; +DROP SCHEMA delta_diff CASCADE; +CREATE SCHEMA delta_stage; +CREATE SCHEMA delta_map; +CREATE SCHEMA delta_diff; +TRUNCATE public.mig_group, public.mig_batch, public.mig_corp_account, + public.mig_corp_batch, public.corp_processing, public.corporation RESTART IDENTITY; + +CREATE UNLOGGED TABLE delta_stage.mig_group (LIKE public.mig_group INCLUDING DEFAULTS); +ALTER TABLE delta_stage.mig_group ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; +CREATE UNLOGGED TABLE delta_stage.mig_batch (LIKE public.mig_batch INCLUDING DEFAULTS); +ALTER TABLE delta_stage.mig_batch ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; +CREATE UNLOGGED TABLE delta_stage.mig_corp_account (LIKE public.mig_corp_account INCLUDING DEFAULTS); +ALTER TABLE delta_stage.mig_corp_account ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; +CREATE UNLOGGED TABLE delta_stage.mig_corp_batch (LIKE public.mig_corp_batch INCLUDING DEFAULTS); +ALTER TABLE delta_stage.mig_corp_batch ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; +CREATE UNLOGGED TABLE delta_stage.corp_processing (LIKE public.corp_processing INCLUDING DEFAULTS); +ALTER TABLE delta_stage.corp_processing ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; + +INSERT INTO delta_ctl.table_config(table_name, load_phase) +VALUES ('mig_group', 20), ('mig_batch', 30), ('mig_corp_batch', 40), -- NOSONAR + ('mig_corp_account', 50), ('corp_processing', 60); -- NOSONAR +SELECT delta_ctl.complete_table_config(); + +INSERT INTO public.mig_group(id, name, target_environment, source_db) +VALUES (1, 'existing-group', 'dev', 'COLIN'); +INSERT INTO delta_stage.mig_group(id, name, target_environment, source_db) +VALUES (1, 'existing-group', 'dev', 'COLIN'); +INSERT INTO delta_stage.mig_batch(id, mig_group_id, name, target_environment) +VALUES (152, 1, 'selected-parent', 'dev'); + +INSERT INTO delta_stage.mig_corp_account(id, corp_num, target_environment, account_id, mig_batch_id) +SELECT id, 'BC' || lpad(id::text, 7, '0'), 'dev', 'acct-' || id, 152 +FROM (SELECT 23 AS id UNION ALL SELECT generate_series(23643, 23658)) q; + +INSERT INTO delta_stage.mig_corp_batch(id, mig_batch_id, corp_num) +SELECT id, 152, 'MB' || lpad(id::text, 7, '0') FROM generate_series(3001, 3020) id; + +INSERT INTO public.corporation(corp_num) +SELECT 'CP' || lpad(id::text, 7, '0') FROM generate_series(4001, 4032) id; +INSERT INTO delta_stage.corp_processing( + id, corp_num, flow_name, environment, mig_batch_id, processed_status, last_modified) +SELECT id, 'CP' || lpad(id::text, 7, '0'), 'flow-' || id, 'dev', 152, 'READY', now() +FROM generate_series(4001, 4032) id; + +SELECT delta_ctl.run_preview_classification(); +INSERT INTO delta_ctl.selection(table_name, class) +SELECT table_name, 'NEW' FROM delta_ctl.table_config; -- NOSONAR +INSERT INTO delta_ctl.selection(table_name, class) +SELECT table_name, 'CHANGED' FROM delta_ctl.table_config; + +DO $$ +DECLARE + c_include_mode constant text := 'include'; +BEGIN + INSERT INTO delta_ctl.row_selection( + table_name, class, mode, kind, value_from, value_to, corp_num, is_range, source_line) + VALUES + ('mig_corp_account', 'NEW', 'exclude', 'id', 23, 23, NULL, false, 1), + ('mig_corp_batch', 'NEW', c_include_mode, 'id', 3001, 3017, NULL, true, 2), + ('mig_corp_batch', 'NEW', 'exclude', 'id', 3017, 3017, NULL, false, 3), + ('corp_processing', 'NEW', c_include_mode, 'id', 4001, 4015, NULL, true, 4), + ('corp_processing', 'NEW', c_include_mode, 'corp', NULL, NULL, 'CP0004016', false, 5); +END $$; + +SELECT delta_ctl.run_apply(); diff --git a/data-tool/tests/delta_restore/fixtures/t13_selector_diagnostics.sql b/data-tool/tests/delta_restore/fixtures/t13_selector_diagnostics.sql new file mode 100644 index 0000000000..9ddfca1c04 --- /dev/null +++ b/data-tool/tests/delta_restore/fixtures/t13_selector_diagnostics.sql @@ -0,0 +1,64 @@ +-- Semantic selector diagnostics: unsupported kinds, class gate, and zero match. +TRUNCATE delta_ctl.table_config, delta_ctl.run_counts, delta_ctl.selection, + delta_ctl.row_selection, delta_ctl.selection_diagnostics, delta_ctl.apply_counts, + delta_ctl.touched_tables, delta_ctl.dependency_violations; +DROP SCHEMA delta_stage CASCADE; DROP SCHEMA delta_map CASCADE; DROP SCHEMA delta_diff CASCADE; +CREATE SCHEMA delta_stage; CREATE SCHEMA delta_map; CREATE SCHEMA delta_diff; + +CREATE UNLOGGED TABLE delta_stage.bar_corps (LIKE public.bar_corps INCLUDING DEFAULTS); +ALTER TABLE delta_stage.bar_corps ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; +CREATE UNLOGGED TABLE delta_stage.mig_group (LIKE public.mig_group INCLUDING DEFAULTS); +ALTER TABLE delta_stage.mig_group ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; +CREATE UNLOGGED TABLE delta_stage.bad_emails (LIKE public.bad_emails INCLUDING DEFAULTS); +ALTER TABLE delta_stage.bad_emails ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; +CREATE UNLOGGED TABLE delta_stage.auth_processing (LIKE public.auth_processing INCLUDING DEFAULTS); +ALTER TABLE delta_stage.auth_processing ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; +CREATE UNLOGGED TABLE delta_stage.auth_component_operation (LIKE public.auth_component_operation INCLUDING DEFAULTS); +ALTER TABLE delta_stage.auth_component_operation ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; +CREATE UNLOGGED TABLE delta_stage.email_domain_groups (LIKE public.email_domain_groups INCLUDING DEFAULTS); +ALTER TABLE delta_stage.email_domain_groups ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; + +INSERT INTO delta_ctl.table_config(table_name, load_phase) +VALUES ('bar_corps', 10), ('mig_group', 20), ('bad_emails', 30), -- NOSONAR + ('auth_processing', 40), ('auth_component_operation', 50), -- NOSONAR + ('email_domain_groups', 60), ('excluded_emails', 70); -- NOSONAR +SELECT delta_ctl.complete_table_config(); +SELECT delta_ctl.create_class_table('bar_corps'); +SELECT delta_ctl.create_class_table('mig_group'); +SELECT delta_ctl.create_class_table('bad_emails'); +SELECT delta_ctl.create_class_table('auth_processing'); +SELECT delta_ctl.create_class_table('auth_component_operation'); +SELECT delta_ctl.create_class_table('email_domain_groups'); + +INSERT INTO delta_stage.bar_corps(identifier, notes) VALUES ('BC1', 'x'); +INSERT INTO delta_stage.mig_group(id, name, target_environment, source_db) VALUES (1, 'g', 'dev', 'C'); +INSERT INTO delta_stage.bad_emails(id, email, notes) VALUES (1, 'a@example.test', 'x'); +INSERT INTO delta_stage.auth_processing(id, corp_num) VALUES (50, 'BCOK'); +INSERT INTO delta_stage.auth_component_operation(id, auth_processing_id, component_name) +VALUES (60, 50, 'component'); +INSERT INTO delta_stage.email_domain_groups(email_domain, group_name) +VALUES ('example.test', 'test-group'); +INSERT INTO delta_diff.bar_corps_class(_delta_row_id, class) VALUES (1, 'NEW'); +INSERT INTO delta_diff.mig_group_class(_delta_row_id, staged_pk, class) VALUES (1, 1, 'NEW'); +INSERT INTO delta_diff.bad_emails_class(_delta_row_id, staged_pk, class) VALUES (1, 1, 'NEW'); +INSERT INTO delta_diff.auth_processing_class(_delta_row_id, staged_pk, class) VALUES (1, 50, 'NEW'); +INSERT INTO delta_diff.auth_component_operation_class(_delta_row_id, staged_pk, class) VALUES (1, 60, 'NEW'); +INSERT INTO delta_diff.email_domain_groups_class(_delta_row_id, class) VALUES (1, 'NEW'); + +INSERT INTO delta_ctl.selection(table_name, class) +VALUES ('bar_corps', 'NEW'), ('mig_group', 'NEW'), ('bad_emails', 'NEW'), + ('auth_processing', 'NEW'), ('auth_component_operation', 'NEW'), + ('email_domain_groups', 'NEW'), ('excluded_emails', 'NEW'); +INSERT INTO delta_ctl.only_corps(corp_num) VALUES ('BCOK'); +INSERT INTO delta_ctl.row_selection( + table_name, class, mode, kind, value_from, value_to, corp_num, is_range, source_line) +VALUES + ('bar_corps', 'NEW', 'include', 'id', 1, 1, NULL, false, 10), -- NOSONAR + ('mig_group', 'NEW', 'include', 'corp', NULL, NULL, 'BC1', false, 11), + ('bad_emails', 'CHANGED', 'include', 'id', 1, 1, NULL, false, 12), + ('bad_emails', 'NEW', 'include', 'id', 999, 999, NULL, false, 13), + ('bar_corps', 'NEW', 'include', 'corp', NULL, NULL, 'BC1', false, 14), + ('auth_component_operation', 'NEW', 'include', 'corp', NULL, NULL, 'BCOK', false, 15), + ('email_domain_groups', 'NEW', 'include', 'row', 1, 1, NULL, false, 16), + ('excluded_emails', 'NEW', 'include', 'row', 1, 1, NULL, false, 17); +SELECT delta_ctl.verify_row_selection(); diff --git a/data-tool/tests/delta_restore/fixtures/t14_parent_dependency_rows.sql b/data-tool/tests/delta_restore/fixtures/t14_parent_dependency_rows.sql new file mode 100644 index 0000000000..b92f233799 --- /dev/null +++ b/data-tool/tests/delta_restore/fixtures/t14_parent_dependency_rows.sql @@ -0,0 +1,22 @@ +-- Selected child with excluded NEW parent produces persisted sample_ids. +TRUNCATE delta_ctl.table_config, delta_ctl.run_counts, delta_ctl.selection, + delta_ctl.row_selection, delta_ctl.selection_diagnostics, delta_ctl.apply_counts, + delta_ctl.touched_tables, delta_ctl.dependency_violations; +DROP SCHEMA delta_stage CASCADE; DROP SCHEMA delta_map CASCADE; DROP SCHEMA delta_diff CASCADE; +CREATE SCHEMA delta_stage; CREATE SCHEMA delta_map; CREATE SCHEMA delta_diff; + +CREATE UNLOGGED TABLE delta_stage.mig_group (LIKE public.mig_group INCLUDING DEFAULTS); +ALTER TABLE delta_stage.mig_group ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; +CREATE UNLOGGED TABLE delta_stage.mig_batch (LIKE public.mig_batch INCLUDING DEFAULTS); +ALTER TABLE delta_stage.mig_batch ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; +INSERT INTO delta_ctl.table_config(table_name, load_phase) VALUES ('mig_group', 20), ('mig_batch', 30); +SELECT delta_ctl.complete_table_config(); +SELECT delta_ctl.create_class_table('mig_group'); +SELECT delta_ctl.create_class_table('mig_batch'); +INSERT INTO delta_stage.mig_group(id, name) VALUES (100, 'parent'); +INSERT INTO delta_stage.mig_batch(id, mig_group_id, name) VALUES (200, 100, 'child'); +INSERT INTO delta_diff.mig_group_class(_delta_row_id, staged_pk, class, selected) +VALUES (1, 100, 'NEW', false); +INSERT INTO delta_diff.mig_batch_class(_delta_row_id, staged_pk, class, selected) +VALUES (1, 200, 'NEW', true); +SELECT delta_ctl.validate_dependencies(); diff --git a/data-tool/tests/delta_restore/fixtures/t15_details.sql b/data-tool/tests/delta_restore/fixtures/t15_details.sql new file mode 100644 index 0000000000..d6254f91b6 --- /dev/null +++ b/data-tool/tests/delta_restore/fixtures/t15_details.sql @@ -0,0 +1,18 @@ +-- Value rendering, old-to-new diffs, escaping, and detail truncation. +TRUNCATE delta_ctl.table_config, delta_ctl.run_counts, delta_ctl.selection, + delta_ctl.row_selection, delta_ctl.selection_diagnostics, delta_ctl.apply_counts, + delta_ctl.touched_tables, delta_ctl.dependency_violations; +DROP SCHEMA delta_stage CASCADE; DROP SCHEMA delta_map CASCADE; DROP SCHEMA delta_diff CASCADE; +CREATE SCHEMA delta_stage; CREATE SCHEMA delta_map; CREATE SCHEMA delta_diff; +TRUNCATE public.bad_emails RESTART IDENTITY; + +CREATE UNLOGGED TABLE delta_stage.bad_emails (LIKE public.bad_emails INCLUDING DEFAULTS); +ALTER TABLE delta_stage.bad_emails ADD COLUMN _delta_row_id bigint GENERATED ALWAYS AS IDENTITY; +INSERT INTO delta_ctl.table_config(table_name, load_phase) VALUES ('bad_emails', 10); +SELECT delta_ctl.complete_table_config(); +INSERT INTO public.bad_emails(id, email, notes) VALUES (1, 'changed@example.test', 'local old'); +INSERT INTO delta_stage.bad_emails(id, email, notes) VALUES + (1, 'changed@example.test', 'dump new'), + (2, 'new@example.test', E'tab\tline\nnext\rreturn\\slash'), + (3, 'newer@example.test', 'third'); +SELECT delta_ctl.run_preview_classification(); diff --git a/data-tool/tests/delta_restore/fixtures/t16_selection_manifest.sql b/data-tool/tests/delta_restore/fixtures/t16_selection_manifest.sql new file mode 100644 index 0000000000..16148fcfea --- /dev/null +++ b/data-tool/tests/delta_restore/fixtures/t16_selection_manifest.sql @@ -0,0 +1,95 @@ +-- Selection manifest structure, default behavior, dependencies, and selector assistance. +TRUNCATE delta_ctl.table_config, delta_ctl.run_metadata, delta_ctl.run_counts, + delta_ctl.selection, delta_ctl.row_selection, delta_ctl.selection_diagnostics; +DROP SCHEMA delta_stage CASCADE; DROP SCHEMA delta_map CASCADE; DROP SCHEMA delta_diff CASCADE; +CREATE SCHEMA delta_stage; CREATE SCHEMA delta_map; CREATE SCHEMA delta_diff; + +INSERT INTO delta_ctl.run_metadata(key, value) VALUES + ('dump_sha256', 'manifest-test-sha'), + ('manifest_default', 'new,changed'); +INSERT INTO delta_ctl.table_config(table_name, load_phase, pk_col, fk_map) VALUES + ('bad_emails', 10, 'id', '{}'::jsonb), -- NOSONAR + ('selector_limit_boundary', 12, 'id', '{}'::jsonb), -- NOSONAR + ('mig_batch', 15, NULL, -- NOSONAR + '{"mig_group_id":"mig_group","external_id":"external:other.id"}'::jsonb), + ('mig_group', 20, 'id', '{}'::jsonb), -- NOSONAR + ('bar_corps', 30, NULL, '{}'::jsonb), -- NOSONAR + ('corp_processing', 40, 'id', -- NOSONAR + '{"mig_batch_id":"mig_batch","corp_num":"external:corporation.corp_num"}'::jsonb), + ('auth_component_operation', 45, 'id', -- NOSONAR + '{"auth_processing_id":"auth_processing"}'::jsonb); + +CREATE TABLE delta_stage.bad_emails ( + id bigint, + _delta_row_id bigint +); +CREATE TABLE delta_diff.bad_emails_class ( + _delta_row_id bigint, + class text +); +INSERT INTO delta_stage.bad_emails(id, _delta_row_id) VALUES + (2, 1), (3, 2), (5, 3), (7, 4), (9, 5); +INSERT INTO delta_diff.bad_emails_class(_delta_row_id, class) VALUES + (1, 'NEW'), (2, 'NEW'), (3, 'NEW'), (4, 'CHANGED'), (5, 'CHANGED'); -- NOSONAR + +CREATE TABLE delta_stage.selector_limit_boundary ( + id bigint, + _delta_row_id bigint +); +CREATE TABLE delta_diff.selector_limit_boundary_class ( + _delta_row_id bigint, + class text +); +INSERT INTO delta_stage.selector_limit_boundary(id, _delta_row_id) +SELECT CASE + -- Five separated two-value runs force wrapped range/count pairs. + WHEN n <= 10 THEN + 100000000000000::bigint + ((n - 1) / 2) * 3 + ((n - 1) % 2) + -- Forty long, separated values force more than eight singleton lines + -- and therefore exercise four-line visual paragraph breaks. + WHEN n <= 50 THEN + 2000000000000000000::bigint + (n - 11) * 2 + ELSE 3000000000000000000::bigint + END, + n +FROM generate_series(1, 51) AS ids(n); +INSERT INTO delta_diff.selector_limit_boundary_class(_delta_row_id, class) +SELECT n, CASE WHEN n <= 50 THEN 'NEW' ELSE 'UNCHANGED' END +FROM generate_series(1, 51) AS ids(n); + +CREATE TABLE delta_stage.mig_batch ( + _delta_row_id bigint +); +CREATE TABLE delta_diff.mig_batch_class ( + _delta_row_id bigint, + class text +); +INSERT INTO delta_stage.mig_batch(_delta_row_id) VALUES (4), (5), (8), (10), (11); +INSERT INTO delta_diff.mig_batch_class(_delta_row_id, class) VALUES + (4, 'NEW'), (5, 'NEW'), (8, 'NEW'), (10, 'CHANGED'), (11, 'CHANGED'); + +-- auth_component_operation supports corp: through this staged parent, not its own TSV columns. +CREATE TABLE delta_stage.auth_processing ( + id bigint, + corp_num text, + _delta_row_id bigint +); + +INSERT INTO delta_ctl.run_counts(table_name, count_name, row_count) VALUES + ('bad_emails', 'STAGED', 5), -- NOSONAR + ('bad_emails', 'NEW', 3), + ('bad_emails', 'CHANGED', 2), + ('selector_limit_boundary', 'STAGED', 51), + ('selector_limit_boundary', 'NEW', 50), + ('mig_batch', 'STAGED', 5), + ('mig_batch', 'NEW', 3), + ('mig_batch', 'CHANGED', 2), + ('mig_group', 'STAGED', 51), + ('mig_group', 'NEW', 51), + ('bar_corps', 'STAGED', 52), + ('bar_corps', 'NEW', 52), + ('corp_processing', 'STAGED', 106), + ('corp_processing', 'NEW', 53), + ('corp_processing', 'CHANGED', 53), + ('auth_component_operation', 'STAGED', 54), + ('auth_component_operation', 'NEW', 54); diff --git a/data-tool/tests/delta_restore/fixtures/t17_selection_cookbook.sql b/data-tool/tests/delta_restore/fixtures/t17_selection_cookbook.sql new file mode 100644 index 0000000000..798db06e91 --- /dev/null +++ b/data-tool/tests/delta_restore/fixtures/t17_selection_cookbook.sql @@ -0,0 +1,2 @@ +-- The selection cookbook is a static grammar reference and must not depend on run state. +TRUNCATE delta_ctl.run_metadata, delta_ctl.run_counts; diff --git a/data-tool/tests/delta_restore/run_tests.sh b/data-tool/tests/delta_restore/run_tests.sh new file mode 100755 index 0000000000..d8ec5071aa --- /dev/null +++ b/data-tool/tests/delta_restore/run_tests.sh @@ -0,0 +1,1088 @@ +#!/usr/bin/env bash +# Lightweight delta restore harness. +# +# Static/AWK checks always run. PostgreSQL-backed checks run only when local +# libpq tooling is present and reachable; otherwise they are skipped with a +# clear message so this harness is safe on workstations/CI without Postgres. + +set -u -o pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +DATA_TOOL_DIR="$ROOT_DIR/data-tool" +TEST_DIR="$DATA_TOOL_DIR/tests/delta_restore" +FIXTURES_DIR="$TEST_DIR/fixtures" +EXPECTED_DIR="$TEST_DIR/expected" +TMP_BASE="${TMPDIR:-/tmp}/delta_restore_tests.$$" + +PGHOST="${PGHOST:-localhost}" +PGPORT="${PGPORT:-5432}" +PGUSER="${PGUSER:-postgres}" +PG_BIN="${PG_BIN:-}" +PGDATABASE_ADMIN="${DELTA_RT_ADMIN_DB:-postgres}" +KEEP_DBS="${DELTA_RT_KEEP_DBS:-false}" +RUN_INTEGRATION="${DELTA_RT_RUN_INTEGRATION:-false}" + +pass_count=0 +fail_count=0 +skip_count=0 +created_dbs="" + +usage() { + cat <<'USAGE' +Usage: run_tests.sh [--integration] [--static-only] [--keep-dbs] [-h|--help] + +Checks: + static Shell syntax plus guarded COPY-header and detail-alignment AWK paths. + sql-smoke If local Postgres tooling is available: compile/install delta SQL + and run a focused preserved-parent BLOCKED_FK smoke fixture. + integration Optional: create source/local throwaway DBs with minimal DDL, + run backup -> preview -> apply for an identical dump scenario. + +Environment: + PGHOST, PGPORT, PGUSER, PGPASSWORD/PGPASSFILE libpq connection settings + PG_BIN=/path/to/postgres/bin optional PostgreSQL client binary directory + DELTA_RT_ADMIN_DB=postgres admin DB for createdb checks + DELTA_RT_RUN_INTEGRATION=true same as --integration + DELTA_RT_KEEP_DBS=true keep throwaway DBs on failure + DELTA_RT_T11_NK_HASH_JOIN_PERF_MAX_MS=60000 t11 runtime ceiling in milliseconds +USAGE + return 0 +} + +while [[ "$#" -gt 0 ]]; do + case "$1" in + --integration) RUN_INTEGRATION="true"; shift ;; + --static-only) RUN_INTEGRATION="false"; PG_SMOKE_DISABLED="true"; shift ;; + --keep-dbs) KEEP_DBS="true"; shift ;; + -h|--help) usage; exit 0 ;; + *) printf >&2 "unknown argument: %s\n" "$1"; usage >&2; exit 2 ;; + esac +done + +mkdir -p "$TMP_BASE" + +pg_tool() { + local tool="$1" + if [[ -n "$PG_BIN" ]]; then + printf '%s/%s' "${PG_BIN%/}" "$tool" + else + printf '%s' "$tool" + fi + return 0 +} + +PSQL_BIN="$(pg_tool psql)" +PG_DUMP_BIN="$(pg_tool pg_dump)" +PG_RESTORE_BIN="$(pg_tool pg_restore)" +CREATEDB_BIN="$(pg_tool createdb)" +DROPDB_BIN="$(pg_tool dropdb)" + +log() { printf '%s\n' "$*"; return 0; } +pass() { pass_count=$((pass_count + 1)); printf 'ok - %s\n' "$*"; return 0; } +fail() { fail_count=$((fail_count + 1)); printf 'not ok - %s\n' "$*"; return 0; } +skip() { skip_count=$((skip_count + 1)); printf 'skip - %s\n' "$*"; return 0; } + +cleanup() { + local db + if [[ "$KEEP_DBS" != "true" ]]; then + for db in $created_dbs; do + "$DROPDB_BIN" -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" --if-exists "$db" >/dev/null 2>&1 || true + done + elif [[ -n "$created_dbs" ]]; then + printf 'Keeping throwaway DBs: %s\n' "$created_dbs" + fi + rm -rf "$TMP_BASE" + return 0 +} +trap cleanup EXIT + +require_file() { + local path="$1" + [[ -f "$path" ]] || { fail "missing required file: $path"; return 1; } + return 0 +} + +path_mode() { + local path="$1" + if stat -f '%Lp' "$path" >/dev/null 2>&1; then + stat -f '%Lp' "$path" + else + stat -c '%a' "$path" + fi + return 0 +} + +run_static_checks() { + log '== static checks ==' + local ok=true + for f in \ + "$DATA_TOOL_DIR/delta_restore_extract.sh" \ + "$DATA_TOOL_DIR/backup_extract_tables.sh" \ + "$DATA_TOOL_DIR/restore_extract.sh" \ + "$TEST_DIR/run_tests.sh"; do + if bash -n "$f"; then + pass "bash -n ${f#$ROOT_DIR/}" + else + fail "bash -n ${f#$ROOT_DIR/}" + ok=false + fi + done + + local delta_usage + delta_usage="$( + unset PG_BIN + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + usage + )" + if grep -q '^Environment:$' <<<"$delta_usage" \ + && grep -Fq 'PG_BIN=/path/to/postgresql/bin' <<<"$delta_usage" \ + && grep -Fq 'optional PostgreSQL client-tools directory; unset or empty uses PATH' <<<"$delta_usage" \ + && ! grep -Fq '/opt/homebrew/opt/postgresql@15/bin/' <<<"$delta_usage" \ + && grep -q 'LOCK_TIMEOUT_SECONDS=30' <<<"$delta_usage" \ + && grep -q '^After run-directory initialization, each invocation prints its artifact directory on exit:$' <<<"$delta_usage" \ + && grep -q 'Artifacts retained for inspection: ' <<<"$delta_usage"; then + pass 'delta restore help documents portable PG_BIN, lock timeout, and artifact handoff' + else + fail 'delta restore environment/artifact help contract' + ok=false + fi + + if ( + unset PG_BIN + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + [[ -z "$PG_BIN" ]] + [[ "$(pg_tool psql)" = "psql" ]] + [[ "$PSQL_BIN" = "psql" ]] + [[ "$PG_RESTORE_BIN" = "pg_restore" ]] + ) && ( + PG_BIN="" + export PG_BIN + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + [[ -z "$PG_BIN" ]] + [[ "$(pg_tool psql)" = "psql" ]] + [[ "$PSQL_BIN" = "psql" ]] + [[ "$PG_RESTORE_BIN" = "pg_restore" ]] + ) && ( + PG_BIN="/custom/postgresql/bin/" + export PG_BIN + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + [[ "$PG_BIN" = "/custom/postgresql/bin/" ]] + [[ "$(pg_tool psql)" = "/custom/postgresql/bin/psql" ]] + [[ "$PSQL_BIN" = "/custom/postgresql/bin/psql" ]] + [[ "$PG_RESTORE_BIN" = "/custom/postgresql/bin/pg_restore" ]] + ); then + pass 'delta PG_BIN resolver handles unset, empty, and explicit directory states' + else + fail 'delta PG_BIN resolver behavior' + ok=false + fi + + local private_tmp + private_tmp="$TMP_BASE/private_artifacts" + if ( + umask 022 + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + mkdir -p "$private_tmp/run" + : > "$private_tmp/run/artifact.txt" + [[ "$(path_mode "$private_tmp/run")" = "700" ]] + [[ "$(path_mode "$private_tmp/run/artifact.txt")" = "600" ]] + ); then + pass 'delta restore artifacts default to private directory/file permissions' + else + fail 'delta restore private artifact permissions' + ok=false + fi + + local awk_tmp expected sidecar out err aligned expected_aligned canonical + awk_tmp="$TMP_BASE/awk" + mkdir -p "$awk_tmp" + expected="$awk_tmp/expected.txt" + sidecar="$awk_tmp/sidecar.tsv" + out="$awk_tmp/out.sql" + err="$awk_tmp/err.txt" + printf 'mig_group\n' > "$expected" + + cat > "$awk_tmp/happy.sql" <<'SQL' +SET transaction_timeout = 0; +COPY public.mig_group (id, name, target_environment, source_db) FROM stdin; +1 group dev COLIN +\. +SQL + if awk -f "$DATA_TOOL_DIR/scripts/restore/delta/rewrite_copy_targets.awk" \ + -v sidecar="$sidecar" -v expected="$expected" \ + "$awk_tmp/happy.sql" > "$out" 2> "$err" \ + && grep -q 'COPY delta_stage.mig_group' "$out" \ + && grep -q 'skipped unsupported client-only setting' "$out" \ + && grep -q $'mig_group\tid,name,target_environment,source_db' "$sidecar"; then + pass 'AWK rewrite happy path captures sidecar, retargets COPY, and filters unsupported SET' + else + fail 'AWK rewrite happy path' + ok=false + fi + + cat > "$awk_tmp/unexpected.sql" <<'SQL' +COPY public.not_preserved (id) FROM stdin; +1 +\. +SQL + if awk -f "$DATA_TOOL_DIR/scripts/restore/delta/rewrite_copy_targets.awk" \ + -v sidecar="$sidecar" -v expected="$expected" \ + "$awk_tmp/unexpected.sql" > "$out" 2> "$err"; then + fail 'AWK rewrite rejects unexpected COPY table' + ok=false + elif grep -q 'not expected' "$err"; then + pass 'AWK rewrite rejects unexpected COPY table' + else + fail 'AWK rewrite failure path produced unclear error' + ok=false + fi + + cat > "$awk_tmp/details.tsv" <<'TSV' +id name note +1 Ada short +22 Longer 1234567890 +# TRUNCATED at 2 rows +TSV + cat > "$awk_tmp/details.expected.txt" <<'TXT' +id name note +-- ------ -------- +1 Ada short +22 Longer 1234567… +# TRUNCATED at 2 rows +TXT + aligned="$awk_tmp/details.txt" + expected_aligned="$awk_tmp/details.expected.txt" + canonical="$awk_tmp/details.canonical.tsv" + cp "$awk_tmp/details.tsv" "$canonical" + if awk -v max_width=8 -f "$DATA_TOOL_DIR/scripts/restore/delta/align_details.awk" \ + "$awk_tmp/details.tsv" "$awk_tmp/details.tsv" > "$aligned" \ + && cmp -s "$expected_aligned" "$aligned" \ + && awk -v max_width=8 -f "$DATA_TOOL_DIR/scripts/restore/delta/align_details.awk" \ + "$awk_tmp/details.tsv" "$awk_tmp/details.tsv" | cmp -s "$expected_aligned" -; then + pass 'detail aligner pads columns, clips cells, preserves sentinel, and is deterministic' + else + fail 'detail aligner formatting contract' + ok=false + fi + + : > "$awk_tmp/empty.tsv" + if awk -f "$DATA_TOOL_DIR/scripts/restore/delta/align_details.awk" \ + "$awk_tmp/empty.tsv" "$awk_tmp/empty.tsv" > "$aligned" \ + && [[ ! -s "$aligned" ]]; then + pass 'detail aligner produces no output for empty input' + else + fail 'detail aligner empty-input contract' + ok=false + fi + + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + parse_args --mode validate --no-aligned-details --align-width 3 --manifest-default none + [[ "$MODE" = "validate" ]] + [[ "$ALIGN_DETAILS" = "false" && "$ALIGN_WIDTH" = "6" ]] + [[ "$MANIFEST_DEFAULT" = "none" ]] + ); then + pass 'validate mode, aligned-detail, and manifest-default CLI controls parse valid values' + else + fail 'validate mode, aligned-detail, or manifest-default CLI control parsing' + ok=false + fi + + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + [[ "$SELECTOR_SUGGESTION_LIMIT" = "50" ]] + parse_args --selector-suggestion-limit 0 + [[ "$SELECTOR_SUGGESTION_LIMIT" = "0" ]] + parse_args --selector-suggestion-limit 37 + [[ "$SELECTOR_SUGGESTION_LIMIT" = "37" ]] + parse_args --selector-suggestion-limit 00007 + [[ "$SELECTOR_SUGGESTION_LIMIT" = "7" ]] + parse_args --selector-suggestion-limit 100000 + [[ "$SELECTOR_SUGGESTION_LIMIT" = "100000" ]] + ); then + pass 'selector-suggestion-limit accepts default, zero, custom, canonical, and maximum values' + else + fail 'selector-suggestion-limit valid-value parsing or canonicalization' + ok=false + fi + + selector_limit_rejections=true + for invalid_limit in -1 text +1 1.5 100001 999999999999999999999999999999; do + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + parse_args --selector-suggestion-limit "$invalid_limit" + ) >/dev/null 2>&1; then + selector_limit_rejections=false + fi + done + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + parse_args --selector-suggestion-limit + ) >/dev/null 2>&1; then + selector_limit_rejections=false + fi + if [[ "$selector_limit_rejections" = "true" ]]; then + pass 'selector-suggestion-limit rejects negative, malformed, above-cap, huge, and missing values' + else + fail 'selector-suggestion-limit accepted an invalid value' + ok=false + fi + + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + help_text="$(usage)" + [[ "$help_text" = *'--selector-suggestion-limit '* ]] + [[ "$help_text" = *'exact manifest suggestions per table/class; default: 50'* ]] + [[ "$help_text" = *'0 disables exact suggestions; maximum: 100000'* ]] + ); then + pass 'selector-suggestion-limit help documents scope, default, zero, and maximum' + else + fail 'selector-suggestion-limit help contract' + ok=false + fi + + ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + parse_args --mode verify + ) >/dev/null 2>&1 + if [[ "$?" -ne 0 ]]; then + pass 'mode CLI rejects unsupported values while accepting validate' + else + fail 'mode CLI accepted an unsupported value' + ok=false + fi + ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + parse_args --align-width 0 + ) >/dev/null 2>&1 + if [[ "$?" -ne 0 ]]; then + pass 'aligned-detail CLI rejects a zero width' + else + fail 'aligned-detail CLI accepted a zero width' + ok=false + fi + ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + parse_args --manifest-default changed + ) >/dev/null 2>&1 + if [[ "$?" -ne 0 ]]; then + pass 'manifest-default CLI rejects unsupported defaults' + else + fail 'manifest-default CLI accepted an unsupported default' + ok=false + fi + printf 'resident dump fixture\n' > "$awk_tmp/resident.dump" + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + RUN_DIR="$awk_tmp" + DUMP="$awk_tmp/resident.dump" + DUMP_SHA256="" + compute_dump_sha + [[ -n "$DUMP_SHA256" ]] + [[ "$DUMP_SHA256" = "$(sha256_file "$DUMP")" ]] + [[ "$(cat "$RUN_DIR/dump.sha256")" = "$DUMP_SHA256" ]] + ); then + pass 'dump sha helper safely sets the shared hash and writes its binding artifact' + else + fail 'dump sha helper shared-state contract' + ok=false + fi + + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + apply_body="$(declare -f run_apply_mode)" + [[ "$apply_body" = *'verify_dump_and_manifest'* ]] + [[ "$apply_body" = *'filter_toc'* ]] + [[ "$apply_body" = *'install_control_schemas'* ]] + [[ "$apply_body" = *'stream_stage_data'* ]] + [[ "$apply_body" = *'run_preview_classification'* ]] + [[ "$apply_body" = *'prepare_apply_selection'* ]] + ); then + pass 'apply retains full dump verification, restaging, and reclassification posture' + else + fail 'apply full-restage posture changed' + ok=false + fi + + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + validate_body="$(declare -f run_validate_mode)" + [[ "$validate_body" = *'verify_resident_run'* ]] + [[ "$validate_body" = *'load_preserved_config'* ]] + [[ "$validate_body" = *'select_tables'* ]] + [[ "$validate_body" = *'prepare_apply_selection'* ]] + [[ "$validate_body" != *'filter_toc'* ]] + [[ "$validate_body" != *'install_control_schemas'* ]] + [[ "$validate_body" != *'stream_stage_data'* ]] + [[ "$validate_body" != *'run_preview_classification'* ]] + ); then + pass 'validate reuses resident selection preparation without staging or classification' + else + fail 'validate resident-only dispatch contract' + ok=false + fi + + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + RUN_DIR="$awk_tmp" + MODE="preview" + DUMP="/tmp/manifest-default.dump" + DUMP_SHA256="manifest-default-sha" + MANIFEST_DEFAULT="none" + psql_file() { :; return 0; } + record_metadata + grep -Fq "('manifest_default', 'none')" "$RUN_DIR/metadata.sql" + grep -Fq "('selector_suggestion_limit', '50')" "$RUN_DIR/metadata.sql" + ); then + pass 'manifest and default selector limits are persisted into run metadata for SQL rendering' + else + fail 'manifest-default or selector-limit run metadata plumbing' + ok=false + fi + + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + parse_args --selector-suggestion-limit 00000 + RUN_DIR="$awk_tmp" + MODE="preview" + DUMP="/tmp/selector-limit.dump" + DUMP_SHA256="selector-limit-sha" + psql_file() { :; return 0; } + record_metadata + grep -Fq "('selector_suggestion_limit', '0')" "$RUN_DIR/metadata.sql" + ); then + pass 'canonical custom selector limit is persisted into run metadata' + else + fail 'custom selector-limit metadata plumbing' + ok=false + fi + + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + manifest_body="$(declare -f write_selection_manifest)" + [[ "$manifest_body" = *'SELECT * FROM delta_ctl.render_selection_manifest();'* ]] + [[ "$manifest_body" != *'$SELECTOR_SUGGESTION_LIMIT'* ]] + ); then + pass 'selection manifest keeps the zero-argument SQL renderer invocation' + else + fail 'selection manifest renderer invocation changed signature or gained shell interpolation' + ok=false + fi + + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + RUN_DIR="$awk_tmp" + ALIGN_DETAILS="true" + ALIGN_WIDTH="8" + write_aligned_detail "$awk_tmp/details.tsv" "10" "NEW" + cmp -s "$canonical" "$awk_tmp/details.tsv" + grep -q '^# rendered 2 of 10 NEW rows — TRUNCATED$' "$awk_tmp/details.txt" + ); then + pass 'aligned companion preserves TSV bytes and appends rendered/total footer' + else + fail 'aligned companion TSV preservation and footer semantics' + ok=false + fi + + rm -f "$awk_tmp/details.txt" + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + RUN_DIR="$awk_tmp" + ALIGN_DETAILS="false" + write_aligned_detail "$awk_tmp/details.tsv" "10" "NEW" + [[ ! -e "$awk_tmp/details.txt" ]] + ); then + pass 'aligned companion generation can be disabled' + else + fail 'aligned companion disable control' + ok=false + fi + + local parser_tmp="$TMP_BASE/selection_parser" + mkdir -p "$parser_tmp" + printf 'bad_emails\nbar_corps\n' > "$parser_tmp/all_conf_tables.txt" + cat > "$parser_tmp/happy.conf" <<'CONF' +# delta-selection v2 +# dump_sha256=abc123 +# staged bad_emails=3 bar_corps=2 +# Operator cookbook (all examples remain comments) +# Class override: [
] include=new,changed +# Disable a table: [
] include= +# [bad_emails] 51 NEW rows: review details/bad_emails.new.tsv; supported selectors: id: +[*] include=new,changed +[bad_emails] new.rows include=id:1-3,8,9007199254740993,9223372036854775807 +[bar_corps] changed.rows exclude=corp:BC0000001,BC\N +CONF + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + RUN_DIR="$parser_tmp"; SELECTION_FILE="$parser_tmp/happy.conf" + build_selection_input + grep -q $'bad_emails\tNEW\tinclude\tid\t1\t3\t\\N\tt' "$RUN_DIR/row_selection_input.tsv" + grep -q $'bad_emails\tNEW\tinclude\tid\t9223372036854775807\t9223372036854775807' "$RUN_DIR/row_selection_input.tsv" + grep -q $'bar_corps\tCHANGED\texclude\tcorp\t\\N\t\\N\tBC0000001' "$RUN_DIR/row_selection_input.tsv" + awk -F '\t' '$4 == "corp" && $7 == "BC\\\\N" { found=1 } END { exit !found }' "$RUN_DIR/row_selection_input.tsv" + grep -q $'dump_sha256\tabc123' "$RUN_DIR/selection_header.tsv" + [[ "$(wc -l < "$RUN_DIR/selection_header.tsv" | tr -d ' ')" = "3" ]] + ! grep -q '
\|51 NEW rows' "$RUN_DIR/selection_input.tsv" "$RUN_DIR/row_selection_input.tsv" + ); then + pass 'selection parser accepts v2 selectors/bindings and ignores generated cookbook comments' + else + fail 'selection parser v2 happy path and cookbook comment compatibility' + ok=false + fi + + printf 'bad_emails\n' > "$parser_tmp/requested_tables.txt" + printf 'bad_emails\tNEW\n' > "$parser_tmp/scoped.expected.tsv" + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + RUN_DIR="$parser_tmp" + SELECTION_FILE="$parser_tmp/happy.conf" + TABLES_ARG="bad_emails" + INCLUDE_CLASSES="new,changed" + EXCLUDE_CLASSES="changed" + build_selection_input + cmp -s "$RUN_DIR/scoped.expected.tsv" "$RUN_DIR/selection_input.tsv" + ); then + pass 'selection file candidates are bounded by explicit table/include/exclude scope' + else + fail 'selection file table/class scope ceiling' + ok=false + fi + + cat > "$parser_tmp/duplicate_sha.conf" <<'CONF' +# dump_sha256=abc123 +# dump_sha256=abc123 +[*] include=new +CONF + cat > "$parser_tmp/duplicate_staged.conf" <<'CONF' +# staged bad_emails=3 +# staged bar_corps=2 bad_emails=3 +[*] include=new +CONF + cat > "$parser_tmp/nondecimal_staged.conf" <<'CONF' +# staged bad_emails=three +[*] include=new +CONF + local bad_header expected_error parser_status + for bad_header in duplicate_sha duplicate_staged nondecimal_staged; do + case "$bad_header" in + duplicate_sha) expected_error='duplicate dump_sha256 header' ;; + duplicate_staged) expected_error='duplicate staged-count header' ;; + nondecimal_staged) expected_error='staged count must be decimal' ;; + *) expected_error='unexpected parser fixture' ;; + esac + ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + RUN_DIR="$parser_tmp"; SELECTION_FILE="$parser_tmp/$bad_header.conf" + build_selection_input + ) >/dev/null 2>&1 + parser_status=$? + if [[ "$parser_status" -eq 4 ]]; then + pass "selection parser rejects $expected_error with exit 4" + else + fail "selection parser $expected_error expected exit 4, got $parser_status" + ok=false + fi + done + + cat > "$parser_tmp/inverted.conf" <<'CONF' +[*] include=new +[bad_emails] new.rows include=id:9-3 +CONF + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + RUN_DIR="$parser_tmp"; SELECTION_FILE="$parser_tmp/inverted.conf" + build_selection_input + ) >/dev/null 2>&1; then + fail 'selection parser rejects inverted ranges' + ok=false + else + pass 'selection parser rejects inverted ranges' + fi + + cat > "$parser_tmp/bad_kind.conf" <<'CONF' +[*] include=new +[bad_emails] new.rows include=pk:1 +CONF + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + RUN_DIR="$parser_tmp"; SELECTION_FILE="$parser_tmp/bad_kind.conf" + build_selection_input + ) >/dev/null 2>&1; then + fail 'selection parser rejects unsupported selector kinds' + ok=false + else + pass 'selection parser rejects unsupported selector kinds' + fi + + cat > "$parser_tmp/bigint_overflow.conf" <<'CONF' +[*] include=new +[bad_emails] new.rows include=id:9223372036854775808 +CONF + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + RUN_DIR="$parser_tmp"; SELECTION_FILE="$parser_tmp/bigint_overflow.conf" + build_selection_input + ) >/dev/null 2>&1; then + fail 'selection parser rejects integers above bigint maximum' + ok=false + else + pass 'selection parser rejects integers above bigint maximum' + fi + + cat > "$parser_tmp/v1.conf" <<'CONF' +[*] include=new,changed +[bar_corps] include= +CONF + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + RUN_DIR="$parser_tmp"; SELECTION_FILE="$parser_tmp/v1.conf" + build_selection_input + [[ ! -s "$RUN_DIR/row_selection_input.tsv" ]] + grep -q $'bad_emails\tNEW' "$RUN_DIR/selection_input.tsv" + ); then + pass 'selection parser preserves class-only v1 compatibility' + else + fail 'selection parser class-only v1 compatibility' + ok=false + fi + + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + RUN_DIR="$parser_tmp"; DUMP_SHA256="current" + printf 'bad_emails\tNEW\tinclude\tid\t1\t1\t\\N\tf\t1\n' > "$RUN_DIR/row_selection_input.tsv" + printf 'dump_sha256\tdifferent\n' > "$RUN_DIR/selection_header.tsv" + verify_selection_binding + ) >/dev/null 2>&1; then + fail 'selection binding rejects a different dump hash' + ok=false + else + pass 'selection binding rejects a different dump hash' + fi + + if ( + DELTA_RESTORE_SOURCE_ONLY=true source "$DATA_TOOL_DIR/delta_restore_extract.sh" + RUN_DIR="$parser_tmp"; DUMP_SHA256="current" + printf 'bar_corps\tNEW\tinclude\trow\t1\t1\t\\N\tf\t1\n' > "$RUN_DIR/row_selection_input.tsv" + printf 'dump_sha256\tcurrent\nstaged\tbar_corps\t9\n' > "$RUN_DIR/selection_header.tsv" + printf 'bar_corps\tSTAGED\t8\n' > "$RUN_DIR/stage_counts.tsv" + verify_selection_binding + ) >/dev/null 2>&1; then + fail 'row selector binding rejects a changed staged count' + ok=false + else + pass 'row selector binding rejects a changed staged count' + fi + + [[ "$ok" = "true" ]] + return $? +} + +tool_available() { + local tool="$1" + case "$tool" in + */*) [[ -x "$tool" ]] ;; + *) command -v "$tool" >/dev/null 2>&1 ;; + esac + return $? +} + +have_pg_tools() { + tool_available "$PSQL_BIN" && \ + tool_available "$CREATEDB_BIN" && \ + tool_available "$DROPDB_BIN" && \ + tool_available "$PG_DUMP_BIN" && \ + tool_available "$PG_RESTORE_BIN" + return $? +} + +can_connect_pg() { + "$PSQL_BIN" -X -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE_ADMIN" -v ON_ERROR_STOP=1 -qAt -c 'SELECT 1' >/dev/null 2>&1 + return $? +} + +create_db() { + local db="$1" + "$DROPDB_BIN" -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" --if-exists "$db" >/dev/null 2>&1 || true + "$CREATEDB_BIN" -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -T template0 "$db" + created_dbs="$created_dbs $db" + return 0 +} + +psql_db() { + local db="$1" + shift + "$PSQL_BIN" -X -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$db" -v ON_ERROR_STOP=1 "$@" + return $? +} + +run_sql_smoke() { + log '== sql smoke ==' + if [[ "${PG_SMOKE_DISABLED:-false}" = "true" ]]; then + skip 'Postgres SQL smoke disabled by --static-only' + return 0 + fi + if ! have_pg_tools; then + skip 'Postgres SQL smoke: psql/createdb/dropdb/pg_dump/pg_restore not all available; set PG_BIN if needed' + return 0 + fi + if ! can_connect_pg; then + skip "Postgres SQL smoke: cannot connect to $PGHOST:$PGPORT/$PGDATABASE_ADMIN as $PGUSER" + return 0 + fi + + local db="delta_rt_compile_$$" + if ! create_db "$db"; then + skip "Postgres SQL smoke: could not create throwaway DB $db" + return 0 + fi + + if psql_db "$db" -v "t11_nk_hash_join_perf_max_ms=${DELTA_RT_T11_NK_HASH_JOIN_PERF_MAX_MS:-60000}" \ + -f "$DATA_TOOL_DIR/scripts/restore/delta/00_install.sql" \ + -f "$DATA_TOOL_DIR/scripts/restore/delta/10_functions.sql" \ + -f "$FIXTURES_DIR/minimal_schema.sql" \ + -f "$FIXTURES_DIR/t08_blocked_fk_smoke.sql" \ + -f "$EXPECTED_DIR/t08_blocked_fk_assert.sql" \ + -f "$FIXTURES_DIR/t09_unenforced_nk_ambiguity.sql" \ + -f "$EXPECTED_DIR/t09_unenforced_nk_ambiguity_assert.sql" \ + -f "$FIXTURES_DIR/t10_auth_component_local_only.sql" \ + -f "$EXPECTED_DIR/t10_auth_component_local_only_assert.sql" \ + -f "$FIXTURES_DIR/t11_nk_hash_join_perf.sql" \ + -f "$EXPECTED_DIR/t11_nk_hash_join_perf_assert.sql" \ + -f "$FIXTURES_DIR/t12_row_selection.sql" \ + -f "$EXPECTED_DIR/t12_row_selection_assert.sql" \ + -f "$FIXTURES_DIR/t13_selector_diagnostics.sql" \ + -f "$EXPECTED_DIR/t13_selector_diagnostics_assert.sql" \ + -f "$FIXTURES_DIR/t14_parent_dependency_rows.sql" \ + -f "$EXPECTED_DIR/t14_parent_dependency_rows_assert.sql" \ + -f "$FIXTURES_DIR/t15_details.sql" \ + -f "$EXPECTED_DIR/t15_details_assert.sql" \ + -f "$FIXTURES_DIR/t16_selection_manifest.sql" \ + -f "$EXPECTED_DIR/t16_selection_manifest_assert.sql" \ + -f "$FIXTURES_DIR/t17_selection_cookbook.sql" \ + -f "$EXPECTED_DIR/t17_selection_cookbook_assert.sql" >/dev/null; then + pass 'SQL compile + classification performance smoke' + else + fail 'SQL compile + classification performance smoke' + return 1 + fi +} + +latest_run_dir() { + local base="$1" + find "$base" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort | tail -n 1 + return $? +} + +run_optional_integration() { + log '== optional integration ==' + if [[ "$RUN_INTEGRATION" != "true" ]]; then + skip 'integration: set DELTA_RT_RUN_INTEGRATION=true or pass --integration to run backup/preview/apply smoke' + return 0 + fi + if ! have_pg_tools || ! can_connect_pg; then + skip 'integration: Postgres tooling/connectivity unavailable' + return 0 + fi + + local src="delta_rt_src_$$" local_db="delta_rt_local_$$" dump_dir report_base dump preview_run apply_run + local scope_dump scope_preview hint_dir hint_only_corps preview_stdout validate_command apply_command + local hint_validate_stdout hint_validate_run hint_validate_status hint_validate_counts + local hint_apply_stdout hint_apply_run hint_apply_status hint_apply_counts + dump_dir="$TMP_BASE/dumps" + report_base="$TMP_BASE/reports" + hint_dir="$TMP_BASE/preview-hint" + hint_only_corps="$hint_dir/only-corps.txt" + preview_stdout="$hint_dir/preview.out" + hint_validate_stdout="$hint_dir/validate.out" + hint_validate_counts="$hint_dir/validate-selected-counts.tsv" + hint_apply_stdout="$hint_dir/apply.out" + hint_apply_counts="$hint_dir/apply-selected-counts.tsv" + mkdir -p "$dump_dir" "$report_base" "$hint_dir" + printf 'BC0000001\n' > "$hint_only_corps" + dump="$dump_dir/keep.dump" + + create_db "$src" || { skip "integration: could not create $src"; return 0; } + create_db "$local_db" || { skip "integration: could not create $local_db"; return 0; } + + psql_db "$src" -f "$FIXTURES_DIR/minimal_schema.sql" -f "$FIXTURES_DIR/t01_identical.sql" >/dev/null || { fail 'integration: seed source DB'; return 1; } + psql_db "$local_db" -f "$FIXTURES_DIR/minimal_schema.sql" -f "$FIXTURES_DIR/t01_identical.sql" >/dev/null || { fail 'integration: seed local DB'; return 1; } + + if (cd "$DATA_TOOL_DIR" && PG_BIN="$PG_BIN" PGDATABASE="$src" BACKUP_DIR="$dump_dir" DUMP="$dump" ./backup_extract_tables.sh >/dev/null); then + pass 'integration: preserved-table backup dump created' + else + fail 'integration: preserved-table backup dump' + return 1 + fi + + if (cd "$DATA_TOOL_DIR" && PG_BIN="$PG_BIN" PGDATABASE="$local_db" ./delta_restore_extract.sh \ + --dump "$dump" --mode preview --report-dir "$report_base" >/dev/null); then + preview_run="$(latest_run_dir "$report_base")" + pass 'integration: preview completed' + else + fail 'integration: preview completed' + return 1 + fi + + if [[ "$(path_mode "$preview_run")" = "700" ]] \ + && [[ "$(path_mode "$preview_run/preview.txt")" = "600" ]] \ + && [[ "$(path_mode "$preview_run/selection.conf")" = "600" ]] \ + && [[ "$(path_mode "$preview_run/selection_cookbook.txt")" = "600" ]] \ + && grep -q '^# OPERATOR COOKBOOK$' "$preview_run/selection_cookbook.txt" \ + && grep -q 'selection_cookbook.txt (this run dir)' "$preview_run/selection.conf"; then + pass 'integration: preview selection artifacts are private and cookbook is split from manifest' + else + fail 'integration: preview artifact permissions' + return 1 + fi + + local pattern missing=false + while read -r pattern; do + [[ -n "$pattern" ]] || continue + if ! grep -q -- "$pattern" "$preview_run/preview.txt"; then + printf >&2 'missing preview pattern: %s\n' "$pattern" + missing=true + fi + done < "$EXPECTED_DIR/t01_preview_patterns.txt" + if [[ "$missing" = "false" ]]; then + pass 'integration: preview report contains expected sections/classes' + else + fail 'integration: preview report expected patterns' + return 1 + fi + + if (cd "$DATA_TOOL_DIR" && PG_BIN="$PG_BIN" PGDATABASE="$local_db" ./delta_restore_extract.sh --dump "$dump" --mode apply --selection-file "$preview_run/selection.conf" --report-dir "$report_base" --yes --keep-artifacts >/dev/null); then + apply_run="$(latest_run_dir "$report_base")" + pass 'integration: apply completed for identical dump' + else + fail 'integration: apply completed for identical dump' + return 1 + fi + + if grep -q 'Apply summary' "$apply_run/apply_summary.txt" \ + && grep -q 'Affected counts' "$apply_run/apply_summary.txt" \ + && [[ -f "$apply_run/selection_cookbook.txt" ]] \ + && grep -q '^# OPERATOR COOKBOOK$' "$apply_run/selection_cookbook.txt"; then + pass 'integration: apply summary and selection cookbook contain expected sections' + else + fail 'integration: apply summary or selection cookbook expected sections' + return 1 + fi + + scope_dump="$dump_dir/scoped-selection.dump" + psql_db "$src" -c "UPDATE bad_emails SET notes = 'source-changed' WHERE id = 1; + INSERT INTO bad_emails(id, email, notes) VALUES (201, 'scope-in@example.test', 'inside'); + INSERT INTO excluded_emails(email, notes) VALUES ('scope-out@example.test', 'outside');" >/dev/null || { + fail 'integration: seed scoped selectable rows'; return 1; + } + if (cd "$DATA_TOOL_DIR" && PG_BIN="$PG_BIN" PGDATABASE="$src" BACKUP_DIR="$dump_dir" DUMP="$scope_dump" ./backup_extract_tables.sh >/dev/null); then + pass 'integration: scoped-selection dump created' + else + fail 'integration: scoped-selection dump creation' + return 1 + fi + if (cd "$hint_dir" && PG_BIN="$PG_BIN" PGDATABASE="$local_db" "$DATA_TOOL_DIR/delta_restore_extract.sh" \ + --dump "$scope_dump" --mode preview --tables bad_emails \ + --include-classes new,changed --exclude-classes changed \ + --only-corps "$hint_only_corps" --report-dir "$report_base" > "$preview_stdout" 2>&1); then + scope_preview="$(latest_run_dir "$report_base")" + pass 'integration: scoped preview completed with selectable rows inside and outside scope' + else + fail 'integration: scoped preview' + return 1 + fi + + cp "$scope_preview/selection.conf" "$hint_dir/my_selection.conf" + validate_command="$(sed -n 's/^ 3[.] To validate: //p' "$preview_stdout" | tail -n 1)" + (cd "$hint_dir" && PG_BIN="$PG_BIN" PGDATABASE="$local_db" bash -c "$validate_command" \ + > "$hint_validate_stdout" 2>&1) + hint_validate_status=$? + hint_validate_run="$(sed -n 's/^Artifacts: //p' "$hint_validate_stdout" | tail -n 1)" + if [[ -f "$hint_validate_run/selected_counts.tsv" ]]; then + cp "$hint_validate_run/selected_counts.tsv" "$hint_validate_counts" + fi + case "$hint_validate_run" in + "$DATA_TOOL_DIR/scripts/generated/delta_restore/"*) rm -rf -- "$hint_validate_run" ;; + *) ;; + esac + if [[ "$hint_validate_status" -eq 0 ]] \ + && grep -q '^ 3[.] To validate:' "$preview_stdout" \ + && grep -Fq -- '--tables bad_emails' <<<"$validate_command" \ + && grep -Fq -- '--include-classes new\,changed' <<<"$validate_command" \ + && grep -Fq -- '--exclude-classes changed' <<<"$validate_command" \ + && grep -Fq -- "--only-corps $hint_only_corps" <<<"$validate_command" \ + && grep -q 'Selection is valid' "$hint_validate_stdout" \ + && awk -F '\t' 'NR == 1 && $1 == "bad_emails" && $2 == "NEW" && $3 == "1" { ok=1 } + END { exit !(NR == 1 && ok) }' "$hint_validate_counts"; then + pass 'integration: printed validate command enforces table/class scope in actual selected counts' + else + fail "integration: printed scoped validate expected one bad_emails NEW row, got status $hint_validate_status" + return 1 + fi + + apply_command="$(sed -n 's/^To apply: //p' "$hint_validate_stdout" | tail -n 1)" + (cd "$hint_dir" && PG_BIN="$PG_BIN" PGDATABASE="$local_db" bash -c "$apply_command" \ + > "$hint_apply_stdout" 2>&1) + hint_apply_status=$? + hint_apply_run="$(sed -n 's/^Artifacts: //p' "$hint_apply_stdout" | tail -n 1)" + if [[ -f "$hint_apply_run/selected_counts.tsv" ]]; then + cp "$hint_apply_run/selected_counts.tsv" "$hint_apply_counts" + fi + if [[ -f "$hint_apply_run/apply_summary.txt" ]]; then + cp "$hint_apply_run/apply_summary.txt" "$hint_dir/scoped-apply-summary.txt" + fi + case "$hint_apply_run" in + "$DATA_TOOL_DIR/scripts/generated/delta_restore/"*) rm -rf -- "$hint_apply_run" ;; + *) ;; + esac + if [[ "$hint_apply_status" -eq 0 ]] \ + && grep -Fq -- '--tables bad_emails' <<<"$apply_command" \ + && grep -Fq -- '--include-classes new\,changed' <<<"$apply_command" \ + && grep -Fq -- '--exclude-classes changed' <<<"$apply_command" \ + && grep -Fq -- "--only-corps $hint_only_corps" <<<"$apply_command" \ + && awk -F '\t' 'NR == 1 && $1 == "bad_emails" && $2 == "NEW" && $3 == "1" { ok=1 } + END { exit !(NR == 1 && ok) }' "$hint_apply_counts" \ + && [[ "$(psql_db "$local_db" -qAt -c "SELECT count(*) FROM bad_emails WHERE id = 201")" = "1" ]] \ + && [[ "$(psql_db "$local_db" -qAt -c "SELECT notes FROM bad_emails WHERE id = 1")" = "seed" ]] \ + && [[ "$(psql_db "$local_db" -qAt -c "SELECT count(*) FROM excluded_emails WHERE email = 'scope-out@example.test'")" = "0" ]] \ + && grep -Eq 'bad_emails[[:space:]]+NEW[[:space:]]+INSERT[[:space:]]+1[[:space:]]+1' "$hint_dir/scoped-apply-summary.txt"; then + pass 'integration: printed apply command cannot select or modify rows outside explicit table/class scope' + else + fail "integration: printed scoped apply expected only one bad_emails NEW row, got status $hint_apply_status" + return 1 + fi + + psql_db "$local_db" -c "UPDATE bad_emails SET notes = 'source-changed' WHERE id = 1; + INSERT INTO excluded_emails(email, notes) VALUES ('scope-out@example.test', 'outside');" >/dev/null || { + fail 'integration: synchronize rows after scoped safety assertions'; return 1; + } + + local row_dump="$dump_dir/row-selection.dump" row_preview row_validate bad_validate_run row_apply + local row_selection bad_selector_selection bad_selection validate_status mismatch_status + psql_db "$src" -c "INSERT INTO bad_emails(id, email, notes) VALUES + (101, 'row101@example.test', 'one'), + (102, 'row102@example.test', 'two'), + (103, 'row103@example.test', 'three');" >/dev/null || { + fail 'integration: seed row-selection source rows'; return 1; + } + if (cd "$DATA_TOOL_DIR" && PG_BIN="$PG_BIN" PGDATABASE="$src" BACKUP_DIR="$dump_dir" DUMP="$row_dump" ./backup_extract_tables.sh >/dev/null); then + pass 'integration: row-selection dump created' + else + fail 'integration: row-selection dump creation' + return 1 + fi + if (cd "$DATA_TOOL_DIR" && PG_BIN="$PG_BIN" PGDATABASE="$local_db" ./delta_restore_extract.sh --dump "$row_dump" --mode preview --report-dir "$report_base" --details-limit 2 >/dev/null); then + row_preview="$(latest_run_dir "$report_base")" + pass 'integration: row-selection preview completed' + else + fail 'integration: row-selection preview' + return 1 + fi + if [[ -f "$row_preview/details/bad_emails.new.tsv" ]] && + grep -q 'row101@example.test' "$row_preview/details/bad_emails.new.tsv"; then + pass 'integration: NEW detail artifact contains staged row values' + else + fail 'integration: NEW detail artifact' + return 1 + fi + if [[ -f "$row_preview/details/bad_emails.new.txt" ]] \ + && [[ "$(path_mode "$row_preview/details/bad_emails.new.txt")" = "600" ]] \ + && grep -q '^# rendered 2 of 3 NEW rows — TRUNCATED$' "$row_preview/details/bad_emails.new.txt" \ + && grep -q '^# TRUNCATED at 2 rows$' "$row_preview/details/bad_emails.new.tsv" \ + && grep -Fq -- '- details/bad_emails.new.tsv — 2 of 3 NEW rows (TRUNCATED) · aligned: details/bad_emails.new.txt' "$row_preview/preview.txt"; then + pass 'integration: truncated aligned NEW detail has private mode, footer, and preview counts' + else + fail 'integration: truncated aligned NEW detail and preview count enrichment' + return 1 + fi + + row_selection="$TMP_BASE/row-selection.conf" + cp "$row_preview/selection.conf" "$row_selection" + printf '\n[bad_emails] new.rows include=id:101-102\n' >> "$row_selection" + + (cd "$DATA_TOOL_DIR" && PG_BIN="$PG_BIN" PGDATABASE="$local_db" ./delta_restore_extract.sh \ + --dump "$row_dump" --mode validate --selection-file "$row_selection" \ + --report-dir "$report_base" > "$TMP_BASE/validate.out" 2>&1) + validate_status=$? + row_validate="$(latest_run_dir "$report_base")" + if [[ "$validate_status" -eq 0 ]] \ + && grep -q 'Selection is valid' "$TMP_BASE/validate.out" \ + && grep -q $'bad_emails\tNEW\t2\t' "$row_validate/selected_counts.tsv"; then + pass 'integration: resident selection validate exits 0 with expected selected counts' + else + fail "integration: resident selection validate expected exit 0, got $validate_status" + return 1 + fi + if [[ -s "$row_validate/stage_counts.tsv" ]] \ + && [[ ! -e "$row_validate/toc.list" ]] \ + && [[ ! -e "$row_validate/stage_counts.sql" ]] \ + && [[ ! -e "$row_validate/create_stage_shells.sql" ]] \ + && [[ ! -e "$row_validate/preview_classification.sql" ]]; then + pass 'integration: validate regenerates staged counts without TOC read, staging, or classification' + else + fail 'integration: validate performed or failed to exclude restaging work' + return 1 + fi + + bad_selector_selection="$TMP_BASE/row-selection-invalid-selector.conf" + cp "$row_selection" "$bad_selector_selection" + printf '[bad_emails] new.rows include=id:999999\n' >> "$bad_selector_selection" + (cd "$DATA_TOOL_DIR" && PG_BIN="$PG_BIN" PGDATABASE="$local_db" ./delta_restore_extract.sh \ + --dump "$row_dump" --mode validate --selection-file "$bad_selector_selection" \ + --report-dir "$report_base" > "$TMP_BASE/validate-invalid.out" 2>&1) + validate_status=$? + bad_validate_run="$(latest_run_dir "$report_base")" + if [[ "$validate_status" -eq 4 ]] \ + && grep -Fq $'bad_emails\tNEW\tinclude\tid\t999999\t' "$bad_validate_run/selection_diagnostics.tsv" \ + && grep -Fq $'\tNO_MATCH' "$bad_validate_run/selection_diagnostics.tsv" \ + && [[ ! -e "$bad_validate_run/toc.list" ]] \ + && [[ ! -e "$bad_validate_run/preview_classification.sql" ]]; then + pass 'integration: invalid resident selector exits 4 with diagnostics and no restaging' + else + fail "integration: invalid resident selector expected exit 4, got $validate_status" + return 1 + fi + + if (cd "$DATA_TOOL_DIR" && PG_BIN="$PG_BIN" PGDATABASE="$local_db" ./delta_restore_extract.sh --dump "$row_dump" --mode apply --selection-file "$row_selection" --report-dir "$report_base" --yes --keep-artifacts >/dev/null); then + row_apply="$(latest_run_dir "$report_base")" + pass 'integration: row-selector apply completed' + else + fail 'integration: row-selector apply' + return 1 + fi + if [[ "$(psql_db "$local_db" -qAt -c "SELECT count(*) FROM bad_emails WHERE id IN (101,102)")" = "2" ]] && + [[ "$(psql_db "$local_db" -qAt -c "SELECT count(*) FROM bad_emails WHERE id = 103")" = "0" ]] && + grep -Eq 'bad_emails[[:space:]]+NEW[[:space:]]+INSERT[[:space:]]+2[[:space:]]+2' "$row_apply/apply_summary.txt"; then + pass 'integration: row selectors applied 2 rows and skipped 1 with expected=affected' + else + fail 'integration: row-selector applied/skipped row assertions' + return 1 + fi + + bad_selection="$TMP_BASE/row-selection-bad-sha.conf" + sed 's/^# dump_sha256=.*/# dump_sha256=0000/' "$row_selection" > "$bad_selection" + (cd "$DATA_TOOL_DIR" && PG_BIN="$PG_BIN" PGDATABASE="$local_db" ./delta_restore_extract.sh --dump "$row_dump" --mode apply --selection-file "$bad_selection" --report-dir "$report_base" --yes --keep-artifacts >/dev/null 2>&1) + mismatch_status=$? + if [[ "$mismatch_status" -eq 4 ]]; then + pass 'integration: row-selector dump hash mismatch exits 4' + else + fail "integration: row-selector hash mismatch expected exit 4, got $mismatch_status" + return 1 + fi +} + +main() { + require_file "$DATA_TOOL_DIR/delta_restore_extract.sh" || return 1 + require_file "$DATA_TOOL_DIR/scripts/restore/delta/10_functions.sql" || return 1 + require_file "$DATA_TOOL_DIR/scripts/restore/delta/rewrite_copy_targets.awk" || return 1 + require_file "$DATA_TOOL_DIR/scripts/restore/delta/align_details.awk" || return 1 + + run_static_checks || true + run_sql_smoke || true + run_optional_integration || true + + printf '\nSummary: %s passed, %s failed, %s skipped\n' "$pass_count" "$fail_count" "$skip_count" + [[ "$fail_count" -eq 0 ]] +} + +main "$@" diff --git a/data-tool/tests/flows/test_auth_invite_diagnostics.py b/data-tool/tests/flows/test_auth_invite_diagnostics.py new file mode 100644 index 0000000000..9cff48094d --- /dev/null +++ b/data-tool/tests/flows/test_auth_invite_diagnostics.py @@ -0,0 +1,950 @@ +import csv +from datetime import datetime, timedelta, timezone +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +FLOWS_PATH = Path(__file__).resolve().parents[2] / "flows" +sys.path.insert(0, str(FLOWS_PATH)) + +from auth import auth_report_helpers # noqa: E402 +from auth.verify_auth_flow import ( # noqa: E402 + CHECK_INVITE, + INSPECTION_FIELDNAMES, + INSPECT_FILTER_ALL, + INSPECTION_LEGEND_LINES, + AuthBusinessState, + AuthInviteDiagnostics, + ConsoleLimit, + _INVITE_READ_SQL, + _AuthInviteAccumulator, + _all_inspection_summary_metric_rows, + _dated_invite_sort_key, + _format_invite_dates, + _format_invite_mix, + _format_invite_summary, + build_inspection_rows, + build_inspection_summary, + build_verification_results, + print_inspection_rows, + parse_invite_criteria, + read_auth_states_for_candidates, + run_auth_batch, + write_inspection_report, + write_inspection_summary_txt, +) + + +class _FakeResult: + def __init__(self, rows, *, allow_fetchall=True): + self._rows = rows + self._allow_fetchall = allow_fetchall + + def __iter__(self): + return iter(self._rows) + + def fetchall(self): + assert self._allow_fetchall, "invitation reads must stream rather than materialize all rows" + return self._rows + + +class _FakeConnection: + def __init__(self, entity_rows, invite_rows): + self.entity_rows = entity_rows + self.invite_rows = invite_rows + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def execute(self, statement, _params): + sql = str(statement) + if "FROM entities" in sql: + return _FakeResult(self.entity_rows) + if "FROM affiliation_invitations" in sql: + return _FakeResult(self.invite_rows, allow_fetchall=False) + return _FakeResult([]) + + +class _FakeEngine: + def __init__(self, entity_rows, invite_rows): + self.connection = _FakeConnection(entity_rows, invite_rows) + + def connect(self): + return self.connection + + +def _diagnostic_engine(cutoff, *, positional_invite_rows=False): + entity_rows = [ + {"id": 1, "business_identifier": "BC123", "business_name": "Alpha"}, + {"id": 2, "business_identifier": "BC123", "business_name": "Alpha Duplicate"}, + ] + invite_rows = [ + { + "id": 101, + "entity_id": 1, + "type": "UNAFFILIATED_EMAIL", + "invitation_status_code": "PENDING", + "sent_date": cutoff - timedelta(days=2), + "accepted_date": None, + "is_deleted": False, + "affiliation_id": None, + }, + { + "id": 102, + "entity_id": 1, + "type": "UNAFFILIATED_EMAIL", + "invitation_status_code": "ACCEPTED", + "sent_date": cutoff - timedelta(days=10), + "accepted_date": cutoff + timedelta(days=1), + "is_deleted": True, + "affiliation_id": 99, + }, + { + "id": 103, + "entity_id": 1, + "type": "UNAFFILIATED_EMAIL", + "invitation_status_code": "EXPIRED", + "sent_date": cutoff - timedelta(days=20), + "accepted_date": None, + "is_deleted": True, + "affiliation_id": None, + }, + { + "id": 201, + "entity_id": 2, + "type": "OTHER", + "invitation_status_code": "PENDING", + "sent_date": (cutoff + timedelta(days=2)).replace(tzinfo=None), + "accepted_date": None, + "is_deleted": False, + "affiliation_id": None, + }, + { + "id": 202, + "entity_id": 2, + "type": None, + "invitation_status_code": None, + "sent_date": None, + "accepted_date": None, + "is_deleted": False, + "affiliation_id": None, + }, + { + "id": 203, + "entity_id": 2, + "type": " UNAFFILIATED_EMAIL ", + "invitation_status_code": " PENDING ", + "sent_date": cutoff.replace(tzinfo=None), + "accepted_date": None, + "is_deleted": False, + "affiliation_id": None, + }, + { + "id": 204, + "entity_id": 2, + "type": "UNAFFILIATED_EMAIL", + "invitation_status_code": "PENDING", + "sent_date": None, + "accepted_date": None, + "is_deleted": False, + "affiliation_id": None, + }, + { + "id": 205, + "entity_id": 2, + "type": "OTHER", + "invitation_status_code": "ACCEPTED", + "sent_date": cutoff - timedelta(days=5), + "accepted_date": cutoff + timedelta(days=2), + "is_deleted": False, + "affiliation_id": 100, + }, + ] + if positional_invite_rows: + invite_rows = [ + ( + row["id"], + row["entity_id"], + row["type"], + row["invitation_status_code"], + row["sent_date"], + row["accepted_date"], + row["is_deleted"], + row["affiliation_id"], + ) + for row in invite_rows + ] + return _FakeEngine(entity_rows, invite_rows) + + +def test_invitation_read_is_row_level_and_excludes_sensitive_columns(): + assert "COUNT(" not in _INVITE_READ_SQL + assert "GROUP BY" not in _INVITE_READ_SQL + assert "ORDER BY entity_id, id" in _INVITE_READ_SQL + assert "recipient_email" not in _INVITE_READ_SQL + assert "token" not in _INVITE_READ_SQL + assert "additional_message" not in _INVITE_READ_SQL + + +def test_dated_invite_sort_key_uses_date_numeric_entity_and_invitation_id_tiebreaks(): + earlier = datetime(2026, 5, 24, 12) + tied = datetime(2026, 5, 25, 12) + dated_invites = [ + (tied, "10", 1), + (tied, "2", 10), + (earlier, "99", 99), + (tied, "2", 2), + ] + + assert sorted(dated_invites, key=_dated_invite_sort_key) == [ + (earlier, "99", 99), + (tied, "2", 2), + (tied, "2", 10), + (tied, "10", 1), + ] + + +@pytest.mark.parametrize("positional_invite_rows", [False, True]) +def test_row_aggregation_merges_entities_and_preserves_invite_count(positional_invite_rows): + cutoff = datetime(2026, 6, 23, 12, tzinfo=timezone.utc) + + state = read_auth_states_for_candidates( + _diagnostic_engine(cutoff, positional_invite_rows=positional_invite_rows), + ["BC123"], + selected_checks=(CHECK_INVITE,), + invite_cutoff=cutoff, + )[0] + + diagnostics = state.invite_diagnostics + assert state.entity_ids == ("1", "2") + assert state.invite_count == diagnostics.total_count == 8 + assert diagnostics.deleted_count == 1 + assert diagnostics.claimed_count == 2 + assert diagnostics.unclaimed_count == 5 + assert diagnostics.type_counts == (("(blank)", 1), ("OTHER", 2), ("UNAFFILIATED_EMAIL", 4)) + assert diagnostics.status_counts == (("(blank)", 1), ("ACCEPTED", 2), ("PENDING", 4)) + assert diagnostics.unaffiliated_unclaimed_count == 3 + assert diagnostics.expired_unclaimed_unaffiliated_count == 1 + assert diagnostics.current_unclaimed_unaffiliated_count == 1 + assert diagnostics.unknown_age_unclaimed_unaffiliated_count == 1 + assert diagnostics.unaffiliated_unclaimed_sent_dates == ( + datetime(2026, 6, 21, 12), + datetime(2026, 6, 23, 12), + ) + assert diagnostics.unaffiliated_unclaimed_count == ( + len(diagnostics.unaffiliated_unclaimed_sent_dates) + + diagnostics.unknown_age_unclaimed_unaffiliated_count + ) + assert diagnostics.oldest_unclaimed_sent_date == datetime(2026, 6, 21, 12) + assert diagnostics.newest_unclaimed_sent_date == datetime(2026, 6, 25, 12) + assert diagnostics.latest_accepted_date == datetime(2026, 6, 25, 12) + assert diagnostics.total_count == ( + diagnostics.claimed_count + diagnostics.deleted_count + diagnostics.unclaimed_count + ) + + +def test_accepted_deleted_invite_renders_as_claimed_with_acceptance_date(): + cutoff = datetime(2026, 6, 23, 12) + accumulator = _AuthInviteAccumulator() + accumulator.add_row( + invite_type="UNAFFILIATED_EMAIL", + invitation_status_code="ACCEPTED", + sent_date=cutoff - timedelta(days=10), + accepted_date=cutoff + timedelta(days=1), + is_deleted=True, + invite_cutoff=cutoff, + ) + diagnostics = accumulator.freeze(cutoff_configured=True) + row = build_inspection_rows( + [ + AuthBusinessState( + business_identifier="BC123", + entity_ids=("1",), + invite_count=1, + invite_diagnostics=diagnostics, + ) + ] + )[0] + + assert diagnostics.unaffiliated_unclaimed_count == 0 + assert _format_invite_summary(row) == "uncl=0 q=0(exp=0,cur=0) clm=1 del=0" + assert _format_invite_dates(row) == "a:06-24" + + +def test_ordered_qualifying_dates_are_business_wide_deterministic_and_exclude_non_qualifying_rows(): + reference = datetime(2026, 7, 24, 12, tzinfo=timezone.utc) + entity_rows = [ + {"id": 10, "business_identifier": "BC123", "business_name": "Alpha"}, + {"id": 2, "business_identifier": "BC123", "business_name": "Alpha"}, + ] + tied = reference - timedelta(days=60) + invite_rows = [ + { + "id": 99, + "entity_id": 10, + "type": "UNAFFILIATED_EMAIL", + "invitation_status_code": "PENDING", + "sent_date": reference + timedelta(days=1), + "accepted_date": None, + "is_deleted": False, + "affiliation_id": None, + }, + { + "id": 20, + "entity_id": 10, + "type": "UNAFFILIATED_EMAIL", + "invitation_status_code": "PENDING", + "sent_date": tied, + "accepted_date": None, + "is_deleted": False, + "affiliation_id": None, + }, + { + "id": 30, + "entity_id": 2, + "type": "UNAFFILIATED_EMAIL", + "invitation_status_code": "PENDING", + "sent_date": tied, + "accepted_date": None, + "is_deleted": False, + "affiliation_id": None, + }, + { + "id": 10, + "entity_id": 10, + "type": "UNAFFILIATED_EMAIL", + "invitation_status_code": "PENDING", + "sent_date": reference - timedelta(days=90), + "accepted_date": None, + "is_deleted": False, + "affiliation_id": None, + }, + { + "id": 40, + "entity_id": 2, + "type": "UNAFFILIATED_EMAIL", + "invitation_status_code": "PENDING", + "sent_date": None, + "accepted_date": None, + "is_deleted": False, + "affiliation_id": None, + }, + { + "id": 50, + "entity_id": 2, + "type": "UNAFFILIATED_EMAIL", + "invitation_status_code": "PENDING", + "sent_date": reference - timedelta(days=120), + "accepted_date": None, + "is_deleted": True, + "affiliation_id": None, + }, + { + "id": 60, + "entity_id": 2, + "type": "UNAFFILIATED_EMAIL", + "invitation_status_code": "ACCEPTED", + "sent_date": reference - timedelta(days=110), + "accepted_date": reference - timedelta(days=100), + "is_deleted": False, + "affiliation_id": 1, + }, + { + "id": 70, + "entity_id": 2, + "type": "OTHER", + "invitation_status_code": "PENDING", + "sent_date": reference - timedelta(days=100), + "accepted_date": None, + "is_deleted": False, + "affiliation_id": None, + }, + ] + + def read(rows): + return read_auth_states_for_candidates( + _FakeEngine(entity_rows, rows), + ["BC123"], + selected_checks=(CHECK_INVITE,), + invite_cutoff=reference - timedelta(days=30), + )[0].invite_diagnostics + + expected_dates = ( + datetime(2026, 4, 25, 12), + datetime(2026, 5, 25, 12), + datetime(2026, 5, 25, 12), + datetime(2026, 7, 25, 12), + ) + diagnostics = read(invite_rows) + reverse_diagnostics = read(list(reversed(invite_rows))) + + assert diagnostics.unaffiliated_unclaimed_sent_dates == expected_dates + assert reverse_diagnostics.unaffiliated_unclaimed_sent_dates == expected_dates + assert diagnostics.unaffiliated_unclaimed_count == 5 + assert diagnostics.unknown_age_unclaimed_unaffiliated_count == 1 + assert diagnostics.unaffiliated_unclaimed_count == ( + len(diagnostics.unaffiliated_unclaimed_sent_dates) + + diagnostics.unknown_age_unclaimed_unaffiliated_count + ) + + +def test_run_auth_batch_threads_optional_cutoff_and_legacy_settings_remain_compatible(): + cutoff = datetime(2026, 6, 23, 12, tzinfo=timezone.utc) + config = SimpleNamespace() + settings_with_cutoff = SimpleNamespace( + batch_size=10, + auth_read_checks=(CHECK_INVITE,), + selected_checks=(CHECK_INVITE,), + run_verify=False, + invite_cutoff=cutoff, + ) + + state = run_auth_batch( + config, + _diagnostic_engine(cutoff), + ["BC123"], + settings=settings_with_cutoff, + ).states[0] + assert state.invite_diagnostics.expired_unclaimed_unaffiliated_count == 1 + + legacy_settings = SimpleNamespace( + batch_size=10, + auth_read_checks=(CHECK_INVITE,), + selected_checks=(CHECK_INVITE,), + run_verify=False, + ) + state_without_cutoff = run_auth_batch( + config, + _diagnostic_engine(cutoff), + ["BC123"], + settings=legacy_settings, + ).states[0] + assert state_without_cutoff.invite_count == 8 + assert state_without_cutoff.invite_diagnostics.expired_unclaimed_unaffiliated_count is None + assert state_without_cutoff.invite_diagnostics.current_unclaimed_unaffiliated_count is None + + +REPORT_NOW = datetime(2026, 7, 24, 12, tzinfo=timezone.utc) +REPORT_CRITERIA = parse_invite_criteria( + "count==2, age[1]>=50", + now=REPORT_NOW, + expiry_days=None, +) +assert REPORT_CRITERIA is not None + + +def _output_states(): + with_cutoff = AuthBusinessState( + business_identifier="BC123", + entity_ids=("1",), + invite_count=3, + invite_diagnostics=AuthInviteDiagnostics( + total_count=3, + claimed_count=1, + unclaimed_count=2, + type_counts=(("OTHER", 1), ("UNAFFILIATED_EMAIL", 2)), + status_counts=(("ACCEPTED", 1), ("PENDING", 2)), + unaffiliated_unclaimed_count=2, + expired_unclaimed_unaffiliated_count=2, + current_unclaimed_unaffiliated_count=0, + unaffiliated_unclaimed_sent_dates=( + datetime(2026, 6, 1, 1, 2, 3, tzinfo=timezone.utc), + datetime(2026, 6, 2, 2, 3, 4), + ), + oldest_unclaimed_sent_date=datetime(2026, 6, 1, 1, 2, 3, tzinfo=timezone.utc), + newest_unclaimed_sent_date=datetime(2026, 6, 2, 2, 3, 4), + latest_accepted_date=datetime(2026, 6, 3, 3, 4, 5), + ), + ) + without_cutoff = AuthBusinessState( + business_identifier="BC456", + entity_ids=("2",), + invite_count=1, + invite_diagnostics=AuthInviteDiagnostics( + total_count=1, + unclaimed_count=1, + type_counts=(("UNAFFILIATED_EMAIL", 1),), + status_counts=(("PENDING", 1),), + unaffiliated_unclaimed_count=1, + unaffiliated_unclaimed_sent_dates=(datetime(2026, 7, 20, 12),), + ), + ) + return [with_cutoff, without_cutoff] + + +def test_csv_rows_append_diagnostics_and_distinguish_blank_from_zero(): + expected_appended_fields = [ + "invite_claimed_count", + "invite_unclaimed_count", + "invite_deleted_count", + "invite_type_counts", + "invite_status_counts", + "unaffiliated_unclaimed_invite_count", + "unaffiliated_unclaimed_expired_count", + "unaffiliated_unclaimed_current_count", + "unaffiliated_unclaimed_unknown_age_count", + "oldest_unclaimed_sent_date", + "newest_unclaimed_sent_date", + "latest_accepted_date", + "unaffiliated_unclaimed_sent_dates", + "unaffiliated_unclaimed_ages_days", + "invite_criteria_pass", + "invite_criteria_failed_clauses", + ] + assert INSPECTION_FIELDNAMES[-16:] == expected_appended_fields + + rows_without_criteria = build_inspection_rows(_output_states()) + assert rows_without_criteria[0]["unaffiliated_unclaimed_sent_dates"] == ( + "2026-06-01T01:02:03;2026-06-02T02:03:04" + ) + assert rows_without_criteria[0]["unaffiliated_unclaimed_ages_days"] == "" + assert rows_without_criteria[0]["invite_criteria_pass"] == "" + assert rows_without_criteria[0]["invite_criteria_failed_clauses"] == "" + + with_cutoff, without_cutoff = build_inspection_rows(_output_states(), REPORT_CRITERIA) + assert with_cutoff["invite_type_counts"] == "OTHER=1;UNAFFILIATED_EMAIL=2" + assert with_cutoff["invite_status_counts"] == "ACCEPTED=1;PENDING=2" + assert with_cutoff["unaffiliated_unclaimed_expired_count"] == 2 + assert with_cutoff["unaffiliated_unclaimed_current_count"] == 0 + assert with_cutoff["oldest_unclaimed_sent_date"] == "2026-06-01T01:02:03" + assert with_cutoff["unaffiliated_unclaimed_sent_dates"] == ( + "2026-06-01T01:02:03;2026-06-02T02:03:04" + ) + assert with_cutoff["unaffiliated_unclaimed_ages_days"] == "53;52" + assert with_cutoff["invite_criteria_pass"] == "true" + assert with_cutoff["invite_criteria_failed_clauses"] == "" + assert without_cutoff["invite_criteria_pass"] == "false" + assert without_cutoff["invite_criteria_failed_clauses"] == "count==2;age[1]>=50" + assert without_cutoff["unaffiliated_unclaimed_expired_count"] == "" + assert without_cutoff["unaffiliated_unclaimed_current_count"] == "" + assert "recipient_email" not in with_cutoff + assert "token" not in with_cutoff + + +def test_reference_now_populates_diagnostic_ages_and_shared_report_writer(tmp_path): + future_state = AuthBusinessState( + business_identifier="BCFUTURE", + entity_ids=("1",), + invite_count=1, + invite_diagnostics=AuthInviteDiagnostics( + total_count=1, + unclaimed_count=1, + unaffiliated_unclaimed_count=1, + unaffiliated_unclaimed_sent_dates=(REPORT_NOW + timedelta(hours=1),), + ), + ) + null_only_state = AuthBusinessState( + business_identifier="BCNULL", + entity_ids=("2",), + invite_count=1, + invite_diagnostics=AuthInviteDiagnostics( + total_count=1, + unclaimed_count=1, + unaffiliated_unclaimed_count=1, + unknown_age_unclaimed_unaffiliated_count=1, + ), + ) + + rows = build_inspection_rows( + [future_state, null_only_state], + reference_now=REPORT_NOW, + ) + criteria_rows = build_inspection_rows( + [future_state, null_only_state], + parse_invite_criteria("count>=1", now=REPORT_NOW, expiry_days=None), + reference_now=REPORT_NOW + timedelta(days=10), + ) + + assert [row["unaffiliated_unclaimed_ages_days"] for row in rows] == ["-1", ""] + assert [row["unaffiliated_unclaimed_ages_days"] for row in rows] == [ + row["unaffiliated_unclaimed_ages_days"] for row in criteria_rows + ] + assert build_inspection_rows([future_state])[0]["unaffiliated_unclaimed_ages_days"] == "" + + report_path = tmp_path / "inspect.csv" + write_inspection_report( + str(report_path), + [future_state], + reference_now=REPORT_NOW, + ) + with report_path.open(newline="", encoding="utf-8") as report_file: + persisted_row = next(csv.DictReader(report_file)) + assert persisted_row["unaffiliated_unclaimed_ages_days"] == "-1" + + +def test_invite_summary_distinguishes_all_unclaimed_from_qualifying_subset(): + assert _format_invite_summary( + { + "invite_unclaimed_count": 5, + "unaffiliated_unclaimed_invite_count": 3, + "unaffiliated_unclaimed_expired_count": 1, + "unaffiliated_unclaimed_current_count": 1, + "unaffiliated_unclaimed_unknown_age_count": 1, + "invite_claimed_count": 1, + "invite_deleted_count": 0, + } + ) == "uncl=5 q=3(exp=1,cur=1,unk=1) clm=1 del=0" + + +@pytest.mark.parametrize( + ("row", "expected"), + [ + ( + { + "oldest_unclaimed_sent_date": "2026-06-01T01:02:03", + "newest_unclaimed_sent_date": "2026-06-02T02:03:04", + "latest_accepted_date": "2026-06-03T03:04:05", + }, + "s:06-01→06-02 a:06-03", + ), + ( + { + "oldest_unclaimed_sent_date": "2026-06-01T01:02:03", + "newest_unclaimed_sent_date": "2026-06-01T01:02:03", + }, + "s:06-01", + ), + ( + { + "oldest_unclaimed_sent_date": "2025-12-31T01:02:03", + "newest_unclaimed_sent_date": "2026-01-01T02:03:04", + "latest_accepted_date": "2026-01-02T03:04:05", + }, + "s:2025-12-31→2026-01-01 a:2026-01-02", + ), + ({"latest_accepted_date": "2026-06-03T03:04:05"}, "a:06-03"), + ({}, ""), + ], +) +def test_format_invite_dates(row, expected): + assert _format_invite_dates(row) == expected + + +@pytest.mark.parametrize( + ("row", "expected"), + [ + ( + { + "invite_type_counts": "(blank)=1;EMAIL=2;UNAFFILIATED_EMAIL=4", + "invite_status_counts": "(blank)=1;ACCEPTED=2;PENDING=4", + }, + "T:(b)=1,EM=2,UE=4 S:(b)=1,ACC=2,PEND=4", + ), + ({}, ""), + ({"invite_type_counts": "Something_Long=3"}, "T:SOMET=3"), + ({"invite_status_counts": "EXPIRED=2"}, "S:EXPD=2"), + ], +) +def test_format_invite_mix_abbreviates_known_buckets_and_uses_stable_fallback(row, expected): + assert _format_invite_mix(row) == expected + + +def test_age_alias_failed_clause_rows_use_canonical_endpoints_without_schema_changes(): + fieldnames_before = tuple(INSPECTION_FIELDNAMES) + criteria = parse_invite_criteria( + "all_ages>=30, any_age>=90", + now=REPORT_NOW, + expiry_days=None, + ) + assert criteria is not None + + _with_cutoff, failing_state = _output_states() + row = build_inspection_rows([failing_state], criteria)[0] + + assert row["invite_criteria_pass"] == "false" + assert row["invite_criteria_failed_clauses"] == "newest_age>=30;age[1]>=90" + assert "all_ages" not in str(row) + assert "any_age" not in str(row) + assert tuple(INSPECTION_FIELDNAMES) == fieldnames_before + + +def test_console_shortens_missing_entity_states_without_changing_report_rows(capsys): + state = AuthBusinessState(business_identifier="BC000", entity_ids=()) + rows = build_inspection_rows([state]) + + assert rows[0]["contact_state"] == "not_applicable_entity_missing" + assert rows[0]["affiliation_state"] == "not_applicable_entity_missing" + assert rows[0]["invite_state"] == "not_applicable_entity_missing" + + print_inspection_rows(rows, ConsoleLimit(max_rows=None)) + console_output = capsys.readouterr().out + assert console_output.count("na_entity_missing") == 3 + assert "not_applicable_entity_missing" not in console_output + + +def test_console_row_column_is_leftmost_and_numbers_business_rows(capsys): + rows = build_inspection_rows(_output_states()) + + print_inspection_rows(rows, ConsoleLimit(max_rows=None)) + + output_lines = capsys.readouterr().out.splitlines() + header_line = next(line for line in output_lines if "business_names" in line) + header_cells = [cell.strip() for cell in header_line.split(" | ")] + assert header_cells == [ + "row", + "identifier", + "business_names", + "entity", + "contact", + "affiliation", + "invite", + "invite_mix", + "invite_summary", + "invite_dates", + "found_account_ids", + "found_account_names", + ] + assert "business_identifier" not in header_line + assert { + "entity_state", + "contact_state", + "affiliation_state", + "invite_state", + }.isdisjoint(header_cells) + business_lines = [line for line in output_lines if "BC123" in line or "BC456" in line] + assert [line.split(" | ", maxsplit=1)[0].strip() for line in business_lines] == ["1", "2"] + + +def test_console_truncation_marker_has_blank_row_cell_and_keeps_separators(capsys): + rows = build_inspection_rows(_output_states()) + + print_inspection_rows(rows, ConsoleLimit(max_rows=1)) + + output_lines = capsys.readouterr().out.splitlines() + header_line = next(line for line in output_lines if "business_names" in line) + header_cells = [cell.strip() for cell in header_line.split(" | ")] + marker_index = next( + index for index, line in enumerate(output_lines) + if "⚠️ 1 MORE ROWS NOT SHOWN" in line + ) + assert "-+-" in output_lines[marker_index - 1] + assert output_lines[marker_index - 1] == output_lines[marker_index + 1] + marker_cells = output_lines[marker_index].split(" | ") + assert marker_cells[0].strip() == "" + assert marker_cells[header_cells.index("invite_mix")].strip() == "" + + +def test_console_row_column_does_not_change_schema_or_inspection_row_dicts(capsys): + fieldnames_before = tuple(INSPECTION_FIELDNAMES) + rows = build_inspection_rows(_output_states()) + row_keys = [tuple(row.keys()) for row in rows] + + print_inspection_rows(rows, ConsoleLimit(max_rows=None)) + capsys.readouterr() + + assert tuple(INSPECTION_FIELDNAMES) == fieldnames_before + assert INSPECTION_FIELDNAMES[0] == "business_identifier" + assert "row" not in INSPECTION_FIELDNAMES + assert "identifier" not in INSPECTION_FIELDNAMES + assert [row["business_identifier"] for row in rows] == ["BC123", "BC456"] + state_keys = { + "entity_state", + "contact_state", + "affiliation_state", + "invite_state", + } + assert state_keys.issubset(INSPECTION_FIELDNAMES) + assert all(state_keys.issubset(row) for row in rows) + assert all("row" not in row for row in rows) + assert [tuple(row.keys()) for row in rows] == row_keys + + +def test_console_and_summary_outputs_include_compact_diagnostics(tmp_path, capsys): + states = _output_states() + rows = build_inspection_rows(states, REPORT_CRITERIA) + + print_inspection_rows(rows, ConsoleLimit(max_rows=1)) + console_output = capsys.readouterr().out + assert "invite_mix" in console_output + assert "invite_summary" in console_output + assert "invite_dates" in console_output + assert "T:OTHER=1,UE=2 S:ACC=1,PEND=2" in console_output + assert "uncl=2 q=2(exp=2,cur=0) clm=1 del=0 ages=53,52d" in console_output + assert "s:06-01→06-02 a:06-03" in console_output + assert "showing 1 of 2 matched" in console_output + assert all(console_output.count(line) == 1 for line in INSPECTION_LEGEND_LINES) + assert "1 MORE ROWS NOT SHOWN" in console_output + + print_inspection_rows(rows, ConsoleLimit(max_rows=None)) + all_console_output = capsys.readouterr().out + assert "showing 2 of 2 matched" in all_console_output + assert all(all_console_output.count(line) == 1 for line in INSPECTION_LEGEND_LINES) + + summary = build_inspection_summary(states, states, INSPECT_FILTER_ALL) + assert summary["has_unclaimed_invite_count"] == 2 + assert summary["has_claimed_invite_count"] == 1 + assert summary["has_unaffiliated_unclaimed_invite_count"] == 2 + assert summary["unclaimed_invite_rows_total"] == 3 + labels = [label for label, _value in _all_inspection_summary_metric_rows(summary)] + assert "HasUnclaimedInvite" in labels + assert "HasClaimedInvite" in labels + assert "HasUnaffiliatedUnclaimedInvite" in labels + assert "UnclaimedInviteRows(invites)" in labels + + summary_path = tmp_path / "summary.txt" + write_inspection_summary_txt(str(summary_path), [], summary) + summary_text = summary_path.read_text(encoding="utf-8") + assert "HasUnclaimedInvite" in summary_text + assert "UnclaimedInviteRows(invites)" in summary_text + + +def test_console_legend_lines_are_self_identifying_and_printed_before_table(monkeypatch): + printed_lines = [] + monkeypatch.setattr( + "builtins.print", + lambda *values, **_kwargs: printed_lines.append(" ".join(str(value) for value in values)), + ) + + rows = build_inspection_rows(_output_states()) + print_inspection_rows(rows, ConsoleLimit(max_rows=None)) + + assert all(line.startswith("Legend invite_") for line in INSPECTION_LEGEND_LINES) + assert all(len(line) <= 120 for line in INSPECTION_LEGEND_LINES) + assert all("\n" not in line for line in INSPECTION_LEGEND_LINES) + assert all("\n" not in line for line in printed_lines) + assert all(printed_lines.count(line) == 1 for line in INSPECTION_LEGEND_LINES) + + title_index = next(index for index, line in enumerate(printed_lines) if line.startswith("🔎 Inspect Auth state")) + legend_indices = [printed_lines.index(line) for line in INSPECTION_LEGEND_LINES] + first_border_index = next(index for index, line in enumerate(printed_lines) if "─┼─" in line) + assert title_index < legend_indices[0] + assert legend_indices == list(range(legend_indices[0], legend_indices[0] + len(INSPECTION_LEGEND_LINES))) + assert legend_indices[-1] < first_border_index + + legend_text = " ".join(INSPECTION_LEGEND_LINES) + assert "q=qualifying UNAFFILIATED_EMAIL unclaimed" in legend_text + assert "exp=sent=Cutoff" in legend_text + assert "unk=NULL sent_date" in legend_text + assert "clm=accepted (Auth soft-deletes)" in legend_text + assert "del=without acceptance" in legend_text + assert "ages=days since sent, qualifying only" in legend_text + assert "s:=unclaimed sent range (all types)" in legend_text + assert "a:=latest accepted date" in legend_text + assert "CSV: inspect-auth-inspection.csv" in legend_text + assert "T:=types" in legend_text + assert "EM=EMAIL UE=UNAFFILIATED_EMAIL" in legend_text + assert "S:=raw Auth status(es)" in legend_text + assert "deleted-w/o-acceptance excluded→del=" in legend_text + assert "all non-deleted invite types" not in legend_text + + +def test_console_legend_is_absent_when_no_rows_or_console_preview_is_disabled(capsys): + print_inspection_rows([], ConsoleLimit(max_rows=None)) + no_rows_output = capsys.readouterr().out + assert all(line not in no_rows_output for line in INSPECTION_LEGEND_LINES) + + rows = build_inspection_rows(_output_states()) + print_inspection_rows(rows, ConsoleLimit(disabled=True, max_rows=0)) + disabled_output = capsys.readouterr().out + assert disabled_output == "" + assert all(line not in disabled_output for line in INSPECTION_LEGEND_LINES) + + +def test_console_business_names_default_uncapped_and_configured_cap_is_console_only( + tmp_path, + capsys, +): + business_name = "Long Business Name " * 4 + account_name = "Long Account Name " * 3 + states = [ + AuthBusinessState( + business_identifier="BC1234567", + entity_ids=("1",), + business_names=(business_name,), + found_account_ids=("123",), + found_account_names=(account_name,), + ), + ] + rows = build_inspection_rows(states) + + assert rows[0]["business_names"] == business_name + assert rows[0]["found_account_names"] == account_name + + print_inspection_rows(rows, ConsoleLimit(max_rows=None)) + uncapped_console = capsys.readouterr().out + assert business_name in uncapped_console + assert f"{account_name[:27]}…" in uncapped_console + assert account_name not in uncapped_console + + print_inspection_rows( + rows, + ConsoleLimit(max_rows=None), + business_names_max_length=18, + ) + capped_console = capsys.readouterr().out + assert f"{business_name[:17]}…" in capped_console + assert business_name not in capped_console + assert f"{account_name[:27]}…" in capped_console + assert account_name not in capped_console + + assert rows[0]["business_names"] == business_name + assert rows[0]["found_account_names"] == account_name + + report_path = tmp_path / "inspect.csv" + write_inspection_report(str(report_path), states) + with report_path.open(newline="", encoding="utf-8") as report_file: + persisted_row = next(csv.DictReader(report_file)) + assert persisted_row["business_names"] == business_name + assert persisted_row["found_account_names"] == account_name + + +def test_console_renders_full_invite_mix_without_changing_report_cells(capsys): + row = build_inspection_rows(_output_states())[0] + type_counts = row["invite_type_counts"] + status_counts = row["invite_status_counts"] + full_mix = _format_invite_mix(row) + assert len(full_mix) > 28 + + print_inspection_rows([row], ConsoleLimit(max_rows=None)) + + console = capsys.readouterr().out + assert full_mix in console + assert f"{full_mix[:27]}…" not in console + assert row["invite_type_counts"] == type_counts + assert row["invite_status_counts"] == status_counts + + +def test_console_caps_ordered_age_preview_at_six_entries(capsys): + sent_dates = tuple( + REPORT_NOW.replace(tzinfo=None) - timedelta(days=age) + for age in range(8, 0, -1) + ) + state = AuthBusinessState( + business_identifier="BC999", + entity_ids=("9",), + invite_count=8, + invite_diagnostics=AuthInviteDiagnostics( + total_count=8, + unclaimed_count=8, + unaffiliated_unclaimed_count=8, + unaffiliated_unclaimed_sent_dates=sent_dates, + ), + ) + criteria = parse_invite_criteria("count>=1", now=REPORT_NOW, expiry_days=None) + assert criteria is not None + + rows = build_inspection_rows([state], criteria) + print_inspection_rows(rows, ConsoleLimit(max_rows=None)) + console = capsys.readouterr().out + + assert "ages=8,7,6,5,4,3d,+2" in console + assert "recipient_email" not in str(rows) + assert "token" not in str(rows) + assert "additional_message" not in str(rows) + + +def test_helper_export_and_verify_classification_compatibility(): + assert auth_report_helpers.AuthInviteDiagnostics is AuthInviteDiagnostics + + legacy_state = AuthBusinessState(business_identifier="BC123", entity_ids=("1",), invite_count=1) + result = build_verification_results([legacy_state], (CHECK_INVITE,))[0] + assert result.invite_success is True + assert result.invite_count == 1 + assert legacy_state.invite_diagnostics == AuthInviteDiagnostics() diff --git a/data-tool/tests/flows/test_auth_invite_filters.py b/data-tool/tests/flows/test_auth_invite_filters.py new file mode 100644 index 0000000000..084ff1be1d --- /dev/null +++ b/data-tool/tests/flows/test_auth_invite_filters.py @@ -0,0 +1,912 @@ +from datetime import datetime, timedelta, timezone +import importlib.util +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +FLOWS_PATH = Path(__file__).resolve().parents[2] / "flows" +CONFIG_PATH = FLOWS_PATH / "config.py" +sys.path.insert(0, str(FLOWS_PATH)) + +import auth.inspect_auth_flow as inspect_module # noqa: E402 +from auth import auth_report_helpers # noqa: E402 +from auth.auth_orchestration import normalize_auth_repeatable_cycle_key # noqa: E402 +from auth.inspect_auth_flow import ( # noqa: E402 + _format_start_message, + _run_inspect_auth_flow_with_engines, + parse_business_names_max_length, + validate_inspect_config, +) +from auth.verify_auth_flow import ( # noqa: E402 + INSPECT_FILTER_ALL, + INSPECT_FILTER_ENTITY_WITHOUT_AFFILIATION, + INSPECT_FILTER_ENTITY_WITHOUT_CONTACT, + INSPECT_FILTER_ENTITY_WITHOUT_INVITE, + INSPECT_FILTER_HAS_AFFILIATION, + INSPECT_FILTER_HAS_ANY_AUTH, + INSPECT_FILTER_HAS_CONTACT, + INSPECT_FILTER_HAS_ENTITY, + INSPECT_FILTER_HAS_INVITE, + INSPECT_FILTER_MISSING_ENTITY, + INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA, + AuthBatchResult, + AuthBusinessState, + AuthInviteDiagnostics, + InviteCriteria, + _AuthInviteAccumulator, + _all_inspection_summary_metric_rows, + _inspection_summary_metric_rows, + auth_state_matches_inspect_filter, + build_inspection_identifier_rows, + build_inspection_rows, + build_inspection_summary, + filter_inspection_states, + format_clock_minutes_z, + format_clock_z, + parse_inspect_filter, + parse_invite_criteria, + parse_invite_expiry_days, + print_inspection_summary, + write_inspection_summary_txt, +) + + +NEW_FILTER = INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA +NOW = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) +NOW_NAIVE = NOW.replace(tzinfo=None) +CUTOFF = NOW_NAIVE - timedelta(days=30) + + +def _criteria(raw="count>=1, all_expired", *, expiry_days=30): + criteria = parse_invite_criteria(raw, now=NOW, expiry_days=expiry_days) + assert criteria is not None + return criteria + + +CRITERIA = _criteria() + + +def _config(tmp_path, **overrides): + values = { + "AUTH_REPORT_BATCHES": "1", + "AUTH_REPORT_BATCH_SIZE": "10", + "INSPECT_AUTH_FILTER": "ALL", + "INSPECT_AUTH_CONSOLE_LIMIT": "0", + "INSPECT_AUTH_INVITE_EXPIRY_DAYS": None, + "INSPECT_AUTH_INVITE_CRITERIA": None, + "AUTH_OUTPUT_PATH": str(tmp_path), + "AUTH_SELECTION_MODE": "MANUAL", + "AUTH_CORP_NUMS": "BC123,BC456", + "AUTH_AFFILIATION_ACCOUNT_IDS_RAW": "", + } + values.update(overrides) + return SimpleNamespace(**values) + + +def _state( + identifier="BC123", + *, + entity=True, + affiliations=(), + ages=(60,), + unknown=0, + qualifying=None, + expired=None, + current=None, + claimed=0, + deleted=0, +): + sent_dates = tuple(sorted(NOW_NAIVE - timedelta(days=age) for age in ages)) + if qualifying is None: + qualifying = len(sent_dates) + unknown + if expired is None: + expired = sum(sent_date < CUTOFF for sent_date in sent_dates) + if current is None: + current = len(sent_dates) - expired + total = qualifying + claimed + deleted + return AuthBusinessState( + business_identifier=identifier, + entity_ids=("1",) if entity else (), + found_account_ids=tuple(affiliations), + invite_count=total, + invite_diagnostics=AuthInviteDiagnostics( + total_count=total, + deleted_count=deleted, + claimed_count=claimed, + unclaimed_count=qualifying, + unaffiliated_unclaimed_count=qualifying, + expired_unclaimed_unaffiliated_count=expired, + current_unclaimed_unaffiliated_count=current, + unknown_age_unclaimed_unaffiliated_count=unknown, + unaffiliated_unclaimed_sent_dates=sent_dates, + ), + ) + + +@pytest.mark.parametrize( + "env_value,expected", + [(None, "7"), ("", ""), (" ", " "), ("30", "30")], +) +def test_config_loading_distinguishes_missing_blank_and_override(monkeypatch, env_value, expected): + if env_value is None: + monkeypatch.delenv("INSPECT_AUTH_INVITE_EXPIRY_DAYS", raising=False) + else: + monkeypatch.setenv("INSPECT_AUTH_INVITE_EXPIRY_DAYS", env_value) + + spec = importlib.util.spec_from_file_location("expiry_config_boundary", CONFIG_PATH) + assert spec is not None and spec.loader is not None + config_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(config_module) + + assert config_module._Config.INSPECT_AUTH_INVITE_EXPIRY_DAYS == expected + + +def test_config_loading_defaults_business_names_max_length_to_all(monkeypatch): + monkeypatch.delenv("INSPECT_AUTH_BUSINESS_NAMES_MAX_LENGTH", raising=False) + + spec = importlib.util.spec_from_file_location("business_names_config_boundary", CONFIG_PATH) + assert spec is not None and spec.loader is not None + config_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(config_module) + + assert config_module._Config.INSPECT_AUTH_BUSINESS_NAMES_MAX_LENGTH == "ALL" + + +@pytest.mark.parametrize( + "raw,expected", + [(None, None), ("", None), (" ", None), ("ALL", None), ("all", None), ("12", 12), (1, 1)], +) +def test_parse_business_names_max_length_accepts_uncapped_and_positive_values(raw, expected): + assert parse_business_names_max_length(raw) == expected + + +@pytest.mark.parametrize("raw", ["0", 0, "-1", -1, "not-a-length"]) +def test_parse_business_names_max_length_rejects_non_positive_or_invalid_values(raw): + with pytest.raises( + ValueError, + match="INSPECT_AUTH_BUSINESS_NAMES_MAX_LENGTH must be a positive integer or ALL", + ): + parse_business_names_max_length(raw) + + +@pytest.mark.parametrize("raw", [None, "", "ALL"]) +def test_inspect_settings_default_business_names_to_uncapped(tmp_path, raw): + config = _config(tmp_path) + if raw is not None: + config.INSPECT_AUTH_BUSINESS_NAMES_MAX_LENGTH = raw + + settings = validate_inspect_config(config) + + assert settings.business_names_max_length is None + + +def test_inspect_settings_apply_business_names_cap_and_reject_invalid_values(tmp_path): + settings = validate_inspect_config( + _config(tmp_path, INSPECT_AUTH_BUSINESS_NAMES_MAX_LENGTH="31") + ) + assert settings.business_names_max_length == 31 + + invalid_config = _config(tmp_path, INSPECT_AUTH_BUSINESS_NAMES_MAX_LENGTH="0") + with pytest.raises(ValueError, match="INSPECT_AUTH_BUSINESS_NAMES_MAX_LENGTH.*positive integer or ALL"): + validate_inspect_config(invalid_config) + + +@pytest.mark.parametrize("raw,expected", [(None, 7), ("30", 30), (1, 1)]) +def test_parse_invite_expiry_days_defaults_missing_and_accepts_positive_values(raw, expected): + assert parse_invite_expiry_days(raw) == expected + + +@pytest.mark.parametrize("raw", ["", " "]) +def test_parse_invite_expiry_days_rejects_explicit_blank(raw): + with pytest.raises(ValueError, match="must not be blank.*default of 7 days"): + parse_invite_expiry_days(raw) + + +@pytest.mark.parametrize("raw", ["0", "-1", "abc"]) +def test_parse_invite_expiry_days_rejects_non_positive_or_invalid(raw): + with pytest.raises(ValueError, match="INSPECT_AUTH_INVITE_EXPIRY_DAYS.*positive integer.*use 1"): + parse_invite_expiry_days(raw) + + +def test_new_filter_token_and_criteria_type_are_exported(): + assert parse_inspect_filter(NEW_FILTER.lower()) == NEW_FILTER + assert auth_report_helpers.INSPECT_FILTER_NO_AFFILIATION_INVITE_CRITERIA == NEW_FILTER + assert auth_report_helpers.InviteCriteria is InviteCriteria + assert auth_report_helpers.format_clock_minutes_z is format_clock_minutes_z + assert auth_report_helpers.format_clock_z is format_clock_z + + +def test_format_clock_minutes_z_truncates_display_without_changing_seconds_formatter(): + value = datetime(2026, 7, 24, 12, 34, 59, 987654, tzinfo=timezone.utc) + + assert format_clock_minutes_z(value) == "2026-07-24T12:34Z" + assert format_clock_z(value) == "2026-07-24T12:34:59Z" + assert format_clock_minutes_z(None) == "" + + +def test_validation_requires_nonblank_criteria_for_new_filter(tmp_path): + config = _config(tmp_path, INSPECT_AUTH_FILTER=NEW_FILTER) + + with pytest.raises(ValueError, match="must contain at least one clause.*NO_AFFILIATION_INVITE_CRITERIA"): + validate_inspect_config(config) + + +def test_validation_rejects_criteria_without_new_filter(tmp_path): + config = _config(tmp_path, INSPECT_AUTH_INVITE_CRITERIA="count>=2") + + with pytest.raises(ValueError, match="INSPECT_AUTH_INVITE_CRITERIA only applies to.*NO_AFFILIATION"): + validate_inspect_config(config) + + +def test_validation_uses_default_expiry_for_all_expired_clause(tmp_path): + settings = validate_inspect_config( + _config( + tmp_path, + INSPECT_AUTH_FILTER=NEW_FILTER, + INSPECT_AUTH_INVITE_CRITERIA="count>=2, all_expired", + ) + ) + assert settings.invite_expiry_days == 7 + assert settings.invite_criteria is not None + assert settings.invite_criteria.cutoff == settings.invite_criteria.now - timedelta(days=7) + + +def test_validation_rejects_explicit_blank_expiry(tmp_path): + config = _config(tmp_path, INSPECT_AUTH_INVITE_EXPIRY_DAYS="") + + with pytest.raises(ValueError, match="must not be blank.*default of 7 days"): + validate_inspect_config(config) + + +def test_validation_defaults_missing_expiry_to_seven_day_diagnostics(tmp_path): + before = datetime.now(timezone.utc).replace(tzinfo=None) + settings = validate_inspect_config(_config(tmp_path)) + after = datetime.now(timezone.utc).replace(tzinfo=None) + + assert settings.invite_expiry_days == 7 + assert before.replace(microsecond=0) <= settings.run_clock <= after + assert settings.run_clock.microsecond == 0 + assert settings.invite_cutoff == settings.run_clock - timedelta(days=7) + assert settings.invite_cutoff.microsecond == 0 + assert before.replace(microsecond=0) - timedelta(days=7) <= settings.invite_cutoff <= after - timedelta(days=7) + message = _format_start_message(settings) + assert "ExpiryDays=7" in message + assert "ExpiryRole=diagnostics-only" in message + + +def test_validation_rejects_oversized_expiry_before_database_work(tmp_path): + config = _config(tmp_path, INSPECT_AUTH_INVITE_EXPIRY_DAYS="999999") + + with pytest.raises(ValueError, match="INSPECT_AUTH_INVITE_EXPIRY_DAYS.*supported range"): + validate_inspect_config(config) + + +def test_validation_builds_criteria_with_one_shared_run_clock(tmp_path): + before = datetime.now(timezone.utc).replace(tzinfo=None) + settings = validate_inspect_config( + _config( + tmp_path, + INSPECT_AUTH_FILTER=NEW_FILTER, + INSPECT_AUTH_INVITE_EXPIRY_DAYS="30", + INSPECT_AUTH_INVITE_CRITERIA="count>=3, age[1]>=90, all_expired", + ) + ) + after = datetime.now(timezone.utc).replace(tzinfo=None) + + criteria = settings.invite_criteria + assert criteria is not None + assert criteria.raw == "count>=3, age[1]>=90, all_expired" + assert before.replace(microsecond=0) <= criteria.now <= after + assert criteria.now.microsecond == 0 + assert criteria.cutoff == criteria.now - timedelta(days=30) + assert criteria.cutoff.microsecond == 0 + assert settings.run_clock == criteria.now + assert settings.invite_cutoff is criteria.cutoff + assert settings.invite_expiry_cutoff is criteria.cutoff + + +def test_validation_allows_expiry_only_as_diagnostics_with_standalone_cutoff(tmp_path): + before = datetime.now(timezone.utc).replace(tzinfo=None) + settings = validate_inspect_config( + _config(tmp_path, INSPECT_AUTH_INVITE_EXPIRY_DAYS="45") + ) + after = datetime.now(timezone.utc).replace(tzinfo=None) + + assert settings.inspect_filter == INSPECT_FILTER_ALL + assert settings.invite_criteria is None + assert before.replace(microsecond=0) <= settings.run_clock <= after + assert settings.invite_cutoff == settings.run_clock - timedelta(days=45) + assert before.replace(microsecond=0) - timedelta(days=45) <= settings.invite_cutoff <= after - timedelta(days=45) + message = _format_start_message(settings) + assert "ExpiryDays=45" in message + assert "ExpiryRole=diagnostics-only" in message + assert "Criteria=" not in message + + +def test_validation_allows_expiry_diagnostics_beside_non_expiry_criteria(tmp_path): + settings = validate_inspect_config( + _config( + tmp_path, + INSPECT_AUTH_FILTER=NEW_FILTER, + INSPECT_AUTH_INVITE_EXPIRY_DAYS="30", + INSPECT_AUTH_INVITE_CRITERIA="count>=3, newest_age>30", + ) + ) + + assert settings.invite_criteria is not None + assert not settings.invite_criteria.requires_expiry() + message = _format_start_message(settings) + assert "CriteriaRole=filter" in message + assert "ExpiryRole=diagnostics-only" in message + + +@pytest.mark.parametrize( + "sent_date,expected_expired,expected_current,expected_unknown", + [ + (CUTOFF - timedelta(seconds=1), 1, 0, 0), + (CUTOFF, 0, 1, 0), + (CUTOFF + timedelta(days=60), 0, 1, 0), + (None, 0, 0, 1), + ((CUTOFF - timedelta(days=1)).replace(tzinfo=timezone.utc), 1, 0, 0), + ], +) +def test_temporal_buckets_use_strict_cutoff_and_normalize_awareness( + sent_date, expected_expired, expected_current, expected_unknown +): + accumulator = _AuthInviteAccumulator() + accumulator.add_row( + invite_type=" UNAFFILIATED_EMAIL ", + invitation_status_code="PENDING", + sent_date=sent_date, + accepted_date=None, + is_deleted=None, + invite_cutoff=CUTOFF, + ) + diagnostics = accumulator.freeze(cutoff_configured=True) + assert diagnostics.expired_unclaimed_unaffiliated_count == expected_expired + assert diagnostics.current_unclaimed_unaffiliated_count == expected_current + assert diagnostics.unknown_age_unclaimed_unaffiliated_count == expected_unknown + + +@pytest.mark.parametrize( + "criteria,state,expected", + [ + (_criteria("count==3, all_expired"), _state(ages=(100, 60, 31)), True), + (_criteria("count==3, all_expired"), _state(ages=(100, 60, 30)), False), + (_criteria("count==3, age[1]>=90, age[3]>=30", expiry_days=None), _state(ages=(90, 60, 30)), True), + (_criteria("count==3, age[1]>=90, age[3]>=30", expiry_days=None), _state(ages=(89, 60, 30)), False), + (_criteria("count>=3, newest_age>30, age[1]>=90", expiry_days=None), _state(ages=(100, 70, 31, 40)), True), + (_criteria("count>=3, newest_age>30, age[1]>=90", expiry_days=None), _state(ages=(100, 70, 30, 40)), False), + ], +) +def test_required_operational_scenarios(criteria, state, expected): + assert auth_state_matches_inspect_filter(state, NEW_FILTER, criteria) is expected + + +@pytest.mark.parametrize( + "raw,ages,expected", + [ + ("count==1, age[1]>=30", (30,), True), + ("count==1, age[1]>=30", (29,), False), + ("count==2, age[2]>=60", (90, 60), True), + ("count==2, age[2]>=60", (90, 59), False), + ("count==3, age[3]>=90", (120, 100, 90), True), + ], +) +def test_reminder_cadence_criteria(raw, ages, expected): + criteria = _criteria(raw, expiry_days=None) + assert auth_state_matches_inspect_filter(_state(ages=ages), NEW_FILTER, criteria) is expected + + +@pytest.mark.parametrize( + "changes", + [ + {"entity": False}, + {"affiliations": ("10",)}, + {"ages": (), "qualifying": 0, "expired": 0, "current": 0}, + ], +) +def test_composite_predicate_requires_entity_no_affiliation_and_nonempty_q(changes): + assert not auth_state_matches_inspect_filter(_state(**changes), NEW_FILTER, CRITERIA) + + +def test_accepted_deleted_invite_is_excluded_from_invite_criteria(): + accumulator = _AuthInviteAccumulator() + accumulator.add_row( + invite_type="UNAFFILIATED_EMAIL", + invitation_status_code="ACCEPTED", + sent_date=NOW_NAIVE - timedelta(days=60), + accepted_date=NOW_NAIVE - timedelta(days=30), + is_deleted=True, + invite_cutoff=CUTOFF, + ) + diagnostics = accumulator.freeze(cutoff_configured=True) + state = AuthBusinessState( + business_identifier="BC123", + entity_ids=("1",), + invite_count=diagnostics.total_count, + invite_diagnostics=diagnostics, + ) + + assert diagnostics.claimed_count == 1 + assert diagnostics.deleted_count == 0 + assert diagnostics.unaffiliated_unclaimed_count == 0 + assert not auth_state_matches_inspect_filter( + state, + NEW_FILTER, + _criteria("count>=1", expiry_days=None), + ) + + +def test_age_aliases_preserve_composite_base_predicates_and_nonempty_floor(): + criteria = _criteria("all_ages>=30", expiry_days=None) + assert criteria.raw == "newest_age>=30" + assert auth_state_matches_inspect_filter(_state(ages=(90, 30)), NEW_FILTER, criteria) + assert not auth_state_matches_inspect_filter( + _state(entity=False, ages=(90, 30)), NEW_FILTER, criteria + ) + assert not auth_state_matches_inspect_filter( + _state(affiliations=("10",), ages=(90, 30)), NEW_FILTER, criteria + ) + assert not auth_state_matches_inspect_filter( + _state(ages=(), qualifying=0, expired=0, current=0), NEW_FILTER, criteria + ) + + +def test_unknown_age_only_poisons_age_and_all_expired_clauses(): + count_only = _criteria("count==2", expiry_days=None) + age_criteria = _criteria("count==2, age[1]>=30", expiry_days=None) + state = _state(ages=(90,), unknown=1) + + assert auth_state_matches_inspect_filter(state, NEW_FILTER, count_only) + assert not auth_state_matches_inspect_filter(state, NEW_FILTER, age_criteria) + assert not auth_state_matches_inspect_filter(state, NEW_FILTER, CRITERIA) + + +def test_contradictory_criteria_match_nothing_without_validation_error(): + criteria = _criteria("count==3, count>=5", expiry_days=None) + assert filter_inspection_states([_state(ages=(90, 60, 30))], NEW_FILTER, criteria) == [] + + +@pytest.mark.parametrize( + "operation,args_factory", + [ + (auth_state_matches_inspect_filter, lambda: (_state(), NEW_FILTER)), + (filter_inspection_states, lambda: ([], NEW_FILTER)), + (build_inspection_identifier_rows, lambda: ([], [], NEW_FILTER)), + (build_inspection_summary, lambda: ([], [], NEW_FILTER)), + ], +) +def test_new_filter_without_criteria_always_raises_even_for_empty_batches( + operation, args_factory +): + args = args_factory() + + with pytest.raises(ValueError, match="INSPECT_AUTH_INVITE_CRITERIA"): + operation(*args) + + +def test_preexisting_filters_are_unchanged_when_criteria_is_present(): + state = AuthBusinessState( + business_identifier="BC123", + entity_ids=("1",), + usable_contact_count=1, + found_account_ids=("10",), + invite_count=1, + ) + expected = { + INSPECT_FILTER_ALL: True, + INSPECT_FILTER_HAS_ANY_AUTH: True, + INSPECT_FILTER_HAS_ENTITY: True, + INSPECT_FILTER_MISSING_ENTITY: False, + INSPECT_FILTER_HAS_CONTACT: True, + INSPECT_FILTER_ENTITY_WITHOUT_CONTACT: False, + INSPECT_FILTER_HAS_AFFILIATION: True, + INSPECT_FILTER_ENTITY_WITHOUT_AFFILIATION: False, + INSPECT_FILTER_HAS_INVITE: True, + INSPECT_FILTER_ENTITY_WITHOUT_INVITE: False, + } + for filter_token, expected_match in expected.items(): + assert auth_state_matches_inspect_filter(state, filter_token) is expected_match + assert auth_state_matches_inspect_filter(state, filter_token, CRITERIA) is expected_match + + +def test_identifier_metric_and_clause_audit_reporting(tmp_path, capsys): + criteria = _criteria("count>=2, age[1]>=90", expiry_days=None) + matching = _state("BC123", ages=(100, 40)) + count_miss = _state("BC456", ages=(100,)) + age_miss = _state("BC789", ages=(80, 40)) + states = [matching, count_miss, age_miss] + + rows_without = build_inspection_identifier_rows(states, states, INSPECT_FILTER_ALL) + assert NEW_FILTER not in {row["inspect_filter"] for row in rows_without} + summary_without = build_inspection_summary(states, states, INSPECT_FILTER_ALL) + labels_without = {label for label, _value in _all_inspection_summary_metric_rows(summary_without)} + assert "NoAffiliationInviteCriteria" not in labels_without + + matched = filter_inspection_states(states, NEW_FILTER, criteria) + rows_with = build_inspection_identifier_rows(states, matched, INSPECT_FILTER_ALL, criteria) + new_row = next(row for row in rows_with if row["inspect_filter"] == NEW_FILTER) + assert new_row == { + "inspect_filter": NEW_FILTER, + "count": 1, + "identifiers_csv": "BC123", + } + + summary = build_inspection_summary(states, matched, INSPECT_FILTER_ALL, criteria) + assert summary["no_affiliation_invite_criteria_count"] == 1 + assert summary["clause_audit"] == (("count>=2", 2), ("age[1]>=90", 2)) + affiliated = _state("BC000", ages=(100, 40), affiliations=("10",)) + affiliated_row = build_inspection_rows([affiliated], criteria)[0] + assert affiliated_row["invite_criteria_pass"] == "true" + assert affiliated_row["invite_criteria_failed_clauses"] == "" + assert filter_inspection_states([affiliated], NEW_FILTER, criteria) == [] + assert "NoAffiliationInviteCriteria" in { + label for label, _value in _all_inspection_summary_metric_rows(summary) + } + filtered_summary = build_inspection_summary(states, matched, NEW_FILTER, criteria) + assert _inspection_summary_metric_rows(filtered_summary)[-1] == ( + "NoAffiliationInviteCriteria", + 1, + ) + + summary_path = tmp_path / "summary.txt" + write_inspection_summary_txt(str(summary_path), rows_with, summary) + summary_text = summary_path.read_text(encoding="utf-8") + assert 'Criteria="count>=2, age[1]>=90", Now=2026-07-24T12:00:00Z' in summary_text + assert "NoAffiliationInviteCriteria" in summary_text + assert "# Invite criteria clause audit (businesses passing each clause, of 3 inspected)" in summary_text + assert "count>=2" in summary_text + assert "age[1]>=90" in summary_text + + print_inspection_summary(states, matched, INSPECT_FILTER_ALL, criteria) + console = capsys.readouterr().out + assert "Invite criteria clause audit" in console + assert "of 3 inspected" in console + + +def test_age_aliases_are_canonical_in_settings_start_summary_audit_and_txt(tmp_path): + settings = validate_inspect_config( + _config( + tmp_path, + INSPECT_AUTH_FILTER=NEW_FILTER, + INSPECT_AUTH_INVITE_CRITERIA="all_ages>=30, any_age>=90", + ) + ) + criteria = settings.invite_criteria + assert criteria is not None + assert criteria.raw == "newest_age>=30, age[1]>=90" + + message = _format_start_message(settings) + assert 'Criteria="newest_age>=30, age[1]>=90"' in message + assert "all_ages" not in message + assert "any_age" not in message + + matching = _state("BC123", ages=(100, 40)) + newest_miss = _state("BC456", ages=(100, 20)) + oldest_miss = _state("BC789", ages=(80, 40)) + states = [matching, newest_miss, oldest_miss] + matched = filter_inspection_states(states, NEW_FILTER, criteria) + assert matched == [matching] + + summary = build_inspection_summary(states, matched, NEW_FILTER, criteria) + assert summary["invite_criteria"].startswith( + 'Criteria="newest_age>=30, age[1]>=90"' + ) + assert summary["clause_audit"] == (("newest_age>=30", 2), ("age[1]>=90", 2)) + assert "all_ages" not in str(summary) + assert "any_age" not in str(summary) + + summary_path = tmp_path / "alias-summary.txt" + write_inspection_summary_txt(str(summary_path), [], summary) + summary_text = summary_path.read_text(encoding="utf-8") + assert 'Criteria="newest_age>=30, age[1]>=90"' in summary_text + assert "newest_age>=30" in summary_text + assert "age[1]>=90" in summary_text + assert "all_ages" not in summary_text + assert "any_age" not in summary_text + + +def test_summary_txt_writes_diagnostics_only_expiry_line(tmp_path): + summary_path = tmp_path / "diagnostics-summary.txt" + summary = { + "inspect_filter": INSPECT_FILTER_ALL, + "expiry_days": 45, + "expiry_cutoff": datetime(2026, 6, 9, 12, 30, 0, 987654), + } + + write_inspection_summary_txt(str(summary_path), [], summary) + + summary_text = summary_path.read_text(encoding="utf-8") + assert "ExpiryDays=45, Cutoff=2026-06-09T12:30:00Z (diagnostics-only)" in summary_text + + +def test_summary_txt_does_not_duplicate_expiry_line_when_criteria_is_present(tmp_path): + summary_path = tmp_path / "criteria-summary.txt" + summary = build_inspection_summary([], [], NEW_FILTER, CRITERIA) + summary["expiry_days"] = 30 + summary["expiry_cutoff"] = CUTOFF + + write_inspection_summary_txt(str(summary_path), [], summary) + + summary_text = summary_path.read_text(encoding="utf-8") + assert 'Criteria="count>=1, all_expired"' in summary_text + assert "(diagnostics-only)" not in summary_text + assert summary_text.count("ExpiryDays=30") == 1 + + +def test_summary_txt_writes_nonempty_criteria_reminder_handoff(tmp_path): + summary_path = tmp_path / "handoff-summary.txt" + identifier_rows = [ + { + "inspect_filter": NEW_FILTER, + "count": 2, + "identifiers_csv": "BC123,BC456", + } + ] + + write_inspection_summary_txt( + str(summary_path), + identifier_rows, + handoff_filter=NEW_FILTER, + ) + + summary_text = summary_path.read_text(encoding="utf-8") + assert "# ── Next step: send reminder invites to this cohort" in summary_text + assert "AUTH_SELECTION_MODE=MIGRATION_FILTER" in summary_text + assert summary_text.count("AUTH_CORP_NUMS=BC123,BC456") == 2 + assert ( + "# REQUIRED: set a new key per reminder round, " + "e.g. AUTH_REPEATABLE_CYCLE_KEY=saf_reminder_2" + ) in summary_text + cycle_key_line = next( + line + for line in summary_text.splitlines() + if line.startswith("AUTH_REPEATABLE_CYCLE_KEY=") + ) + assert cycle_key_line == "AUTH_REPEATABLE_CYCLE_KEY=" + emitted_cycle_key = cycle_key_line.partition("=")[2] + config = SimpleNamespace(AUTH_REPEATABLE_CYCLE_KEY=emitted_cycle_key) + with pytest.raises(ValueError, match="AUTH_REPEATABLE_CYCLE_KEY is required"): + normalize_auth_repeatable_cycle_key(config) + assert normalize_auth_repeatable_cycle_key( + SimpleNamespace(AUTH_REPEATABLE_CYCLE_KEY="saf_reminder_2") + ) == "saf_reminder_2" + assert "AUTH_INVITE_IS_REMINDER=True" in summary_text + assert "Auth stores no reminder ordinal" in summary_text + + +@pytest.mark.parametrize( + "handoff_filter,count", + [ + (None, 2), + (INSPECT_FILTER_ALL, 2), + (NEW_FILTER, 0), + ], +) +def test_summary_txt_omits_handoff_outside_nonempty_criteria_cohort( + tmp_path, handoff_filter, count +): + summary_path = tmp_path / f"no-handoff-{handoff_filter}-{count}.txt" + identifier_rows = [ + { + "inspect_filter": NEW_FILTER, + "count": count, + "identifiers_csv": "BC123,BC456" if count else "", + } + ] + + write_inspection_summary_txt( + str(summary_path), + identifier_rows, + handoff_filter=handoff_filter, + ) + + summary_text = summary_path.read_text(encoding="utf-8") + assert "Next step: send reminder invites" not in summary_text + assert "AUTH_SELECTION_MODE=MIGRATION_FILTER" not in summary_text + assert "AUTH_REPEATABLE_CYCLE_KEY=" not in summary_text + assert "AUTH_INVITE_IS_REMINDER=True" not in summary_text + + +def test_start_message_reports_filter_expiry_and_normative_time_semantics(tmp_path): + settings = validate_inspect_config( + _config( + tmp_path, + INSPECT_AUTH_FILTER=NEW_FILTER, + INSPECT_AUTH_INVITE_EXPIRY_DAYS="30", + INSPECT_AUTH_INVITE_CRITERIA="count==2, all_expired, age[1]>=90", + ) + ) + message = _format_start_message(settings) + assert 'Criteria="count==2, all_expired, age[1]>=90"' in message + assert "CriteriaRole=filter" in message + assert "ExpiryRole=filter-clause" in message + assert "AgeRule=timestamp-precision vs Now" in message + assert "PositionRule=1=oldest" in message + assert "NULL sent_date fails age clauses and all_expired" in message + assert "CutoffClock=UTC once-per-run" in message + + +def test_inspect_integration_threads_same_criteria_to_batches_rows_reports_and_summary( + tmp_path, monkeypatch, capsys +): + settings = validate_inspect_config( + _config( + tmp_path, + INSPECT_AUTH_FILTER=NEW_FILTER, + INSPECT_AUTH_INVITE_CRITERIA="count>=1, all_ages>=30", + INSPECT_AUTH_BUSINESS_NAMES_MAX_LENGTH="37", + ) + ) + matching = _state("BC123", ages=(40,)) + nonmatching = _state("BC456", ages=(10,)) + captured = {} + + monkeypatch.setattr(inspect_module, "get_inspect_candidate_count_task", lambda *_args: 2) + monkeypatch.setattr( + inspect_module, + "get_inspect_candidate_page_task", + lambda *_args: ["BC123", "BC456"], + ) + + def fake_submit(_config_value, _engine, identifiers, _expected, submitted_settings): + captured["identifiers"] = identifiers + captured["batch_criteria"] = submitted_settings.invite_criteria + captured["cutoff"] = submitted_settings.invite_cutoff + return AuthBatchResult(states=[matching, nonmatching], verification_results=[]) + + real_build_rows = build_inspection_rows + + def capture_rows(states, criteria, *, reference_now=None): + captured["rows_criteria"] = criteria + captured["rows_reference_now"] = reference_now + return real_build_rows(states, criteria, reference_now=reference_now) + + monkeypatch.setattr(inspect_module, "_submit_inspect_batch", fake_submit) + monkeypatch.setattr(inspect_module, "build_inspection_rows", capture_rows) + monkeypatch.setattr(inspect_module, "print_inspection_summary", lambda *_args: None) + monkeypatch.setattr( + inspect_module, + "print_inspection_rows", + lambda *_args, **kwargs: captured.update(print_rows_kwargs=kwargs), + ) + monkeypatch.setattr(inspect_module, "print_inspection_identifier_rows", lambda *_args, **_kwargs: None) + monkeypatch.setattr( + inspect_module, + "write_inspection_report", + lambda _path, states, criteria, *, reference_now=None: captured.update( + report_states=states, + report_criteria=criteria, + report_reference_now=reference_now, + ), + ) + monkeypatch.setattr( + inspect_module, + "write_inspection_summary_txt", + lambda _path, rows, summary, *, handoff_filter=None: captured.update( + identifier_rows=rows, + summary=summary, + handoff_filter=handoff_filter, + ), + ) + + summary = _run_inspect_auth_flow_with_engines( + SimpleNamespace(), settings, object(), object() + ) + + assert captured["identifiers"] == ["BC123", "BC456"] + assert captured["batch_criteria"] is settings.invite_criteria + assert captured["rows_criteria"] is settings.invite_criteria + assert captured["rows_reference_now"] is settings.run_clock + assert captured["print_rows_kwargs"]["business_names_max_length"] == 37 + assert captured["report_criteria"] is settings.invite_criteria + assert captured["report_reference_now"] is settings.run_clock + assert captured["cutoff"] is settings.invite_cutoff + assert settings.invite_expiry_days == 7 + assert captured["report_states"] == [matching] + assert captured["summary"] is summary + assert captured["handoff_filter"] == NEW_FILTER + assert summary["expiry_days"] == settings.invite_expiry_days + assert summary["expiry_cutoff"] is settings.invite_cutoff + assert summary["inspected_count"] == 2 + assert summary["matched_count"] == 1 + assert summary["no_affiliation_invite_criteria_count"] == 1 + assert settings.invite_criteria.raw == "count>=1, newest_age>=30" + assert summary["clause_audit"] == (("count>=1", 2), ("newest_age>=30", 1)) + assert "all_ages" not in str(summary) + assert captured["identifier_rows"] == [ + {"inspect_filter": NEW_FILTER, "count": 1, "identifiers_csv": "BC123"} + ] + console = capsys.readouterr().out + clock_line = ( + f"🕐 Invite clock: Now={format_clock_minutes_z(settings.run_clock)} " + f"Cutoff={format_clock_minutes_z(settings.invite_cutoff)} " + f"ExpiryDays={settings.invite_expiry_days} " + "(expired means sent_date < Cutoff; one clock per run)" + ) + assert clock_line in console + assert format_clock_minutes_z(settings.run_clock).count(":") == 1 + assert format_clock_minutes_z(settings.invite_cutoff).count(":") == 1 + assert format_clock_z(settings.run_clock).count(":") == 2 + assert format_clock_z(settings.invite_cutoff).count(":") == 2 + assert ( + f"➡️ Reminder handoff block written to {settings.summary_path} — " + "set a new AUTH_REPEATABLE_CYCLE_KEY before running make run-auth-invite." + ) in console + + +def test_inspect_diagnostics_only_prints_clock_and_writes_expiry_without_handoff( + tmp_path, monkeypatch, capsys +): + settings = validate_inspect_config( + _config(tmp_path, INSPECT_AUTH_INVITE_EXPIRY_DAYS="45") + ) + monkeypatch.setattr(inspect_module, "get_inspect_candidate_count_task", lambda *_args: 0) + + summary = _run_inspect_auth_flow_with_engines( + SimpleNamespace(), settings, object(), object() + ) + + console = capsys.readouterr().out + clock_line = ( + f"🕐 Invite clock: Now={format_clock_minutes_z(settings.run_clock)} " + f"Cutoff={format_clock_minutes_z(settings.invite_cutoff)} ExpiryDays=45 " + "(expired means sent_date < Cutoff; one clock per run)" + ) + assert clock_line in console + assert format_clock_minutes_z(settings.run_clock).count(":") == 1 + assert format_clock_minutes_z(settings.invite_cutoff).count(":") == 1 + assert format_clock_z(settings.run_clock).count(":") == 2 + assert format_clock_z(settings.invite_cutoff).count(":") == 2 + assert "Reminder handoff block written" not in console + summary_text = Path(settings.summary_path).read_text(encoding="utf-8") + assert ( + f"ExpiryDays=45, Cutoff={format_clock_z(settings.invite_cutoff)} " + "(diagnostics-only)" + ) in summary_text + assert "Next step: send reminder invites" not in summary_text + assert summary["expiry_days"] == 45 + assert summary["expiry_cutoff"] is settings.invite_cutoff + + +def test_inspect_batch_failure_still_prevents_partial_report_writes(tmp_path, monkeypatch): + settings = validate_inspect_config( + _config( + tmp_path, + INSPECT_AUTH_FILTER=NEW_FILTER, + INSPECT_AUTH_INVITE_CRITERIA="count>=1", + ) + ) + writes = [] + monkeypatch.setattr(inspect_module, "get_inspect_candidate_count_task", lambda *_args: 1) + monkeypatch.setattr( + inspect_module, + "get_inspect_candidate_page_task", + lambda *_args: ["BC123"], + ) + monkeypatch.setattr( + inspect_module, + "_submit_inspect_batch", + lambda *_args: (_ for _ in ()).throw(RuntimeError("batch failed")), + ) + monkeypatch.setattr( + inspect_module, + "write_inspection_report", + lambda *_args: writes.append("inspection"), + ) + monkeypatch.setattr( + inspect_module, + "write_inspection_summary_txt", + lambda *_args: writes.append("summary"), + ) + + config = SimpleNamespace() + legal_engine = object() + auth_engine = object() + with pytest.raises(RuntimeError, match="batch failed"): + _run_inspect_auth_flow_with_engines(config, settings, legal_engine, auth_engine) + assert writes == [] diff --git a/data-tool/tests/flows/test_auth_invite_reminder.py b/data-tool/tests/flows/test_auth_invite_reminder.py new file mode 100644 index 0000000000..29cc3b6278 --- /dev/null +++ b/data-tool/tests/flows/test_auth_invite_reminder.py @@ -0,0 +1,218 @@ +import importlib +import json +import sys +from http import HTTPStatus +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + + +FLOWS_PATH = Path(__file__).resolve().parents[2] / 'flows' +sys.path.insert(0, str(FLOWS_PATH)) + +from auth.auth_create_flow import _build_auth_create_plan # noqa: E402 +from auth.auth_invite_flow import _build_auth_invite_plan # noqa: E402 +from auth.auth_models import AuthCreatePlan # noqa: E402 +from auth.auth_tasks import perform_auth_create_for_corp # noqa: E402 +from common.auth_service import AuthService # noqa: E402 + + +@pytest.fixture +def config(): + return SimpleNamespace( + AUTH_SVC_URL='https://auth.example', + ACCOUNT_SVC_TIMEOUT=17, + USE_CUSTOM_CONTACT_EMAIL=False, + ) + + +def test_config_boolean_defaults_false_and_parses_true(monkeypatch): + import config as config_module + + with monkeypatch.context() as env: + env.delenv('AUTH_INVITE_IS_REMINDER', raising=False) + config_module = importlib.reload(config_module) + assert config_module._Config.AUTH_INVITE_IS_REMINDER is False + + env.setenv('AUTH_INVITE_IS_REMINDER', 'true') + config_module = importlib.reload(config_module) + assert config_module._Config.AUTH_INVITE_IS_REMINDER is True + + importlib.reload(config_module) + + +def test_auth_create_plan_reminder_defaults_false(): + assert AuthCreatePlan().invite_is_reminder is False + + +@pytest.mark.parametrize('builder', [_build_auth_create_plan, _build_auth_invite_plan]) +@pytest.mark.parametrize(('configured_value', 'expected'), [(None, False), (False, False), (True, True)]) +def test_flow_plan_builders_propagate_reminder_config(builder, configured_value, expected): + config = SimpleNamespace() + if configured_value is not None: + config.AUTH_INVITE_IS_REMINDER = configured_value + + assert builder(config).invite_is_reminder is expected + + +@pytest.mark.parametrize(('configured_value', 'expected'), [(None, False), (False, False), (True, True)]) +def test_auth_create_plan_builder_preserves_invite_send_config(configured_value, expected): + config = SimpleNamespace() + if configured_value is not None: + config.AUTH_SEND_UNAFFILIATED_EMAIL = configured_value + + assert _build_auth_create_plan(config).send_unaffiliated_invite is expected + + +@pytest.mark.parametrize('configured_value', [None, False, True]) +def test_auth_invite_plan_builder_always_sends_invite(configured_value): + config = SimpleNamespace() + if configured_value is not None: + config.AUTH_SEND_UNAFFILIATED_EMAIL = configured_value + + assert _build_auth_invite_plan(config).send_unaffiliated_invite is True + + +@pytest.mark.parametrize('explicit_false', [False, True]) +def test_service_omitted_or_false_sends_empty_body_without_params(monkeypatch, config, explicit_false): + response = SimpleNamespace(status_code=HTTPStatus.CREATED) + post = Mock(return_value=response) + monkeypatch.setattr('common.auth_service.requests.post', post) + + kwargs = {'token': 'token'} + if explicit_false: + kwargs['is_reminder'] = False + + status = AuthService.send_unaffiliated_email(config, 'BC123', 'user@example.com', **kwargs) + + assert status == HTTPStatus.OK + post.assert_called_once_with( + url='https://auth.example/affiliationInvitations/unaffiliated/BC123', + headers={'Content-Type': 'application/json', 'Authorization': 'Bearer token'}, + data=json.dumps({}), + timeout=17, + ) + + +def test_service_true_sends_exact_reminder_body_without_params(monkeypatch, config): + response = SimpleNamespace(status_code=HTTPStatus.OK) + post = Mock(return_value=response) + monkeypatch.setattr('common.auth_service.requests.post', post) + + status = AuthService.send_unaffiliated_email( + config, + 'BC123', + 'user@example.com', + token='token', + is_reminder=True, + ) + + assert status == HTTPStatus.OK + post.assert_called_once_with( + url='https://auth.example/affiliationInvitations/unaffiliated/BC123', + headers={'Content-Type': 'application/json', 'Authorization': 'Bearer token'}, + data=json.dumps({'isReminder': True}), + timeout=17, + ) + + +@pytest.mark.parametrize( + ('response_code', 'expected_status'), + [(HTTPStatus.BAD_REQUEST, HTTPStatus.BAD_REQUEST), (599, HTTPStatus.BAD_REQUEST)], +) +def test_service_reminder_preserves_non_success_status_mapping( + monkeypatch, config, response_code, expected_status +): + response = SimpleNamespace(status_code=response_code) + post = Mock(return_value=response) + monkeypatch.setattr('common.auth_service.requests.post', post) + + status = AuthService.send_unaffiliated_email( + config, + 'BC123', + 'user@example.com', + token='token', + is_reminder=True, + ) + + assert status == expected_status + assert post.call_args.kwargs['data'] == json.dumps({'isReminder': True}) + assert 'params' not in post.call_args.kwargs + + +def test_service_without_token_never_posts(monkeypatch, config): + post = Mock() + monkeypatch.setattr('common.auth_service.requests.post', post) + monkeypatch.setattr(AuthService, 'get_bearer_token', Mock(return_value=None)) + + status = AuthService.send_unaffiliated_email( + config, + 'BC123', + 'user@example.com', + is_reminder=True, + ) + + assert status == HTTPStatus.UNAUTHORIZED + post.assert_not_called() + + +@pytest.mark.parametrize('is_reminder', [False, True]) +def test_task_passes_reminder_and_reports_detail(monkeypatch, config, is_reminder): + send = Mock(return_value=HTTPStatus.OK) + monkeypatch.setattr(AuthService, 'send_unaffiliated_email', send) + plan = AuthCreatePlan( + create_entity=False, + send_unaffiliated_invite=True, + invite_is_reminder=is_reminder, + ) + + result = perform_auth_create_for_corp.fn( + config, + 'BC123', + {'identifier': 'BC123', 'admin_email': 'user@example.com'}, + [], + plan, + 'token', + ) + + send.assert_called_once_with( + config=config, + identifier='BC123', + email='user@example.com', + token='token', + is_reminder=is_reminder, + ) + assert ('invite:reminder' in result['action_detail']) is is_reminder + + +@pytest.mark.parametrize( + ('profile', 'plan'), + [ + ( + {'identifier': 'BC123', 'admin_email': 'user@example.com'}, + AuthCreatePlan( + create_entity=False, + send_unaffiliated_invite=True, + invite_is_reminder=True, + dry_run=True, + ), + ), + ( + {'identifier': 'BC123'}, + AuthCreatePlan( + create_entity=False, + send_unaffiliated_invite=True, + invite_is_reminder=True, + ), + ), + ], +) +def test_task_dry_run_and_missing_email_do_not_call_service(monkeypatch, config, profile, plan): + send = Mock() + monkeypatch.setattr(AuthService, 'send_unaffiliated_email', send) + + perform_auth_create_for_corp.fn(config, 'BC123', profile, [], plan, 'token') + + send.assert_not_called() diff --git a/data-tool/tests/flows/test_invite_criteria_parser.py b/data-tool/tests/flows/test_invite_criteria_parser.py new file mode 100644 index 0000000000..80435efa6d --- /dev/null +++ b/data-tool/tests/flows/test_invite_criteria_parser.py @@ -0,0 +1,451 @@ +from datetime import datetime, timedelta, timezone +import sys +from pathlib import Path + +import pytest + + +FLOWS_PATH = Path(__file__).resolve().parents[2] / "flows" +sys.path.insert(0, str(FLOWS_PATH)) + +from auth import auth_report_helpers # noqa: E402 +from auth.verify_auth_flow import ( # noqa: E402 + AuthInviteDiagnostics, + InviteClause, + InviteClauseKind, + evaluate_invite_clauses, + format_invite_criteria, + parse_invite_criteria, +) + + +NOW = datetime(2026, 7, 24, 12, 0, tzinfo=timezone.utc) +AGE_ALIAS_CASES = ( + ("all_ages", ">=", "newest_age>=30", True), + ("all_ages", ">", "newest_age>30", False), + ("all_ages", "<=", "age[1]<=30", True), + ("all_ages", "<", "age[1]<30", False), + ("any_age", ">=", "age[1]>=30", True), + ("any_age", ">", "age[1]>30", False), + ("any_age", "<=", "newest_age<=30", True), + ("any_age", "<", "newest_age<30", False), +) +AGE_ALIAS_CASE_IDS = tuple(f"{alias}{operator}" for alias, operator, _canonical, _expected in AGE_ALIAS_CASES) + + +def _parse(raw, *, expiry_days=None): + return parse_invite_criteria(raw, now=NOW, expiry_days=expiry_days) + + +def _results(raw, diagnostics, *, expiry_days=None): + criteria = _parse(raw, expiry_days=expiry_days) + assert criteria is not None + return tuple(passed for _clause, passed in evaluate_invite_clauses(diagnostics, criteria)) + + +def _age_diagnostics(*age_days, unknown_count=0): + now = NOW.replace(tzinfo=None) + return AuthInviteDiagnostics( + unaffiliated_unclaimed_count=len(age_days) + unknown_count, + unknown_age_unclaimed_unaffiliated_count=unknown_count, + unaffiliated_unclaimed_sent_dates=tuple(now - timedelta(days=days) for days in age_days), + ) + + +@pytest.mark.parametrize("raw", [None, "", " "]) +def test_blank_criteria_is_unset(raw): + assert _parse(raw) is None + + +def test_parser_normalizes_case_whitespace_aliases_and_full_expression(): + criteria = _parse(" COUNT >= 3 , AGE[ 1 ] >= 90, Newest_Age > 30, All_Expired ", expiry_days=30) + + assert criteria is not None + assert criteria.raw == "count>=3, age[1]>=90, newest_age>30, all_expired" + assert criteria.now == datetime(2026, 7, 24, 12, 0) + assert criteria.cutoff == datetime(2026, 6, 24, 12, 0) + assert criteria.requires_expiry() + assert criteria.has_age_clauses() + assert [clause.op for clause in criteria.clauses] == ["GTE", "GTE", "GT", None] + + +def test_oldest_age_normalizes_to_the_same_clause_as_age_one(): + oldest = _parse("oldest_age>=90") + positional = _parse("age[1]>=90") + + assert oldest is not None and positional is not None + assert oldest.clauses == positional.clauses == ( + InviteClause(InviteClauseKind.AGE, "age[1]>=90", op="GTE", value=90, position=1), + ) + + +@pytest.mark.parametrize( + "alias,operator,canonical,_boundary_expected", + AGE_ALIAS_CASES, + ids=AGE_ALIAS_CASE_IDS, +) +def test_explicit_age_aliases_normalize_to_canonical_endpoint_clauses( + alias, + operator, + canonical, + _boundary_expected, +): + canonical_criteria = _parse(canonical) + assert canonical_criteria is not None + + for raw in (f"{alias}{operator}30", f" {alias.upper()} {operator} 30 "): + alias_criteria = _parse(raw) + + assert alias_criteria == canonical_criteria + assert alias_criteria is not None + assert alias_criteria.raw == canonical + assert alias_criteria.clauses[0] == canonical_criteria.clauses[0] + assert alias_criteria.clauses[0].raw == canonical + + +def test_explicit_age_aliases_preserve_expression_order_and_canonical_duplicates(): + criteria = _parse( + "ALL_AGES >= 30, count>=1, any_age>=90, age[2]<60, all_expired, newest_age>=30", + expiry_days=30, + ) + + assert criteria is not None + assert criteria.raw == ( + "newest_age>=30, count>=1, age[1]>=90, age[2]<60, all_expired, newest_age>=30" + ) + assert criteria.clauses[0] == criteria.clauses[-1] + assert "all_ages" not in criteria.raw + assert "any_age" not in criteria.raw + rendered = format_invite_criteria(criteria) + assert 'Criteria="newest_age>=30, count>=1, age[1]>=90, age[2]<60, all_expired, newest_age>=30"' in rendered + assert "all_ages" not in rendered + assert "any_age" not in rendered + + +def test_duplicate_clauses_are_preserved_and_longest_match_operators_are_used(): + criteria = _parse("count>=3, count>=3, age[2]<=60") + + assert criteria is not None + assert criteria.clauses[0] == criteria.clauses[1] + assert [clause.op for clause in criteria.clauses] == ["GTE", "GTE", "LTE"] + + +@pytest.mark.parametrize( + "raw,fragment", + [ + ("count=3", "clause 1 'count=3': unknown operator '='; expected one of == >= <= > <"), + ("age[0]>=30", "clause 1 'age[0]>=30': position must be >= 1 (1 = oldest)"), + ("age[1]==90", "clause 1 'age[1]==90': age clauses accept >= > <= < only"), + ( + "count>=0", + "clause 1 'count>=0': value must be a positive integer; use ENTITY_WITHOUT_INVITE for zero invites", + ), + ( + "all_expired>=1", + "clause 1 'all_expired>=1': all_expired takes no operator or value", + ), + ( + "expired_all", + "clause 1 'expired_all': unknown clause; expected count, all_expired, age[n], " + "oldest_age, newest_age, all_ages, any_age", + ), + ("count>=2, , age[1]>30", "clause 2 '': must contain a clause between commas"), + ], +) +def test_invalid_clauses_name_the_index_clause_and_reason(raw, fragment): + with pytest.raises(ValueError) as error: + _parse(raw) + + assert "INSPECT_AUTH_INVITE_CRITERIA" in str(error.value) + assert fragment in str(error.value) + + +def test_all_expired_requires_expiry_days_and_reports_its_actual_clause_index(): + with pytest.raises(ValueError) as error: + _parse("count>=3, all_expired") + + assert "clause 2 'all_expired': all_expired requires INSPECT_AUTH_INVITE_EXPIRY_DAYS" in str(error.value) + + +def test_oversized_age_value_fails_during_parsing_with_clause_context(): + oversized = "9" * 30 + with pytest.raises(ValueError) as error: + _parse(f"count>=1, age[1]>={oversized}") + + message = str(error.value) + assert f"clause 2 'age[1]>={oversized}'" in message + assert "outside the supported range for the reference clock" in message + + +@pytest.mark.parametrize("selector", ["all_ages", "any_age"]) +def test_explicit_age_aliases_reject_equality(selector): + raw = f"{selector}==30" + + with pytest.raises(ValueError) as error: + _parse(raw) + + assert f"clause 1 '{raw}': age clauses accept >= > <= < only" in str(error.value) + + +@pytest.mark.parametrize( + "raw", + [ + "all_ages>=0", + "all_ages>=-1", + "any_age<=0", + "any_age<=-1", + ], +) +def test_explicit_age_aliases_reject_nonpositive_thresholds(raw): + with pytest.raises(ValueError) as error: + _parse(raw) + + assert f"clause 1 '{raw}': value must be a positive integer" in str(error.value) + + +@pytest.mark.parametrize( + "raw", + [ + "all_ages", + "any_age", + "all_ages=30", + "any_age=30", + "all_ages>=days", + "any_age= > <= < and a positive integer" + in str(error.value) + ) + + +@pytest.mark.parametrize("raw", ["all_age>=30", "any_ages>=30", "every_age>=30"]) +def test_unknown_age_quantifier_spellings_remain_unknown_clauses(raw): + with pytest.raises(ValueError) as error: + _parse(raw) + + assert ( + f"clause 1 '{raw}': unknown clause; expected count, all_expired, age[n], oldest_age, " + "newest_age, all_ages, any_age" + in str(error.value) + ) + + +def test_malformed_explicit_age_alias_reports_its_actual_clause_index(): + with pytest.raises(ValueError) as error: + _parse("count>=1, any_age=30") + + assert "clause 2 'any_age=30': expected an age selector" in str(error.value) + + +@pytest.mark.parametrize( + "raw", + ["age[*]>=30", "age[ * ]>=30", "age[]>=30", "age[**]>=30", "age[*]"], +) +def test_bare_age_wildcard_and_malformed_variants_remain_invalid(raw): + with pytest.raises(ValueError) as error: + _parse(raw) + + assert ( + f"clause 1 '{raw}': expected an age selector followed by one of >= > <= < and a positive integer" + in str(error.value) + ) + + +@pytest.mark.parametrize( + "alias,operator,canonical,_boundary_expected", + AGE_ALIAS_CASES, + ids=AGE_ALIAS_CASE_IDS, +) +def test_oversized_explicit_age_alias_threshold_reports_canonical_clause( + alias, + operator, + canonical, + _boundary_expected, +): + oversized = "9" * 30 + canonical_prefix = canonical.removesuffix("30") + + with pytest.raises(ValueError) as error: + _parse(f"count>=1, {alias}{operator}{oversized}") + + message = str(error.value) + assert f"clause 2 '{canonical_prefix}{oversized}'" in message + assert "outside the supported range for the reference clock" in message + + +def test_formatter_and_helper_exports_expose_the_normalized_model(): + criteria = _parse("count == 3, all_expired", expiry_days=30) + + assert criteria is not None + assert format_invite_criteria(criteria) == ( + 'Criteria="count==3, all_expired", Now=2026-07-24T12:00:00Z, ' + "ExpiryDays=30, Cutoff=2026-06-24T12:00:00Z" + ) + assert auth_report_helpers.InviteClause is InviteClause + assert auth_report_helpers.InviteClauseKind is InviteClauseKind + assert auth_report_helpers.parse_invite_criteria is parse_invite_criteria + assert auth_report_helpers.evaluate_invite_clauses is evaluate_invite_clauses + + +@pytest.mark.parametrize( + "operator,actual,expected", + [ + ("==", 3, True), + ("==", 2, False), + (">=", 3, True), + (">=", 2, False), + ("<=", 3, True), + ("<=", 4, False), + (">", 4, True), + (">", 3, False), + ("<", 2, True), + ("<", 3, False), + ], +) +def test_count_clause_operator_boundaries(operator, actual, expected): + diagnostics = AuthInviteDiagnostics(unaffiliated_unclaimed_count=actual) + assert _results(f"count{operator}3", diagnostics) == (expected,) + + +@pytest.mark.parametrize( + "operator,expected", + [(">=", True), (">", False), ("<=", True), ("<", False)], +) +def test_age_clause_timestamp_precision_at_exact_threshold(operator, expected): + sent_date = NOW.replace(tzinfo=None) - timedelta(days=30) + diagnostics = AuthInviteDiagnostics( + unaffiliated_unclaimed_count=1, + unaffiliated_unclaimed_sent_dates=(sent_date,), + ) + + assert _results(f"age[1]{operator}30", diagnostics) == (expected,) + + +@pytest.mark.parametrize( + "alias,operator,canonical,boundary_expected", + AGE_ALIAS_CASES, + ids=AGE_ALIAS_CASE_IDS, +) +def test_explicit_age_aliases_match_canonical_endpoints_at_exact_boundaries( + alias, + operator, + canonical, + boundary_expected, +): + diagnostics = _age_diagnostics(30, 30) + alias_result = _results(f"{alias}{operator}30", diagnostics) + canonical_result = _results(canonical, diagnostics) + + assert alias_result == canonical_result == (boundary_expected,) + + +@pytest.mark.parametrize( + "diagnostics", + [ + pytest.param(_age_diagnostics(100, 60, 31, -1), id="future-newest"), + pytest.param(_age_diagnostics(30, 30), id="tied-endpoints"), + pytest.param(_age_diagnostics(100, unknown_count=1), id="known-and-unknown"), + pytest.param(_age_diagnostics(), id="empty"), + pytest.param(_age_diagnostics(unknown_count=2), id="all-null"), + ], +) +@pytest.mark.parametrize( + "alias,operator,canonical,_boundary_expected", + AGE_ALIAS_CASES, + ids=AGE_ALIAS_CASE_IDS, +) +def test_explicit_age_alias_direct_evaluation_equals_canonical_endpoint_across_edge_states( + diagnostics, + alias, + operator, + canonical, + _boundary_expected, +): + assert _results(f"{alias}{operator}30", diagnostics) == _results(canonical, diagnostics) + + +def test_explicit_age_aliases_preserve_future_date_endpoint_semantics(): + diagnostics = _age_diagnostics(100, -1) + + assert _results("all_ages>=30, any_age>=30", diagnostics) == (False, True) + + +@pytest.mark.parametrize( + "diagnostics", + [ + pytest.param(_age_diagnostics(100, unknown_count=1), id="known-and-unknown"), + pytest.param(_age_diagnostics(), id="empty"), + pytest.param(_age_diagnostics(unknown_count=2), id="all-null"), + ], +) +def test_explicit_age_aliases_preserve_global_null_poisoning_and_false_on_empty(diagnostics): + assert _results("all_ages>=30, any_age>=30, all_ages<=90, any_age<=90", diagnostics) == ( + False, + False, + False, + False, + ) + + +def test_positional_and_newest_age_clauses_use_oldest_first_dates(): + now = NOW.replace(tzinfo=None) + diagnostics = AuthInviteDiagnostics( + unaffiliated_unclaimed_count=4, + unaffiliated_unclaimed_sent_dates=( + now - timedelta(days=100), + now - timedelta(days=60), + now - timedelta(days=31), + now + timedelta(days=1), + ), + ) + + assert _results("age[1]>=90, age[3]>=30, newest_age>=1", diagnostics) == (True, True, False) + assert _results("age[5]>=1", diagnostics) == (False,) + + +def test_unknown_age_poisons_age_clauses_but_not_count_clauses(): + diagnostics = AuthInviteDiagnostics( + unaffiliated_unclaimed_count=2, + unknown_age_unclaimed_unaffiliated_count=1, + unaffiliated_unclaimed_sent_dates=(NOW.replace(tzinfo=None) - timedelta(days=90),), + ) + + assert _results("count==2, age[1]>=30, newest_age<999", diagnostics) == (True, False, False) + + +@pytest.mark.parametrize( + "diagnostics,expected", + [ + ( + AuthInviteDiagnostics( + unaffiliated_unclaimed_count=2, + expired_unclaimed_unaffiliated_count=2, + ), + True, + ), + ( + AuthInviteDiagnostics( + unaffiliated_unclaimed_count=2, + expired_unclaimed_unaffiliated_count=1, + ), + False, + ), + ( + AuthInviteDiagnostics( + unaffiliated_unclaimed_count=2, + expired_unclaimed_unaffiliated_count=2, + unknown_age_unclaimed_unaffiliated_count=1, + ), + False, + ), + (AuthInviteDiagnostics(), False), + ], +) +def test_all_expired_preserves_nonempty_strict_cutoff_diagnostics_semantics(diagnostics, expected): + assert _results("all_expired", diagnostics, expiry_days=30) == (expected,) diff --git a/docs/business.yaml b/docs/business.yaml index c9ae54479f..305391ea2d 100644 --- a/docs/business.yaml +++ b/docs/business.yaml @@ -3825,6 +3825,267 @@ paths: - $ref: '#/components/parameters/accountId' x-internal: false parameters: [] + '/documents/client/{filingType}/{entityType}/{documentType}': + post: + tags: + - document + summary: Save a new document. + description:

Upload a document for the first time. Information associated with the document may be submitted as any of the following request parameters

  • businessIdentifier
  • filingDate
  • filename
  • filingId
The response JSON key value is the unique identifier for the document.

+ operationId: post-document + parameters: + - schema: + type: string + description: API filing type that the client submitted document belongs to. + name: filingType + in: path + required: true + - schema: + type: string + description: API business entity type that the client submitted document belongs to. + name: entityType + in: path + required: true + - schema: + type: string + description: API document type for the client submitted document. + name: documentType + in: path + required: true + - $ref: '#/components/parameters/accountId' + - $ref: '#/components/parameters/contentType' + - schema: + type: string + description: The identifier of an entity such as a BC Company. Optionally submitted if available. + name: businessIdentifer + in: query + required: false + example: BC0799342 + - schema: + type: string + description: The document file name. Optionally submitted if available. + name: filename + in: query + required: false + - schema: + type: integer + description: The unique API filing id. Optionally submitted if available. + name: filingId + in: query + required: false + requestBody: + content: + application/pdf: + schema: + type: string + format: binary + responses: + '201': + description: Created + headers: + Access-Control-Allow-Origin: + $ref: '#/components/headers/AccessControlAllowOrigin' + Access-Control-Allow-Methods: + $ref: '#/components/headers/AccessControlAllowMethods' + Access-Control-Allow-Headers: + $ref: '#/components/headers/AccessControlAllowHeaders' + Access-Control-Max-Age: + $ref: '#/components/headers/AccessControlMaxAge' + content: + application/json: + schema: + $ref: '#/components/schemas/Document_summary' + examples: + /documents/client/specialResolution/CP/coop_rules/post: + summary: Save a new document response. + value: + key: 'COOP-DS0000101951' + documentServiceId: 'DS0000101951' + consumerFilename: '' + consumerDocumentId: '0100000683' + consumerIdentifer: 'CP02345678' + consumerReferenceId: '' + documentType: COOP + documentTypeDescription: Cooperative Rules + documentClass: COOP + createDateTime: '2024-08-01T22:58:45+00:00' + documentURL: 'https://storage.googleapis.com/business_documents/2024/08/03/corr-0100001000.pdf?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=ppr-dev-sa%40eogruh-dev.iam.gserviceaccount.com%2F20240717%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20240717T222117Z&X-Goog-Expires=604800&X-Goog-SignedHeaders=host&X-Goog-Signature=382451ed1016bdc123083a58aa2358a713628e86d3730d43301e3b7d0e088dc1762ac6be3d56fe07193c02be82aab0c9b05fef52f4919292df470ffc3298442d4e6eda90a94b7fa564d2661f43eeec33a65f72314e9a1376e89adf1fc0b5f932cd72b30c921a44eac68cac82fdd5583474de5076715ac6738918f758f2c4abb71413efeb348604baf811cd14ecb005f6977270726094a1ee0d1060f786450f6b1ef7f1251c39376b4d80f438056b6b1bed57cd0b44d4616300ae031cbfc7703b61592b2519a2d01688bafd4d24ee83fecd0a425daf50c8319d94c4a8862fde89eca86a2dd974d01e5a7b7f7343cddefb70a1d1d61d27e1bb53b400d039e08cb2' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '404': + $ref: '#/components/responses/NotFound' + '500': + $ref: '#/components/responses/InternalServerError' + '/documents/client/{documentKey}': + patch: + tags: + - document + summary: Update information associated with an existing client document. + description: '

Update document record properties for an existing document identified by a document key.

' + operationId: patch-document + parameters: + - $ref: '#/components/parameters/accountId' + - $ref: '#/components/parameters/documentKey' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Document_update' + examples: + business-example: + summary: Update a business document request. + value: + businessIdentifer: BC1500943 + filingId: 19140001 + filingDate: '2024-08-04T23:02:02+00:00' + responses: + '200': + description: OK + headers: + Access-Control-Allow-Origin: + $ref: '#/components/headers/AccessControlAllowOrigin' + Access-Control-Allow-Methods: + $ref: '#/components/headers/AccessControlAllowMethods' + Access-Control-Allow-Headers: + $ref: '#/components/headers/AccessControlAllowHeaders' + Access-Control-Max-Age: + $ref: '#/components/headers/AccessControlMaxAge' + content: + application/json: + schema: + $ref: '#/components/schemas/Document_response' + examples: + /documents/client/CORP-DS0100001003/patch: + summary: Update a business document response. + value: + message: 'Document record CORP-DS0100001003 updated successfully.' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '404': + $ref: '#/components/responses/NotFound' + '500': + $ref: '#/components/responses/InternalServerError' + put: + tags: + - document + summary: Add or replace an existing document. + description:

Add or replace the existing document record specified by the document key with the payload document.

+ operationId: put-document + parameters: + - $ref: '#/components/parameters/accountId' + - $ref: '#/components/parameters/contentType' + - $ref: '#/components/parameters/documentKey' + requestBody: + content: + application/pdf: + schema: + type: string + format: binary + responses: + '201': + description: Created + headers: + Access-Control-Allow-Origin: + $ref: '#/components/headers/AccessControlAllowOrigin' + Access-Control-Allow-Methods: + $ref: '#/components/headers/AccessControlAllowMethods' + Access-Control-Allow-Headers: + $ref: '#/components/headers/AccessControlAllowHeaders' + Access-Control-Max-Age: + $ref: '#/components/headers/AccessControlMaxAge' + content: + application/json: + schema: + $ref: '#/components/schemas/Document_response' + examples: + /documents/client/CORP-DS0100001003/patch: + summary: Replace a business document response. + value: + message: 'Document CORP-DS0100001003 replaced successfully.' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '404': + $ref: '#/components/responses/NotFound' + '500': + $ref: '#/components/responses/InternalServerError' + delete: + tags: + - document + summary: Permanently delete an existing document. + description: '

Permanently delete from document storage the existing document associated record specified by the document key. If no document exists (previously deleted or never uploaded), a 400 status code is returned.

' + operationId: delete-document + parameters: + - $ref: '#/components/parameters/accountId' + - $ref: '#/components/parameters/documentKey' + responses: + '200': + description: Ok + headers: + Access-Control-Allow-Origin: + $ref: '#/components/headers/AccessControlAllowOrigin' + Access-Control-Allow-Methods: + $ref: '#/components/headers/AccessControlAllowMethods' + Access-Control-Allow-Headers: + $ref: '#/components/headers/AccessControlAllowHeaders' + Access-Control-Max-Age: + $ref: '#/components/headers/AccessControlMaxAge' + content: + application/json: + schema: + $ref: '#/components/schemas/Document_response' + examples: + /documents/client/CORP-DS0100001003/patch: + summary: Delete a business document response. + value: + message: 'Document CORP-DS0100001003 deleted successfully.' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '404': + $ref: '#/components/responses/NotFound' + '500': + $ref: '#/components/responses/InternalServerError' + get: + tags: + - document + summary: Retrieve an existing document. + description: '

Retrieve from document storage the existing document specified by the document key. If no document exists (previously deleted or never uploaded), a 400 status code is returned.

' + operationId: get-document + parameters: + - $ref: '#/components/parameters/accountId' + - $ref: '#/components/parameters/contentType' + - $ref: '#/components/parameters/documentKey' + responses: + '200': + description: Ok + headers: + Access-Control-Allow-Origin: + $ref: '#/components/headers/AccessControlAllowOrigin' + Access-Control-Allow-Methods: + $ref: '#/components/headers/AccessControlAllowMethods' + Access-Control-Allow-Headers: + $ref: '#/components/headers/AccessControlAllowHeaders' + Access-Control-Max-Age: + $ref: '#/components/headers/AccessControlMaxAge' + content: + application/pdf: + schema: + type: string + format: binary + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '404': + $ref: '#/components/responses/NotFound' + '500': + $ref: '#/components/responses/InternalServerError' components: schemas: Address: @@ -4102,9 +4363,9 @@ components: type: string title: The type of Amalgamation enum: - - regular - - vertical - - horizontal + - regular + - vertical + - horizontal amalgamatingBusinesses: type: array items: @@ -4115,9 +4376,9 @@ components: role: type: string enum: - - amalgamating - - holding - - primary + - amalgamating + - holding + - primary identifier: type: string foreignJurisdiction: @@ -5531,6 +5792,156 @@ components: example: '***Item not used***' required: - dissolution + Document_update: + title: Document_update + type: object + description: '

For PATCH requests, holds information to update for a document identified by documentKey.

' + properties: + businessIdentifer: + type: string + description: The identifier of an entity such as a BC Company to be associated with the stored document. + example: BC1500943 + filingDate: + type: string + format: date-time + description: 'The fiing date or date and time associated with the document. If a date is submitted, the time value is set to 12 PM in the local time zone.' + example: '2024-07-30T19:00:00+00:00' + filename: + type: string + description: The file name associated with the stored document. + example: coop_rules_2026.pdf + filingId: + type: integer + description: The unique identifier for the API filing to be associated with the stored document. + example: 19140001 + x-examples: + filer-example: + businessIdentifer: BC1500943 + filename: coop_rules_2026.pdf + filingId: 19140001 + filingDate: '2024-08-04' + Document_response: + type: object + properties: + message: + type: string + description: Message indicating the status of the request. + description: Object to hold an API document request response. + required: + - message + x-examples: + example-1: + message: 'Document {document_key} deleted successfully.' + Document_summary: + title: Document_summary + type: object + description:

Holds information about a single document. For all documents the unique identifier is the key.

+ properties: + key: + type: string + description: 'Generated by the API, the business API unique identifier of the document.' + documentServiceId: + type: string + description: 'Generated by the DRS, the DRS unique identifier of the document.' + consumerDocumentId: + type: string + description: >- + The identifier of one or more documents associated with the consumer + application entity. Optionally submitted with either a POST or a PATCH + request, if not available the response returns the documentServiceId as + this value. + example: '80164321' + consumerIdentifer: + type: string + description: >- + The identifier of an entity such as a BC Company or a manufactured home + in the consumer application. Optionally submitted with POST or PATCH + requests. + example: 'BC1094361' + consumerFilename: + type: string + description: >- + The consumer application full filename of the document for display and + downloading. Not included if consumerFilenames is present. + example: coop_rules_2026.pdf + documentType: + type: string + description: The DRS document type. + documentTypeDescription: + type: string + description: >- + A description of the document type, generated by the system and returned + in the response. + documentClass: + type: string + description: The DRS document class. + createDateTime: + type: string + format: date-time + description: >- + Generated by the system, the date and time the document is + saved/uploaded. + example: '2024-08-30T00:28:24+00:00' + consumerFilingDateTime: + type: string + format: date-time + description: >- + Provided by the consumer as a request parameter, the date and time the + of the consumer application registration/filing associated with the + document. If only a date is submitted, the time value is set to 12 PM in + the local time zone. + example: '2024-07-30T16:28:24+00:00' + documentURL: + type: string + description: A time limited link to download the document from storage. + description: + type: string + description: A free text description of the document or set of documents. + example: A description of the document. + consumerReferenceId: + type: string + description: >- + The business filing ID as the DRS unique identifier for the consumer application + transaction/filing/registration that is associated with a document + record. + example: '19140001' + required: + - key + - documentServiceId + - documentType + - documentClass + - createDateTime + x-examples: + mhr-example: + documentServiceId: '0100001003' + consumerFilename: Transport Permit.pdf + consumerDocumentId: '20164321' + consumerIdentifer: '094361' + documentType: TRAN + documentTypeDescription: MHR miscellaneous documents + documentClass: MHR + createDateTime: '2024-08-03T22:58:45+00:00' + consumerFilingDateTime: '2024-08-03T19:00:00+00:00' + documentURL: >- + https://storage.googleapis.com/mhr_documents/2024/08/03/corr-0100001003.pdf?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=ppr-dev-sa%40eogruh-dev.iam.gserviceaccount.com%2F20240717%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20240717T222117Z&X-Goog-Expires=604800&X-Goog-SignedHeaders=host&X-Goog-Signature=382451ed1016bdc123083a58aa2358a713628e86d3730d43301e3b7d0e088dc1762ac6be3d56fe07193c02be82aab0c9b05fef52f4919292df470ffc3298442d4e6eda90a94b7fa564d2661f43eeec33a65f72314e9a1376e89adf1fc0b5f932cd72b30c921a44eac68cac82fdd5583474de5076715ac6738918f758f2c4abb71413efeb348604baf811cd14ecb005f6977270726094a1ee0d1060f786450f6b1ef7f1251c39376b4d80f438056b6b1bed57cd0b44d4616300ae031cbfc7703b61592b2519a2d01688bafd4d24ee83fecd0a425daf50c8319d94c4a8862fde89eca86a2dd974d01e5a7b7f7343cddefb70a1d1d61d27e1bb53b400d039e08cb2 + scanningInformation: + scanDateTime: '2024-05-08' + accessionNumber: 91-0584-1234 + batchId: '1254' + pageCount: 3 + author: John Smith + Error_response: + type: object + properties: + message: + type: string + description: Message identifying the source of the error. + description: Object to hold an API request error. + required: + - message + x-examples: + error-example-1: + message: 'Error creating document record.' filing_header: type: object description: 'Information identifying the filing. ' @@ -7486,6 +7897,47 @@ components: x-examples: notify_1: value: 362880 + responses: + BadRequest: + description: 'The server cannot or will not process the request due to an apparent client error (e.g., malformed request syntax, size too large, invalid request message framing, or deceptive request routing).' + content: + application/json: + schema: + $ref: '#/components/schemas/Error_response' + examples: + example-1: + value: + message: Bad request message. + Unauthorized: + description: 'Similar to 403 Forbidden, but specifically for use when authentication is required and has failed or has not yet been provided. The user does not have valid authentication credentials for the target resource.' + content: + application/json: + schema: + $ref: '#/components/schemas/Error_response' + examples: + example-1: + value: + message: Unauthorized message. + NotFound: + description: The requested resource could not be found but may be available in the future. Subsequent requests by the client are permissible. + content: + application/json: + schema: + $ref: '#/components/schemas/Error_response' + examples: + example-1: + value: + message: Not found message. + InternalServerError: + description: 'A generic error message, given when an unexpected condition was encountered and no more specific message is suitable.' + content: + application/json: + schema: + $ref: '#/components/schemas/Error_response' + examples: + example-1: + value: + message: Default message. parameters: accountId: name: Account-Id @@ -7516,9 +7968,26 @@ components: description: The name of the legalName of the document. schema: type: string + documentKey: + name: documentKey + in: path + description: 'The unique identifier of an existing client document, returned in a successful POST response when the document is first stored. If an invalid id is submitted with a GET/PATCH/PUT then the response is a [404] status code.' + required: true + schema: + type: string + example: CORP-DS0100001023 + contentType: + name: Content-Type + in: header + description: Specifies the format of the submitted document/file data. + required: false + schema: + type: string tags: - name: business description: business services + - name: document + description: document services - name: search description: search services security: diff --git a/gcp-jobs/bn-retry/poetry.lock b/gcp-jobs/bn-retry/poetry.lock index 557c9b06ce..0e677d7bdb 100644 --- a/gcp-jobs/bn-retry/poetry.lock +++ b/gcp-jobs/bn-retry/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "aiofiles" @@ -286,10 +286,10 @@ files = [ [[package]] name = "business-model" -version = "3.3.13" +version = "3.4.1" description = "" optional = false -python-versions = ">=3.9,<4" +python-versions = ">=3.13,<3.14" groups = ["main"] files = [] develop = false @@ -306,14 +306,14 @@ pg8000 = ">=1.31.2,<2.0.0" pycountry = ">=24.6.1,<25.0.0" pydantic = ">=2.10.6,<3.0.0" pytz = ">=2025.1,<2026.0" -registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.54"} +registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.74"} sql-versioning = {git = "https://github.com/bcgov/lear.git", rev = "main", subdirectory = "python/common/sql-versioning-alt"} [package.source] type = "git" url = "https://github.com/bcgov/lear.git" reference = "main" -resolved_reference = "05a0c09709460b0ba216089efc2ebed555ed3a4e" +resolved_reference = "ab63e41cf55bbea31a66c4f12d608bd619eef05d" subdirectory = "python/common/business-registry-model" [[package]] @@ -1531,7 +1531,7 @@ fqdn = {version = "*", optional = true, markers = "extra == \"format\""} idna = {version = "*", optional = true, markers = "extra == \"format\""} isoduration = {version = "*", optional = true, markers = "extra == \"format\""} jsonpointer = {version = ">1.13", optional = true, markers = "extra == \"format\""} -jsonschema-specifications = ">=2023.03.6" +jsonschema-specifications = ">=2023.3.6" referencing = ">=0.28.4" rfc3339-validator = {version = "*", optional = true, markers = "extra == \"format\""} rfc3987 = {version = "*", optional = true, markers = "extra == \"format\""} @@ -2439,7 +2439,7 @@ rpds-py = ">=0.7.0" [[package]] name = "registry_schemas" -version = "2.18.54" +version = "2.18.74" description = "A short description of the project" optional = false python-versions = ">=3.6" @@ -2457,8 +2457,8 @@ strict-rfc3339 = "*" [package.source] type = "git" url = "https://github.com/bcgov/business-schemas.git" -reference = "2.18.54" -resolved_reference = "be9cf4995e213d185ca2bd4e5f600e878661e734" +reference = "2.18.74" +resolved_reference = "5abdc89a0f6eb06cbbac9ad66b580d7c76190ebc" [[package]] name = "requests" diff --git a/gcp-jobs/bn-retry/pyproject.toml b/gcp-jobs/bn-retry/pyproject.toml index f328452081..40457e85db 100644 --- a/gcp-jobs/bn-retry/pyproject.toml +++ b/gcp-jobs/bn-retry/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "bn-retry" -version = "0.1.1" +version = "0.1.2" description = "BN15 retry job for firms in LEAR" authors = [{name = "vysakh-menon"}] license = {text = "BSD-3-clause"} diff --git a/gcp-jobs/email-reminder/poetry.lock b/gcp-jobs/email-reminder/poetry.lock index 1f6197ec6c..0c78b261d6 100644 --- a/gcp-jobs/email-reminder/poetry.lock +++ b/gcp-jobs/email-reminder/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "aiofiles" @@ -294,7 +294,7 @@ files = [ [[package]] name = "business-model" -version = "3.3.22" +version = "3.4.1" description = "" optional = false python-versions = ">=3.13,<3.14" @@ -314,14 +314,14 @@ pg8000 = ">=1.31.2,<2.0.0" pycountry = ">=24.6.1,<25.0.0" pydantic = ">=2.10.6,<3.0.0" pytz = ">=2025.1,<2026.0" -registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.62"} +registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.74"} sql-versioning = {git = "https://github.com/bcgov/lear.git", rev = "main", subdirectory = "python/common/sql-versioning-alt"} [package.source] type = "git" url = "https://github.com/bcgov/lear.git" reference = "main" -resolved_reference = "0078d2e1d22319c9875e06ad77ee680973c09552" +resolved_reference = "ab63e41cf55bbea31a66c4f12d608bd619eef05d" subdirectory = "python/common/business-registry-model" [[package]] @@ -351,23 +351,25 @@ subdirectory = "python/common/business-registry-account" [[package]] name = "business-registry-common" -version = "0.1.0" +version = "0.1.1" description = "" optional = false -python-versions = "^3.13" +python-versions = ">=3.13,<3.14" groups = ["main"] files = [] develop = false [package.dependencies] -datedelta = "^1.4" -pytz = "^2025.1" +datedelta = ">=1.4,<2.0" +flask = ">=3.1.0,<4.0.0" +python-dateutil = ">=2.9.0,<3.0.0" +pytz = ">=2025.1,<2026.0" [package.source] type = "git" url = "https://github.com/bcgov/lear.git" reference = "main" -resolved_reference = "a18fd64eeac8736a8481b90e50409188a91cc601" +resolved_reference = "deecf506fc9e76fb9e52e91a69db0db10d93f068" subdirectory = "python/common/business-registry-common" [[package]] @@ -2541,7 +2543,7 @@ rpds-py = ">=0.7.0" [[package]] name = "registry_schemas" -version = "2.18.62" +version = "2.18.74" description = "A short description of the project" optional = false python-versions = ">=3.6" @@ -2559,8 +2561,8 @@ strict-rfc3339 = "*" [package.source] type = "git" url = "https://github.com/bcgov/business-schemas.git" -reference = "2.18.62" -resolved_reference = "1cea81cf14104fb7d4989169236eff13b3df16c4" +reference = "2.18.74" +resolved_reference = "5abdc89a0f6eb06cbbac9ad66b580d7c76190ebc" [[package]] name = "requests" diff --git a/gcp-jobs/email-reminder/pyproject.toml b/gcp-jobs/email-reminder/pyproject.toml index 21c3c6012d..b25fee9d0f 100644 --- a/gcp-jobs/email-reminder/pyproject.toml +++ b/gcp-jobs/email-reminder/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "email-reminder" -version = "0.1.3" +version = "0.1.4" description = "" authors = [ {name = "SBC-Connect"} diff --git a/gcp-jobs/filings-notebook-report/poetry.lock b/gcp-jobs/filings-notebook-report/poetry.lock index a5429ca4d1..d4a4fbc19e 100644 --- a/gcp-jobs/filings-notebook-report/poetry.lock +++ b/gcp-jobs/filings-notebook-report/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "aiofiles" @@ -1056,10 +1056,10 @@ botocore = ["botocore"] [[package]] name = "business-model" -version = "3.1.0" +version = "3.4.1" description = "" optional = false -python-versions = ">=3.9,<4" +python-versions = ">=3.13,<3.14" groups = ["test"] files = [] develop = false @@ -1076,14 +1076,14 @@ pg8000 = ">=1.31.2,<2.0.0" pycountry = ">=24.6.1,<25.0.0" pydantic = ">=2.10.6,<3.0.0" pytz = ">=2025.1,<2026.0" -registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.39"} +registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.74"} sql-versioning = {git = "https://github.com/bcgov/lear.git", rev = "main", subdirectory = "python/common/sql-versioning-alt"} [package.source] type = "git" url = "https://github.com/bcgov/lear.git" reference = "HEAD" -resolved_reference = "e35f54b80a210de908855cc09b3f09d4efed048f" +resolved_reference = "ab63e41cf55bbea31a66c4f12d608bd619eef05d" subdirectory = "python/common/business-registry-model" [[package]] @@ -5368,7 +5368,7 @@ files = [ [[package]] name = "registry_schemas" -version = "2.18.39" +version = "2.18.74" description = "A short description of the project" optional = false python-versions = ">=3.6" @@ -5386,8 +5386,8 @@ strict-rfc3339 = "*" [package.source] type = "git" url = "https://github.com/bcgov/business-schemas.git" -reference = "2.18.39" -resolved_reference = "c0aeb44d47f8cd126c85b6ca9a8a747f33266db1" +reference = "2.18.74" +resolved_reference = "5abdc89a0f6eb06cbbac9ad66b580d7c76190ebc" [[package]] name = "requests" diff --git a/gcp-jobs/filings-notebook-report/pyproject.toml b/gcp-jobs/filings-notebook-report/pyproject.toml index 5df84d6a24..34d6c0c8b7 100644 --- a/gcp-jobs/filings-notebook-report/pyproject.toml +++ b/gcp-jobs/filings-notebook-report/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "filings-notebook-report" -version = "0.1.2" +version = "0.1.3" description = "" authors = ["BrandonSharratt "] readme = "README.md" diff --git a/gcp-jobs/involuntary-dissolutions/poetry.lock b/gcp-jobs/involuntary-dissolutions/poetry.lock index b852438a8f..0e961d6dc2 100644 --- a/gcp-jobs/involuntary-dissolutions/poetry.lock +++ b/gcp-jobs/involuntary-dissolutions/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "aiofiles" @@ -294,10 +294,10 @@ files = [ [[package]] name = "business-model" -version = "3.0.0" +version = "3.4.1" description = "" optional = false -python-versions = ">=3.9,<4" +python-versions = ">=3.13,<3.14" groups = ["main"] files = [] develop = false @@ -314,14 +314,14 @@ pg8000 = ">=1.31.2,<2.0.0" pycountry = ">=24.6.1,<25.0.0" pydantic = ">=2.10.6,<3.0.0" pytz = ">=2025.1,<2026.0" -registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.39"} +registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.74"} sql-versioning = {git = "https://github.com/bcgov/lear.git", rev = "main", subdirectory = "python/common/sql-versioning-alt"} [package.source] type = "git" url = "https://github.com/bcgov/lear.git" reference = "HEAD" -resolved_reference = "f888b656fa266eb108880a467bf6728dc6efc171" +resolved_reference = "ab63e41cf55bbea31a66c4f12d608bd619eef05d" subdirectory = "python/common/business-registry-model" [[package]] @@ -351,23 +351,25 @@ subdirectory = "python/common/business-registry-account" [[package]] name = "business-registry-common" -version = "0.1.0" +version = "0.1.1" description = "" optional = false -python-versions = "^3.13" +python-versions = ">=3.13,<3.14" groups = ["main"] files = [] develop = false [package.dependencies] -datedelta = "^1.4" -pytz = "^2025.1" +datedelta = ">=1.4,<2.0" +flask = ">=3.1.0,<4.0.0" +python-dateutil = ">=2.9.0,<3.0.0" +pytz = ">=2025.1,<2026.0" [package.source] type = "git" url = "https://github.com/bcgov/lear.git" reference = "HEAD" -resolved_reference = "f888b656fa266eb108880a467bf6728dc6efc171" +resolved_reference = "deecf506fc9e76fb9e52e91a69db0db10d93f068" subdirectory = "python/common/business-registry-common" [[package]] @@ -1240,7 +1242,7 @@ files = [ [package.dependencies] google-auth = ">=2.14.1,<3.0.0" googleapis-common-protos = ">=1.56.2,<2.0.0" -grpcio = {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} +grpcio = {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} grpcio-status = {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""} proto-plus = {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""} protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" @@ -1248,7 +1250,7 @@ requests = ">=2.18.0,<3.0.0" [package.extras] async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.dev0)"] -grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\""] +grpc = ["grpcio (>=1.33.2,<2.0.dev0)", "grpcio (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\""] grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] @@ -1506,7 +1508,7 @@ files = [ [package.dependencies] googleapis-common-protos = ">=1.5.5" grpcio = ">=1.71.0" -protobuf = ">=5.26.1,<6.0dev" +protobuf = ">=5.26.1,<6.0.dev0" [[package]] name = "idna" @@ -1634,7 +1636,7 @@ fqdn = {version = "*", optional = true, markers = "extra == \"format\""} idna = {version = "*", optional = true, markers = "extra == \"format\""} isoduration = {version = "*", optional = true, markers = "extra == \"format\""} jsonpointer = {version = ">1.13", optional = true, markers = "extra == \"format\""} -jsonschema-specifications = ">=2023.03.6" +jsonschema-specifications = ">=2023.3.6" referencing = ">=0.28.4" rfc3339-validator = {version = "*", optional = true, markers = "extra == \"format\""} rfc3987 = {version = "*", optional = true, markers = "extra == \"format\""} @@ -2553,7 +2555,7 @@ rpds-py = ">=0.7.0" [[package]] name = "registry_schemas" -version = "2.18.39" +version = "2.18.74" description = "A short description of the project" optional = false python-versions = ">=3.6" @@ -2571,8 +2573,8 @@ strict-rfc3339 = "*" [package.source] type = "git" url = "https://github.com/bcgov/business-schemas.git" -reference = "2.18.39" -resolved_reference = "c0aeb44d47f8cd126c85b6ca9a8a747f33266db1" +reference = "2.18.74" +resolved_reference = "5abdc89a0f6eb06cbbac9ad66b580d7c76190ebc" [[package]] name = "requests" diff --git a/gcp-jobs/involuntary-dissolutions/pyproject.toml b/gcp-jobs/involuntary-dissolutions/pyproject.toml index 6eae234134..78fb2b2e2d 100644 --- a/gcp-jobs/involuntary-dissolutions/pyproject.toml +++ b/gcp-jobs/involuntary-dissolutions/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "involuntary_dissolutions" -version = "0.1.1" +version = "0.1.2" description = "" authors = ["BrandonSharratt "] readme = "README.md" diff --git a/jobs/colin-extract-refresh/.env.sample b/jobs/colin-extract-refresh/.env.sample index 1e25ada2fc..999ed2a926 100644 --- a/jobs/colin-extract-refresh/.env.sample +++ b/jobs/colin-extract-refresh/.env.sample @@ -9,4 +9,15 @@ DATABASE_USERNAME_COLIN_MIGR= DATABASE_PASSWORD_COLIN_MIGR= DATABASE_NAME_COLIN_MIGR= DATABASE_HOST_COLIN_MIGR= -DATABASE_PORT_COLIN_MIGR= \ No newline at end of file +DATABASE_PORT_COLIN_MIGR= + +RESERVE_STATEMENT_TIMEOUT_MS=20000 +DATABASE_POOL_PRE_PING=True +DATABASE_POOL_SIZE=5 +DATABASE_MAX_OVERFLOW=10 +DATABASE_POOL_RECYCLE=3600 + +TARGET_SCHEMA= +TARGET_CONNECTION= +MIG_BATCH_IDS= +MIG_GROUP_IDS= \ No newline at end of file diff --git a/jobs/colin-extract-refresh/Dockerfile b/jobs/colin-extract-refresh/Dockerfile index 7f70d720d0..be6b6fb756 100644 --- a/jobs/colin-extract-refresh/Dockerfile +++ b/jobs/colin-extract-refresh/Dockerfile @@ -40,7 +40,7 @@ ENV DBSCHEMA_VERSION=9_7_1 RUN set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends default-jre-headless curl;\ - curl -fsSL "https://www.dbschema.com/download/dbschema_unix_9_7_1.tar.gz" -o /tmp/dbschema.tar.gz;\ + curl --http1.1 -fsSL "https://www.dbschema.com/download/dbschema_unix_9_7_1.tar.gz" -o /tmp/dbschema.tar.gz;\ tar -xzf /tmp/dbschema.tar.gz -C /opt; \ rm /tmp/dbschema.tar.gz; \ mkdir -p "${DBSCHEMA_HOME}/lib"; \ @@ -50,9 +50,21 @@ RUN set -eux; \ rm -rf /var/lib/apt/lists/* ENV HOME=/opt/app-root -RUN mkdir -p /opt/app-root/.DbSchema/cli /opt/app-root/.DbSchema/drivers/PostgreSql && \ - curl -fsSL "https://repo1.maven.org/maven2/org/postgresql/postgresql/42.7.4/postgresql-42.7.4.jar" -o /opt/app-root/.DbSchema/drivers/PostgreSql/postgresql.jar && \ - chmod -R g+rwX /opt/app-root/.DbSchema && \ + +RUN mkdir -p ${HOME}/.DbSchema/cli ${HOME}/.DbSchema/drivers/PostgreSql ${HOME}/.DbSchema/drivers/Oracle && \ + curl -fsSL "https://repo1.maven.org/maven2/org/postgresql/postgresql/42.7.4/postgresql-42.7.4.jar" -o ${HOME}/.DbSchema/drivers/PostgreSql/postgresql.jar && \ + printf '%s\n' 'download driver oracle' > /tmp/download_oracle_driver.sql && \ + dbschemacli /tmp/download_oracle_driver.sql && \ + if [ -f /root/.DbSchema/drivers/oracle/driver.zip ]; then \ + unzip -o /root/.DbSchema/drivers/oracle/driver.zip -d ${HOME}/.DbSchema/drivers/Oracle/; \ + elif [ -f ${HOME}/.DbSchema/drivers/oracle/driver.zip ]; then \ + unzip -o ${HOME}/.DbSchema/drivers/oracle/driver.zip -d ${HOME}/.DbSchema/drivers/Oracle/; \ + else \ + echo "Oracle JDBC driver download failed" >&2; exit 1; \ + fi && \ + rm -rf /root/.DbSchema/drivers/oracle ${HOME}/.DbSchema/drivers/oracle/driver.zip + +RUN chmod -R g+rwX ${HOME}/.DbSchema && \ chmod 755 /opt/app-root WORKDIR /opt/app-root @@ -64,7 +76,7 @@ RUN apt-get update; \ ENV PATH="${DBSCHEMA_HOME}:${PATH}" -COPY src/ ./colin-exrtact-refresh/src/ +COPY src/ ./colin-extract-refresh/src/ RUN mkdir -p /opt/app-root/colin-extract-refresh/src/subset/generated && \ chmod -R 777 /opt/app-root/colin-extract-refresh/src/subset/generated /opt/app-root/.DbSchema @@ -74,5 +86,4 @@ EXPOSE 8080 COPY src . -ENTRYPOINT ["python", "generate_cprd_subset_extract.py"] -CMD ["--corp-file", "corp.txt", "--mode", "refresh"] +CMD ["/bin/sh", "-c", "python /opt/app-root/colin-extract-refresh/src/test_connectivity.py && python /opt/app-root/colin-extract-refresh/src/refresh_extract_subset_flow.py --target-connection my_proxy_test --target-schema colin_extract --mode refresh"] \ No newline at end of file diff --git a/jobs/colin-extract-refresh/Makefile b/jobs/colin-extract-refresh/Makefile index fd41bbcdd9..22914d0963 100644 --- a/jobs/colin-extract-refresh/Makefile +++ b/jobs/colin-extract-refresh/Makefile @@ -19,7 +19,7 @@ install: ## Install python virtual environment pip install -Ur requirements.txt run: ## Run the project in local - . .venv/bin/activate && python src/test_connectivity.py + . .venv/bin/activate && python src/test_connectivity.py && python src/refresh_extract_subset_flow.py build: docker build ${DOCKER_PLATFORM} -t $(DOCKER_NAME):$(TAG_NAME) $(CURRENT_ABS_DIR) \ diff --git a/jobs/colin-extract-refresh/requirements.txt b/jobs/colin-extract-refresh/requirements.txt index d8dc4d73b4..32a54c4981 100644 --- a/jobs/colin-extract-refresh/requirements.txt +++ b/jobs/colin-extract-refresh/requirements.txt @@ -3,5 +3,8 @@ psycopg2-binary==2.9.9 oracledb==3.1.1 python-dotenv==1.1.1 pg8000==1.31.4 +pandas -cloud-sql-connector @ git+https://github.com/bcgov/sbc-connect-common.git@main#subdirectory=python/cloud-sql-connector \ No newline at end of file +cloud-sql-connector @ git+https://github.com/bcgov/sbc-connect-common.git@main#subdirectory=python/cloud-sql-connector + +common @ git+https://github.com/bcgov/lear@main#subdirectory=data-tool/flows \ No newline at end of file diff --git a/jobs/colin-extract-refresh/scripts/transfer_cprd_corps_insert.sql b/jobs/colin-extract-refresh/scripts/transfer_cprd_corps_insert.sql deleted file mode 100644 index a81d9189d3..0000000000 --- a/jobs/colin-extract-refresh/scripts/transfer_cprd_corps_insert.sql +++ /dev/null @@ -1,12 +0,0 @@ -vset cli.settings.ignore_errors=false -vset cli.settings.transfer_threads=4 -vset format.date=YYYY-MM-dd'T'hh:mm:ss'Z' -vset format.timestamp=YYYY-MM-dd'T'hh:mm:ss'Z' - -connect cdev_pg_test; - -learn schema colin_extract_temp; - --- cutoff timestamp -insert into colin_extract_temp.colin_extract_version (extracted_at) -values (current_timestamp); diff --git a/jobs/colin-extract-refresh/src/checks/check_colin.py b/jobs/colin-extract-refresh/src/checks/check_colin.py deleted file mode 100644 index e2da3fbb50..0000000000 --- a/jobs/colin-extract-refresh/src/checks/check_colin.py +++ /dev/null @@ -1,37 +0,0 @@ -import os -import sys -import oracledb -from sqlalchemy import create_engine, text -from config import get_named_config - -def _colin_oracle_init() -> None: - lib_dir = os.environ.get("ORACLE_CLIENT_LIB_DIR", "") - oracledb.init_oracle_client(lib_dir=lib_dir) - print('👷 Enable thick mode:', not oracledb.is_thin_mode()) - print('👷 Instant Client version:', oracledb.clientversion()) - - -def run_check() -> int: - cfg = get_named_config() - if not all([cfg.DB_USER_COLIN_ORACLE, cfg.DB_PASSWORD_COLIN_ORACLE, cfg.DB_NAME_COLIN_ORACLE, cfg.DB_HOST_COLIN_ORACLE, cfg.DB_PORT_COLIN_ORACLE]): - raise RuntimeError( - "Missing colin env vars" - ) - print("== running check_colin.py ==") - _colin_oracle_init() - engine = create_engine(cfg.SQLALCHEMY_DATABASE_URI_COLIN_ORACLE) - with engine.connect() as conn: - row = conn.execute(text("SELECT * FROM corporation FETCH FIRST 1 ROWS ONLY")).mappings().first() - if row is None: - print("no rows in COLIN db.") - else: - print(f"COLIN sample row found.") - return 0 - - -if __name__ == "__main__": - try: - raise SystemExit(run_check()) - except Exception as exc: - print(f"business db check failed.") - raise diff --git a/jobs/colin-extract-refresh/src/checks/test_dbschemacli.sql b/jobs/colin-extract-refresh/src/checks/test_dbschemacli.sql index 4b6e0280dd..97193568fd 100644 --- a/jobs/colin-extract-refresh/src/checks/test_dbschemacli.sql +++ b/jobs/colin-extract-refresh/src/checks/test_dbschemacli.sql @@ -1,2 +1,2 @@ connect my_proxy_test; -SELECT extracted_at from colin_extract_temp.colin_extract_version; \ No newline at end of file +SELECT extracted_at from colin_extract.colin_extract_version; \ No newline at end of file diff --git a/jobs/colin-extract-refresh/src/checks/utils.py b/jobs/colin-extract-refresh/src/checks/utils.py new file mode 100644 index 0000000000..5ba0a4f101 --- /dev/null +++ b/jobs/colin-extract-refresh/src/checks/utils.py @@ -0,0 +1,249 @@ + +from math import ceil +import os +import re +from typing import List, Literal + +import oracledb +import pandas as pd +from sqlalchemy import create_engine, text + +def colin_oracle_init(config) -> None: + try: + lib_dir = os.environ.get("ORACLE_CLIENT_LIB_DIR", "") + oracledb.init_oracle_client(lib_dir=lib_dir) + print('👷 Enable thick mode:', not oracledb.is_thin_mode()) + print('👷 Instant Client version:', oracledb.clientversion()) + engine = create_engine(config.SQLALCHEMY_DATABASE_URI_COLIN_ORACLE) + # Check oracle connection + with engine.connect() as conn: + res = conn.execute( + text("""SELECT SYS_CONTEXT('USERENV', 'DB_NAME') FROM DUAL""") + ).scalar() + if not res: + raise ValueError("Failed to retrieve the current database name.") + print(f'✅ Connected to Oracle database: {res}') + return engine + except Exception as e: + raise Exception('Failed to create engine for COLIN Oracle DB') from e + +def build_corp_list(corp_list: str, chunksize: int) -> str: + if not str(corp_list).strip(): + raise ValueError('empty corp_list') + corp_nums = re.findall(r"'([^']*)'", corp_list) + batch_size = min(chunksize, 999) + num_batches = ceil(len(corp_nums) / batch_size) + + batch_ctes: list[str] = [] + batch_names: list[str] = [] + for idx in range(num_batches): + batch = corp_nums[idx * batch_size: (idx +1)* batch_size] + name = f'corp_list_{idx+1}' + batch_names.append(name) + args_sql = ",".join("'" + x.replace("'", "''") + "'" for x in batch) + batch_ctes.append( + f'{name} AS (SELECT column_value AS corp_num FROM TABLE(sys.odcivarchar2list({args_sql})))' + ) + + union_lines = [f' SELECT corp_num FROM {batch_names[0]}'] + union_lines.extend(f' UNION ALL SELECT corp_num FROM {name}' for name in batch_names[1:]) + corp_list_cte = 'corp_list AS (\n'+ '\n'.join(union_lines) + '\n)' + return ',\n'.join([*batch_ctes, corp_list_cte]) + +def get_updated_identifiers(timestamp: str, corp_list: str, chunk_size: int, scope: Literal['batch', 'full']) -> str: + corp_list_ctes = 'WITH ' + frozen_ctes = '' + join_ctes = 'corporation' + if scope == 'batch': + join_ctes = 'corp_list' + if not str(corp_list).strip(): + raise ValueError('empty corp_list') + corp_list_ctes += build_corp_list(corp_list, chunk_size) + ',\n' + if scope == 'full': + frozen_ctes = frozen_cte() + query = f""" + {corp_list_ctes} + latest_event AS ( + SELECT e.event_id, + e.corp_num, + e.event_typ_cd, + e.event_timestmp, + e.trigger_dts, + ROW_NUMBER() OVER ( + PARTITION BY e.corp_num + ORDER BY e.event_timestmp DESC, e.event_id DESC + ) AS rn + FROM event e + JOIN {join_ctes} c + ON c.corp_num = e.corp_num + WHERE e.event_timestmp > TIMESTAMP '{timestamp}' - INTERVAL '2' HOUR + {frozen_ctes} + ) SELECT le.EVENT_ID, + le.corp_num, + le.event_typ_cd, + le.event_timestmp, + le.trigger_dts, + f.FILING_TYP_CD + FROM latest_event le + LEFT JOIN filing f on le.EVENT_ID = f.EVENT_ID + WHERE rn = 1 + ORDER BY le.event_timestmp DESC, le.EVENT_ID DESC + """ + return query + +def get_identifiers_per_batch(mig_batch_id: int, target_schema: str) -> str: + return f""" + SELECT string_agg(pg_catalog.quote_literal(trim(CAST(mcb.corp_num AS text))), ',') AS corp_list + FROM {target_schema}.mig_corp_batch mcb + WHERE mcb.mig_batch_id IN ({mig_batch_id}) + """ + +def unfreeze_identifiers(target_schema: str) -> str: + return f""" + UPDATE {target_schema}.corporation AS c + SET corp_frozen_type_cd = NULL + FROM {target_schema}.mig_group AS mg + JOIN {target_schema}.mig_batch AS mb ON mb.mig_group_id = mg.id + JOIN {target_schema}.mig_corp_batch AS mcb ON mcb.mig_batch_id = mb.id + WHERE c.corp_num = mcb.corp_num + -- cprd + and mg.name in ('group_0', 'group_1', 'group_3', 'group_4','gcp_migration_group_test','misc_group') + and mg.source_db = 'cprd' + and mg.target_environment = 'prod' + AND c.corp_frozen_type_cd IS NOT NULL; + """ + +def frozen_cte() -> str: + return f""" + AND NOT ( + EXISTS ( + SELECT 1 + FROM corporation c2 + WHERE c2.corp_num = e.corp_num + AND c2.corp_frozen_typ_cd = 'C' + ) + AND EXISTS ( + SELECT 1 + FROM corp_early_adopters cea + WHERE cea.corp_num = e.corp_num + ) + ) + """ + +def get_updated_identifiers_for_batch(timestamp: str, corp_list: str, chunk_size: int, scope: str) -> str: + """per batch get identifiers""" + return get_updated_identifiers(timestamp, corp_list, chunk_size, scope) + + +BC_PREFIX_RE = re.compile(r"^BC(\d+)$", re.IGNORECASE) + +def convert_result_set_to_dict(rs): + df = pd.DataFrame(rs, columns=rs.keys()) + result_dict = df.to_dict('records') + return result_dict + +def corpnum_to_oracle_ids(target_ids: str | bytes | tuple | list | None) -> List[str]: + """ + Convert TARGET/Postgres corp ids into Oracle corporation.corp_num values. + + For ids like BC0460007 -> 0460007 + Otherwise leave as-is (A1234567 -> A1234567) + + De-dupe while preserving order (avoid wasting Oracle IN-list slots). + """ + if target_ids is None: + return None + + if isinstance(target_ids, (tuple, list)) and len(target_ids) == 1: + target_ids = target_ids[0] + + if isinstance(target_ids, bytes): + target_ids = target_ids.decode() + + raw = str(target_ids).strip() + if not raw: + return None + parsed = re.findall(r"'((?:''|[^'])*)'", raw) + if not parsed: + return None + target_ids = [p.strip() for p in parsed if p.strip()] + + out: List[str] = [] + seen: set[str] = set() + + for target_id in target_ids: + m = BC_PREFIX_RE.match(target_id) + oracle_id = m.group(1) if m else target_id + + if oracle_id not in seen: + out.append(oracle_id) + seen.add(oracle_id) + + if not out: + return None + return ",".join("'" + x.replace("'", "''") + "'" for x in out) +def colin_oracle_corp_num_list_format(corp_nums: list[str]) -> str: + def q(s: str) -> str: + return "'" + str(s).replace("'","''") + "'" + return '(' + ','.join(q(c) for c in corp_nums) + ')' + +def get_candidates_not_matching_saf_criteria_query(updated_corp_nums: list, target_schema: str) -> str: + in_list = colin_oracle_corp_num_list_format(updated_corp_nums) + return f""" + SELECT corp_num FROM {target_schema}.mv_legacy_corps_data + WHERE 1 = 1 + AND corp_num IN {in_list} + AND corp_num NOT IN ( + SELECT corp_num FROM {target_schema}.mv_legacy_corps_data + WHERE 1 = 1 + AND is_active = true + AND is_frozen = false + AND in_dissolution = false + AND migrated <> 'Y' + AND has_password = true + AND meets_main_criteria = true + AND has_3rd_party = false + AND admin_email IS NOT NULL + AND email_used_count = 1 + AND director_count = 1 + AND address_all_any_bad_count = 0 + AND meets_share_criteria = true + AND has_bar_filing = false + AND directors_within_bc = true + AND is_bad_email = false + AND is_email_excluded = false + AND is_migration_excluded = false + ) +""" + +def get_fallout_corp_nums(criteria: str, updated_corp_nums: list, target_schema: str) -> str: + key = (criteria or '').strip().upper() + if key == 'SAF': + return get_candidates_not_matching_saf_criteria_query(updated_corp_nums, target_schema) + raise ValueError(f'unsupported criteria: {criteria}') + +def prune_candidates_from_cp(pruning_corps_list: list, target_schema: str) -> str: + in_list = colin_oracle_corp_num_list_format(pruning_corps_list) + return f""" + DELETE FROM {target_schema}.corp_processing + WHERE corp_num IN {in_list} + """ + +def prune_candidates_from_batch(pruning_corps_list: list, target_schema: str) -> str: + in_list = colin_oracle_corp_num_list_format(pruning_corps_list) + return f""" + DELETE FROM {target_schema}.mig_corp_batch + WHERE corp_num IN {in_list} + """ + +def prune_candidates_from_account(pruning_corps_list: list, target_schema: str) -> str: + in_list = colin_oracle_corp_num_list_format(pruning_corps_list) + return f""" + DELETE FROM {target_schema}.mig_corp_account + WHERE corp_num IN {in_list} + """ + +def get_cutoff_timestamp_query(target_schema: str) -> str: + return f""" + SELECT extracted_at FROM {target_schema}.colin_extract_version + """ diff --git a/jobs/colin-extract-refresh/src/config.py b/jobs/colin-extract-refresh/src/config.py index 8072acd992..eff39618e2 100644 --- a/jobs/colin-extract-refresh/src/config.py +++ b/jobs/colin-extract-refresh/src/config.py @@ -85,6 +85,9 @@ class _Config(): # pylint: disable=too-few-public-methods DB_HOST_COLIN_MIGR = os.getenv('DATABASE_HOST_COLIN_MIGR', '') DB_PORT_COLIN_MIGR = os.getenv('DATABASE_PORT_COLIN_MIGR', '5432') CLOUDSQL_INSTANCE_CONNECTION_NAME = os.getenv('CLOUDSQL_INSTANCE_CONNECTION_NAME', '') + TARGET_CONNECTION = os.getenv('TARGET_CONNECTION', 'ctst_pg') + TARGET_SCHEMA = os.getenv('TARGET_SCHEMA', '') + MIG_BATCH_IDS = os.getenv('MIG_BATCH_IDS') DATABASE_IP_TYPE = os.getenv('DATABASE_IP_TYPE', 'private').lower() if CLOUDSQL_INSTANCE_CONNECTION_NAME: SQLALCHEMY_DATABASE_URI_COLIN_MIGR = "postgresql+pg8000://" diff --git a/jobs/colin-extract-refresh/src/dbschemacli_init.py b/jobs/colin-extract-refresh/src/dbschemacli_init.py index 33bf617f57..c2f84b7f87 100644 --- a/jobs/colin-extract-refresh/src/dbschemacli_init.py +++ b/jobs/colin-extract-refresh/src/dbschemacli_init.py @@ -11,11 +11,19 @@ def write_dbschema_init(cfg: _Config) -> Path: host = cfg.DB_HOST_COLIN_MIGR port = int(cfg.DB_PORT_COLIN_MIGR) name = cfg.DB_NAME_COLIN_MIGR + + ora_user = cfg.DB_USER_COLIN_ORACLE + ora_password = cfg.DB_PASSWORD_COLIN_ORACLE + ora_host = cfg.DB_HOST_COLIN_ORACLE + ora_port = int(cfg.DB_PORT_COLIN_ORACLE) + ora_name = cfg.DB_NAME_COLIN_ORACLE lines = [ 'register driver PostgreSql org.postgresql.Driver jdbc:postgresql://:/?reWriteBatchedInserts=true "port=5432"', + 'register driver Oracle oracle.jdbc.OracleDriver jdbc:oracle:thin:@:: "port=1521"', f'connection my_proxy_test -d PostgreSql -u {user} -p {password} -h {host} -P {port} -D {name}', - + f'connection cprd -d Oracle -u {ora_user} -p {ora_password} -h {ora_host} -P {ora_port} -D {ora_name}', + ] init_path.write_text('\n'.join(lines) + '\n') diff --git a/jobs/colin-extract-refresh/src/generate_cprd_subset_extract.py b/jobs/colin-extract-refresh/src/generate_cprd_subset_extract.py index 2286e6cc9e..fae5116329 100644 --- a/jobs/colin-extract-refresh/src/generate_cprd_subset_extract.py +++ b/jobs/colin-extract-refresh/src/generate_cprd_subset_extract.py @@ -112,8 +112,11 @@ class tmpl_TemplateSpec: class tmpl_TemplateBundle: pg_acquire_advisory_lock: tmpl_TemplateSpec pg_release_advisory_lock: tmpl_TemplateSpec + pg_call_address_transpose: tmpl_TemplateSpec pg_prepare_address_stage: tmpl_TemplateSpec pg_cleanup_address_stage: tmpl_TemplateSpec + pg_prepare_boolean_stage: tmpl_TemplateSpec + pg_cleanup_boolean_stage: tmpl_TemplateSpec pg_cleanup_orphan_children: tmpl_TemplateSpec disable_triggers: tmpl_TemplateSpec enable_triggers: tmpl_TemplateSpec @@ -125,6 +128,7 @@ class tmpl_TemplateBundle: transfer_chunk: tmpl_TemplateSpec delete_cars: tmpl_TemplateSpec transfer_cars: tmpl_TemplateSpec + validate_staged_transfers: tmpl_TemplateSpec # ========================= @@ -288,6 +292,8 @@ def sql_render_pg_session_probe(label: str) -> str: " clock_timestamp() AS observed_at;" ) +def sql_running(word: str) -> str: + return f"SELECT {sql_quote_literal(word)} AS running;" # ========================= # tmpl_* (template loading/validation/rendering) @@ -306,6 +312,11 @@ def tmpl_default_bundle(repo_root: Path, schema: str) -> tmpl_TemplateBundle: path=subset_dir / "subset_pg_release_advisory_lock.sql", schema=schema, ) + pg_call_address_transpose = tmpl_TemplateSpec( + name="subset_pg_call_address_transpose", + path=subset_dir / "subset_pg_call_address_transpose.sql", + schema=schema, + ) pg_prepare_address_stage = tmpl_TemplateSpec( name="subset_pg_prepare_address_stage", path=subset_dir / "subset_pg_prepare_address_stage.sql", @@ -316,6 +327,16 @@ def tmpl_default_bundle(repo_root: Path, schema: str) -> tmpl_TemplateBundle: path=subset_dir / "subset_pg_cleanup_address_stage.sql", schema=schema, ) + pg_prepare_boolean_stage = tmpl_TemplateSpec( + name="subset_pg_prepare_boolean_stage", + path=subset_dir/ "subset_pg_prepare_boolean_stage.sql", + schema=schema, + ) + pg_cleanup_boolean_stage = tmpl_TemplateSpec( + name="subset_pg_cleanup_boolean_stage", + path=subset_dir/ "subset_pg_cleanup_boolean_stage.sql", + schema=schema, + ) pg_cleanup_orphan_children = tmpl_TemplateSpec( name="subset_pg_cleanup_orphan_children", path=subset_dir / "subset_pg_cleanup_orphan_children.sql", @@ -373,12 +394,21 @@ def tmpl_default_bundle(repo_root: Path, schema: str) -> tmpl_TemplateBundle: path=subset_dir / "subset_transfer_cars.sql", schema=schema, ) + validate_staged_transfers = tmpl_TemplateSpec( + name="subset_validate_staged_transfers", + path=subset_dir / "subset_validate_staged_transfers.sql", + schema=schema, + required_tokens=(TMPL_TOKEN_CORP_IDS, TMPL_TOKEN_ORACLE_PRED, TMPL_TOKEN_ORACLE_CORP_TYPE_PRED), + ) return tmpl_TemplateBundle( pg_acquire_advisory_lock=pg_acquire_advisory_lock, pg_release_advisory_lock=pg_release_advisory_lock, + pg_call_address_transpose=pg_call_address_transpose, pg_prepare_address_stage=pg_prepare_address_stage, pg_cleanup_address_stage=pg_cleanup_address_stage, + pg_prepare_boolean_stage=pg_prepare_boolean_stage, + pg_cleanup_boolean_stage=pg_cleanup_boolean_stage, pg_cleanup_orphan_children=pg_cleanup_orphan_children, disable_triggers=disable_triggers, enable_triggers=enable_triggers, @@ -390,6 +420,7 @@ def tmpl_default_bundle(repo_root: Path, schema: str) -> tmpl_TemplateBundle: transfer_chunk=transfer_chunk, delete_cars=delete_cars, transfer_cars=transfer_cars, + validate_staged_transfers=validate_staged_transfers ) @@ -592,9 +623,9 @@ def _gen_emit_pg_disable_begin(lines: List[str], *, cfg: cfg_GenerationConfig, t lines.append(f"execute {tmpl_resolve_execute_path(templates.disable_triggers, out_dir=cfg.out_chunks_dir).as_posix()}") if cfg.mode == cfg_GenerationMode.REFRESH: lines.append("-- Refresh-only: preserved processing/tracking tables still reference corporation/event rows.") - lines.append("ALTER TABLE corp_processing DISABLE TRIGGER ALL;") - # lines.append("ALTER TABLE auth_processing DISABLE TRIGGER ALL;") - lines.append("ALTER TABLE colin_tracking DISABLE TRIGGER ALL;") + # lines.append("ALTER TABLE corp_processing DISABLE TRIGGER ALL;") + # # lines.append("ALTER TABLE auth_processing DISABLE TRIGGER ALL;") + # lines.append("ALTER TABLE colin_tracking DISABLE TRIGGER ALL;") lines.append("") return @@ -612,9 +643,9 @@ def _gen_emit_pg_disable_end(lines: List[str], *, cfg: cfg_GenerationConfig, tem lines.append(f"execute {tmpl_resolve_execute_path(templates.enable_triggers, out_dir=cfg.out_chunks_dir).as_posix()}") if cfg.mode == cfg_GenerationMode.REFRESH: lines.append("-- Refresh-only: restore preserved processing/tracking table triggers too.") - lines.append("ALTER TABLE corp_processing ENABLE TRIGGER ALL;") - # lines.append("ALTER TABLE auth_processing ENABLE TRIGGER ALL;") - lines.append("ALTER TABLE colin_tracking ENABLE TRIGGER ALL;") + # lines.append("ALTER TABLE corp_processing ENABLE TRIGGER ALL;") + # # lines.append("ALTER TABLE auth_processing ENABLE TRIGGER ALL;") + # lines.append("ALTER TABLE colin_tracking ENABLE TRIGGER ALL;") lines.append("") return @@ -660,6 +691,7 @@ def gen_build_master_script_inline( delete_chunk_files: Sequence[Path], transfer_chunk_files: Sequence[Path], pg_purge_script_path: Path, + validate_script_path: Path, ) -> str: lines: List[str] = [] lines.append("vset cli.settings.ignore_errors=false") @@ -674,6 +706,9 @@ def gen_build_master_script_inline( lines.append("") lines.append("-- Prepare shared address staging table before learning schema") lines.append(f"execute {tmpl_resolve_execute_path(templates.pg_prepare_address_stage, out_dir=cfg.out_chunks_dir).as_posix()}") + lines.append("") + lines.append("-- Prepare shared boolean staging table before learning schema") + lines.append(f"execute {tmpl_resolve_execute_path(templates.pg_prepare_boolean_stage, out_dir=cfg.out_chunks_dir).as_posix()}") lines.append(f"learn schema {cfg.target_schema};") lines.append("") lines.append(f"SET search_path TO {cfg.target_schema};") @@ -689,6 +724,7 @@ def gen_build_master_script_inline( lines.append("") if cfg.mode == cfg_GenerationMode.REFRESH: + lines.append(sql_running("orphans")) lines.append("-- Cleanup stale orphan child rows before entering the trigger-suppressed refresh window.") lines.append(f"execute {tmpl_resolve_execute_path(templates.pg_cleanup_orphan_children, out_dir=cfg.out_chunks_dir).as_posix()}") lines.append("") @@ -696,11 +732,13 @@ def gen_build_master_script_inline( _gen_emit_pg_disable_begin(lines, cfg=cfg, templates=templates) _gen_emit_refresh_fk_note(lines, cfg=cfg) if cfg.pg_debug_session_probes: + lines.append(sql_running("disable")) lines.append("-- Debug probe: backend/session state in the master script after trigger/FK suppression.") lines.append(sql_render_pg_session_probe("master:after_disable")) lines.append("") if cfg.include_cars: + lines.append(sql_running("carsdel")) lines.append("-- global cars* refresh (not corp-scoped; full dataset truncate + reload)") lines.append(f"execute {tmpl_resolve_execute_path(templates.delete_cars, out_dir=cfg.out_chunks_dir).as_posix()}") lines.append(f"execute {tmpl_resolve_execute_path(templates.transfer_cars, out_dir=cfg.out_chunks_dir).as_posix()}") @@ -709,6 +747,7 @@ def gen_build_master_script_inline( if delete_chunk_files: total = len(delete_chunk_files) lines.append("-- delete corp-scoped subset (refresh mode)") + lines.append(sql_running("delete")) for idx, chunk_file in enumerate(delete_chunk_files, start=1): lines.append(f"-- delete chunk {idx:03d}/{total:03d}") lines.append(f"execute {chunk_file.as_posix()}") @@ -716,21 +755,29 @@ def gen_build_master_script_inline( if transfer_chunk_files: total = len(transfer_chunk_files) lines.append("-- transfer corp-scoped subset from Oracle to Postgres") + lines.append(sql_running("transfer")) for idx, chunk_file in enumerate(transfer_chunk_files, start=1): lines.append(f"-- transfer chunk {idx:03d}/{total:03d}") lines.append(f"execute {chunk_file.as_posix()}") lines.append("") lines.append("-- purge BCOMPS-excluded corps (computed in Postgres after load)") + lines.append(sql_running("purge")) lines.append(f"execute {pg_purge_script_path.as_posix()}") lines.append("") + lines.append(sql_running("enable")) _gen_emit_pg_disable_end(lines, cfg=cfg, templates=templates) lines.append("-- Cleanup shared address staging table") + lines.append(sql_running("unstage")) lines.append(f"execute {tmpl_resolve_execute_path(templates.pg_cleanup_address_stage, out_dir=cfg.out_chunks_dir).as_posix()}") lines.append("") + lines.append("-- Cleanup shared idicators staging table") + lines.append(sql_running("unstage")) + lines.append(f"execute {tmpl_resolve_execute_path(templates.pg_cleanup_boolean_stage, out_dir=cfg.out_chunks_dir).as_posix()}") lines.append("-- Release subset-run advisory lock") + lines.append(sql_running("unlock")) lines.append(f"execute {tmpl_resolve_execute_path(templates.pg_release_advisory_lock, out_dir=cfg.out_chunks_dir).as_posix()}") lines.append("") @@ -739,6 +786,21 @@ def gen_build_master_script_inline( lines.append(f"execute {tmpl_resolve_execute_path(templates.pg_fastload_end, out_dir=cfg.out_chunks_dir).as_posix()}") lines.append("") + lines.append("-- Execute the provisioned address transpose while the advisory lock is held.") + lines.append( + f"execute {tmpl_resolve_execute_path(templates.pg_call_address_transpose, out_dir=cfg.out_chunks_dir).as_posix()}" + ) + lines.append("") + lines.append("-- Validate Staged table rows and events") + lines.append( + f"execute {validate_script_path.as_posix()}" + ) + lines.append("") + lines.append("-- Release subset-run advisory lock") + lines.append( + f"execute {tmpl_resolve_execute_path(templates.pg_release_advisory_lock, out_dir=cfg.out_chunks_dir).as_posix()}" + ) + return "\n".join(lines) @@ -773,6 +835,8 @@ def gen_build_master_script_vset( lines.append("") lines.append("-- Prepare shared address staging table before learning schema") lines.append(f"execute {tmpl_resolve_execute_path(templates.pg_prepare_address_stage, out_dir=cfg.out_chunks_dir).as_posix()}") + lines.append("-- Prepare shared address indicator table before learning schema") + lines.append(f"execute {tmpl_resolve_execute_path(templates.pg_prepare_boolean_stage, out_dir=cfg.out_chunks_dir).as_posix()}") lines.append(f"learn schema {cfg.target_schema};") lines.append("") @@ -860,6 +924,9 @@ def _vset_predicates(target_values: Sequence[str], oracle_values: Sequence[str]) lines.append("-- Cleanup shared address staging table") lines.append(f"execute {tmpl_resolve_execute_path(templates.pg_cleanup_address_stage, out_dir=cfg.out_chunks_dir).as_posix()}") lines.append("") + lines.append("-- Cleanup shared indicator staging table") + lines.append(f"execute {tmpl_resolve_execute_path(templates.pg_cleanup_boolean_stage, out_dir=cfg.out_chunks_dir).as_posix()}") + lines.append("") lines.append("-- Release subset-run advisory lock") lines.append(f"execute {tmpl_resolve_execute_path(templates.pg_release_advisory_lock, out_dir=cfg.out_chunks_dir).as_posix()}") lines.append("") @@ -869,6 +936,16 @@ def _vset_predicates(target_values: Sequence[str], oracle_values: Sequence[str]) lines.append(f"execute {tmpl_resolve_execute_path(templates.pg_fastload_end, out_dir=cfg.out_chunks_dir).as_posix()}") lines.append("") + lines.append("-- Execute the provisioned address transpose while the advisory lock is held.") + lines.append( + f"execute {tmpl_resolve_execute_path(templates.pg_call_address_transpose, out_dir=cfg.out_chunks_dir).as_posix()}" + ) + lines.append("") + lines.append("-- Release subset-run advisory lock") + lines.append( + f"execute {tmpl_resolve_execute_path(templates.pg_release_advisory_lock, out_dir=cfg.out_chunks_dir).as_posix()}" + ) + return "\n".join(lines) @@ -1070,8 +1147,11 @@ def run(cfg: cfg_GenerationConfig) -> int: for spec in ( templates.pg_acquire_advisory_lock, templates.pg_release_advisory_lock, + templates.pg_call_address_transpose, templates.pg_prepare_address_stage, templates.pg_cleanup_address_stage, + templates.pg_prepare_boolean_stage, + templates.pg_cleanup_boolean_stage, templates.pg_cleanup_orphan_children, templates.disable_triggers, templates.enable_triggers, @@ -1081,6 +1161,7 @@ def run(cfg: cfg_GenerationConfig) -> int: templates.pg_purge_bcomps_excluded, templates.delete_cars, templates.transfer_cars, + templates.validate_staged_transfers, ): if not spec.path.exists(): raise SystemExit(f"Missing required template: {spec.name}\nPath: {spec.path}") @@ -1161,12 +1242,33 @@ def run(cfg: cfg_GenerationConfig) -> int: probe_label="execute:pg_purge_bcomps_excluded", ) + validate_script_path = cfg.out_chunks_dir / "validate_staged_transfers.sql" + gen_write_text( + validate_script_path, + tmpl_render( + tmpl_load_text(templates.validate_staged_transfers), + replacements={ + TMPL_TOKEN_CORP_IDS: sql_render_in_list(target_ids, multiline=True, indent=" "), + TMPL_TOKEN_ORACLE_PRED: sql_render_in_predicate( + "c.CORP_NUM", + corp_to_oracle_ids(target_ids), + max_in_list=cfg.chunk_size, + multiline=True, + indent=" ", + ), + TMPL_TOKEN_ORACLE_CORP_TYPE_PRED: sql_render_oracle_corp_type_predicate(include_cp=cfg.include_cp), + TMPL_TOKEN_SCHEMA: cfg.target_schema, + }, + ), + ) + master_text = gen_build_master_script_inline( cfg=cfg, templates=templates, delete_chunk_files=delete_files, transfer_chunk_files=transfer_files, pg_purge_script_path=pg_purge_script_path, + validate_script_path=validate_script_path, ) gen_write_text(cfg.out_master, master_text) diff --git a/jobs/colin-extract-refresh/src/refresh_extract_subset_flow.py b/jobs/colin-extract-refresh/src/refresh_extract_subset_flow.py new file mode 100644 index 0000000000..8bb3df962d --- /dev/null +++ b/jobs/colin-extract-refresh/src/refresh_extract_subset_flow.py @@ -0,0 +1,249 @@ +import argparse +import os +from pathlib import Path +import re +import subprocess +import sys + +from sqlalchemy import create_engine, text +from sqlalchemy.engine import Engine +from datetime import datetime +from config import get_named_config +from checks.utils import colin_oracle_init, get_fallout_corp_nums, get_cutoff_timestamp_query, unfreeze_identifiers, get_identifiers_per_batch, corpnum_to_oracle_ids, get_updated_identifiers_for_batch + + +_DEFAULT_TARGET_CONNECTION = get_named_config().TARGET_CONNECTION +_DEFAULT_TARGET_SCHEMA = get_named_config().TARGET_SCHEMA +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT_PATH = _REPO_ROOT / 'colin-extract-refresh' / 'src' / 'generate_cprd_subset_extract.py' +_GENERATED_DIR = _REPO_ROOT / 'colin-extract-refresh' / 'src' / 'subset' / 'generated' +_SUBSET = _GENERATED_DIR / 'subset_refresh.sql' + + +def _resolve_master_script_path(out: str | None) -> Path: + if not out: + return _SUBSET.resolve() + p = Path(out).expanduser() + if p.is_absolute(): + return p.resolve() + return (_REPO_ROOT / p).resolve() + +def require_file(path: str | Path, description: str) -> Path: + """File Not Found Error""" + resolved = Path(path).expanduser().resolve() + if not resolved.is_file(): + raise FileNotFoundError(f'{description} not found (expected a file): {resolved}') + return resolved + + +def get_fallen_identifiers(updated_corp_nums: list) -> list[dict]: + """ + Get updated corp nums from colin with cutoff timestamp + """ + if not updated_corp_nums: + return [] + cfg = get_named_config() + corp_nums_prune_list_query = get_fallout_corp_nums('SAF', updated_corp_nums, target_schema=cfg.TARGET_SCHEMA) + with create_engine(cfg.SQLALCHEMY_DATABASE_URI_COLIN_MIGR).connect() as conn: + result = conn.execute(text(corp_nums_prune_list_query)).scalars().all() + rows = [str(row).strip() for row in result] + return rows + +def get_cuttoff_timestamp() -> datetime: + + cfg = get_named_config() + cuttoff_timestamp = get_cutoff_timestamp_query(cfg.TARGET_SCHEMA) + with create_engine(cfg.SQLALCHEMY_DATABASE_URI_COLIN_MIGR).begin() as conn: + cuttoff_timestamp_result = conn.execute(text(cuttoff_timestamp)).scalar() + print(f"cuttoff timestamp is {cuttoff_timestamp_result}") + return cuttoff_timestamp_result + + +def run_unfreeze_identifiers() -> None: + cfg = get_named_config() + with create_engine(cfg.SQLALCHEMY_DATABASE_URI_COLIN_MIGR).begin() as conn: + result = conn.execute(text(unfreeze_identifiers(target_schema=cfg.TARGET_SCHEMA))) + print(f'Unfroze corporation rows={result.rowcount}') + +def get_updated_identifiers_colin(cutoff_timestamp: str, mig_batch_id: int, colin_oracle_engine: Engine, chunk_size: int, scope: str) -> list[dict]: + """ + Get updated corp nums from colin with cutoff timestamp + """ + cfg = get_named_config() + corp_list = '' + if scope == 'batch': + mig_sql = get_identifiers_per_batch(mig_batch_id, target_schema=cfg.TARGET_SCHEMA) + with create_engine(cfg.SQLALCHEMY_DATABASE_URI_COLIN_MIGR).connect() as conn: + row = conn.execute(text(mig_sql)).fetchone() + + corp_list = corpnum_to_oracle_ids(row[0]) if row else None + colin_sql = get_updated_identifiers_for_batch(cutoff_timestamp, str(corp_list or ''), chunk_size, scope) + + with colin_oracle_engine.connect() as conn: + result = conn.execute(text(colin_sql)) + rows = [dict(row) for row in result.mappings()] + return rows + + +def run_cprd_subset_extract_generator( + corp_file: str, + mode: str, + chunk_size: int, + threads: int, + pg_fastload: bool, + pg_disable_method: str, + out: str | None, + include_cp: bool = False, + target_connection: str = _DEFAULT_TARGET_CONNECTION, + target_schema: str = _DEFAULT_TARGET_SCHEMA, + prefix_numeric_bc: bool = False, +) -> subprocess.CompletedProcess: + """ + Generate Commands + """ + require_file(_SCRIPT_PATH, 'Generated script') + corp_path =require_file(corp_file, 'Corp list file') + + argv = [ + sys.executable, + str(_SCRIPT_PATH), + '--corp-file', + str(corp_path), + '--mode', + mode, + '--chunk-size', + str(chunk_size), + '--threads', + str(threads), + '--pg-disable-method', + pg_disable_method, + ] + argv.extend(['--target-connection', target_connection]) + argv.extend(['--target-schema', target_schema]) + if pg_fastload: + argv.append('--pg-fastload') + if include_cp: + argv.append('--include-cp') + if prefix_numeric_bc: + argv.append('--prefix-numeric-bc') + out_path = _resolve_master_script_path(out) + out_path.parent.mkdir(parents=True, exist_ok=True) + argv.extend(['--out', str(out_path)]) + + return subprocess.run( + argv, + cwd=str(_REPO_ROOT), + capture_output=False, + text=True, + ) + +def run_dbschemacli_task(master_script: str, dbschemacli_cmd: str = 'dbschemacli') -> subprocess.CompletedProcess: + master_script_path = Path(master_script) + if not master_script_path.exists(): + raise FileNotFoundError(f'Generated script not found: {master_script_path}') + print(f'Running: {dbschemacli_cmd} {master_script_path}') + return subprocess.run( + [dbschemacli_cmd, str(master_script_path)], + cwd=str(_REPO_ROOT), + capture_output=False, + text=True, + ) + +def extract_pull_flow( + mode: str = 'load', + chunk_size: int = 999, + threads: int = 4, + pg_fastload: bool = False, + pg_disable_method: str = 'table_triggers', + out: str | None=None, + run_dbschemacli: bool = False, + dbschemacli_cmd: str = 'dbschemacli', + include_cp: bool = False, + target_connection: str = _DEFAULT_TARGET_CONNECTION, + target_schema: str = _DEFAULT_TARGET_SCHEMA, + delta_scope: str = 'batch' +) -> None: + """ + Generate files + """ + + cutoff = get_cuttoff_timestamp() + + config = get_named_config() + colin_oracle_engine = colin_oracle_init(config) + # Get Identifiers + feed_path: Path | None = None + if mode == 'refresh': + updated_rows = get_updated_identifiers_colin(cutoff_timestamp=cutoff, mig_batch_id=config.MIG_BATCH_IDS, colin_oracle_engine=colin_oracle_engine, chunk_size=chunk_size, scope=delta_scope) + print(f'Colin updated identifiers : {len(updated_rows)} rows') + _GENERATED_DIR.mkdir(parents=True, exist_ok=True) + feed_path = _GENERATED_DIR / f'refresh_corp_feed_{os.getpid()}.tmp' + seen = set() + lines = [] + updated_corp_nums = [] + for row in updated_rows: + for k, v in row.items(): + if k is None or v is None: + continue + if str(k).lower() == 'corp_num': + c = str(v).strip() + if c and c not in seen: + seen.add(c) + lines.append(c) + updated_corp_nums.append('BC'+c) + break + if not lines: + raise ValueError('refresh: no corp_num in updated_rows') + feed_path.write_text('\n'.join(lines) + '\n', encoding='utf-8') + corp_file = str(feed_path) + result: subprocess.CompletedProcess | None = None + print(f'Running CPRD subset extract generator {corp_file}') + try: + result = run_cprd_subset_extract_generator( + corp_file=corp_file, + mode=mode, + chunk_size=chunk_size, + threads=threads, + pg_fastload=pg_fastload, + include_cp=include_cp, + pg_disable_method=pg_disable_method, + out=out, + target_connection=target_connection, + target_schema=target_schema, + prefix_numeric_bc=(mode=='refresh'), + ) + finally: + if feed_path is not None: + feed_path.unlink(missing_ok=True) + if result.returncode != 0 and result is not None: + raise RuntimeError(f'Generator exited with code {result.returncode}') + print(f'generator completed successfully') + + if run_dbschemacli: + master_script = _resolve_master_script_path(out=out) + run_result = run_dbschemacli_task( + master_script=str(master_script), + dbschemacli_cmd=dbschemacli_cmd, + ) + if run_result.returncode != 0: + raise RuntimeError(f'DbSchemaCLI exited with code {run_result.returncode}') + + print('Running Unfreezing Corps.......') + run_unfreeze_identifiers() + + +if __name__ == '__main__': + p = argparse.ArgumentParser(description='Run Extract-Pull flow....') + p.add_argument('--mode', default='refresh', choices=('refresh', 'load')) + p.add_argument('--delta-scope', default='full', choices=('batch', 'full')) + p.add_argument('--chunk-size', type=int, default=900, help='Max items per IN list.') + p.add_argument('--threads', type=int, default=4, help='DBSchemaCLI transfer threads') + p.add_argument('--pg-fastload', action='store_true', help='Enable Postgres fast-load') + p.add_argument('--include-cp', action='store_true', help='Include corp type CP in subset extract queries') + p.add_argument('--pg-disable-method', default='table_triggers', choices=('table_triggers', 'replica_role')) + p.add_argument('--out', default=_SUBSET, help='Output path for generated master script.') + p.add_argument('--run-dbschemacli', action='store_false') + p.add_argument('--dbschemacli-cmd', default='dbschemacli') + p.add_argument('--target-connection', default=_DEFAULT_TARGET_CONNECTION) + p.add_argument('--target-schema', default=_DEFAULT_TARGET_SCHEMA) + extract_pull_flow(**vars(p.parse_args())) diff --git a/jobs/colin-extract-refresh/src/subset/subset_disable_triggers.sql b/jobs/colin-extract-refresh/src/subset/subset_disable_triggers.sql index 631d8e0015..c92380bc72 100644 --- a/jobs/colin-extract-refresh/src/subset/subset_disable_triggers.sql +++ b/jobs/colin-extract-refresh/src/subset/subset_disable_triggers.sql @@ -1,24 +1,2 @@ SET search_path TO TARGET_SCHEMA; - -ALTER TABLE TARGET_SCHEMA.notification - DROP CONSTRAINT fk_notification_filing ; -ALTER TABLE TARGET_SCHEMA.office - DROP CONSTRAINT fk_office_mailing_address ; -ALTER TABLE TARGET_SCHEMA.office -DROP CONSTRAINT fk_office_delivery_address ; -ALTER TABLE TARGET_SCHEMA.corp_party - DROP CONSTRAINT fk_corp_party_mailing_address ; -ALTER TABLE TARGET_SCHEMA.completing_party - DROP CONSTRAINT fk_completing_party_address ; -ALTER TABLE TARGET_SCHEMA.office - DROP CONSTRAINT fk_corp_party_delivery_address ; -ALTER TABLE TARGET_SCHEMA.notification - DROP CONSTRAINT fk_notification_address ; -ALTER TABLE TARGET_SCHEMA.filing ALTER COLUMN arrangement_ind TYPE CHARACTER VARYING(255); -ALTER TABLE TARGET_SCHEMA.corporation ALTER COLUMN send_ar_ind TYPE CHARACTER VARYING(255); -ALTER TABLE TARGET_SCHEMA.share_series ALTER COLUMN max_share_ind TYPE CHARACTER VARYING(255); -ALTER TABLE TARGET_SCHEMA.share_struct_cls ALTER COLUMN max_share_ind TYPE CHARACTER VARYING(255); -ALTER TABLE TARGET_SCHEMA.filing ALTER COLUMN court_appr_ind TYPE CHARACTER VARYING(255); -ALTER TABLE TARGET_SCHEMA.share_struct_cls ALTER COLUMN spec_rights_ind TYPE CHARACTER VARYING(255); -ALTER TABLE TARGET_SCHEMA.share_struct_cls ALTER COLUMN par_value_ind TYPE CHARACTER VARYING(255); - +call TARGET_SCHEMA.colin_fk_drop_all(NULL); \ No newline at end of file diff --git a/jobs/colin-extract-refresh/src/subset/subset_enable_triggers.sql b/jobs/colin-extract-refresh/src/subset/subset_enable_triggers.sql index 0e627bb954..8bb34d0eb9 100644 --- a/jobs/colin-extract-refresh/src/subset/subset_enable_triggers.sql +++ b/jobs/colin-extract-refresh/src/subset/subset_enable_triggers.sql @@ -1,18 +1,2 @@ SET search_path TO TARGET_SCHEMA; - -ALTER TABLE TARGET_SCHEMA.notification ADD CONSTRAINT fk_notification_filing FOREIGN KEY (event_id) REFERENCES TARGET_SCHEMA.filing (event_id); -ALTER TABLE TARGET_SCHEMA.office ADD CONSTRAINT fk_office_mailing_address FOREIGN KEY (mailing_addr_id) REFERENCES TARGET_SCHEMA.address (addr_id); -ALTER TABLE TARGET_SCHEMA.office ADD CONSTRAINT fk_office_delivery_address FOREIGN KEY (delivery_addr_id) REFERENCES TARGET_SCHEMA.address (addr_id); -ALTER TABLE TARGET_SCHEMA.office ADD CONSTRAINT fk_corp_party_delivery_address FOREIGN KEY (delivery_addr_id) REFERENCES TARGET_SCHEMA.address (addr_id); -ALTER TABLE TARGET_SCHEMA.corp_party ADD CONSTRAINT fk_corp_party_mailing_address FOREIGN KEY (mailing_addr_id) REFERENCES TARGET_SCHEMA.address (addr_id); -ALTER TABLE TARGET_SCHEMA.completing_party ADD CONSTRAINT fk_completing_party_address FOREIGN KEY (mailing_addr_id) REFERENCES TARGET_SCHEMA.address (addr_id); -ALTER TABLE TARGET_SCHEMA.notification ADD CONSTRAINT fk_notification_address FOREIGN KEY (mailing_addr_id) REFERENCES TARGET_SCHEMA.address (addr_id); - - -ALTER TABLE TARGET_SCHEMA.corporation ALTER COLUMN send_ar_ind TYPE boolean USING (CASE send_ar_ind WHEN 'true' THEN true WHEN 'false' THEN false ELSE true END); -ALTER TABLE TARGET_SCHEMA.filing ALTER COLUMN arrangement_ind TYPE boolean USING (CASE arrangement_ind WHEN 'true' THEN true WHEN 'false' THEN false ELSE true END); -ALTER TABLE TARGET_SCHEMA.filing ALTER COLUMN court_appr_ind TYPE boolean USING (CASE court_appr_ind WHEN 'true' THEN true WHEN 'false' THEN false ELSE true END); -ALTER TABLE TARGET_SCHEMA.share_series ALTER COLUMN max_share_ind TYPE boolean USING (CASE max_share_ind WHEN 'true' THEN true WHEN 'false' THEN false ELSE true END); -ALTER TABLE TARGET_SCHEMA.share_struct_cls ALTER COLUMN max_share_ind TYPE boolean USING (CASE max_share_ind WHEN 'true' THEN true WHEN 'false' THEN false ELSE true END); -ALTER TABLE TARGET_SCHEMA.share_struct_cls ALTER COLUMN spec_rights_ind TYPE boolean USING (CASE spec_rights_ind WHEN 'true' THEN true WHEN 'false' THEN false ELSE true END); -ALTER TABLE TARGET_SCHEMA.share_struct_cls ALTER COLUMN par_value_ind TYPE boolean USING (CASE par_value_ind WHEN 'true' THEN true WHEN 'false' THEN false ELSE true END); +call TARGET_SCHEMA.colin_fk_restore_all(NULL); \ No newline at end of file diff --git a/jobs/colin-extract-refresh/src/subset/subset_pg_call_address_transpose.sql b/jobs/colin-extract-refresh/src/subset/subset_pg_call_address_transpose.sql new file mode 100644 index 0000000000..203d265608 --- /dev/null +++ b/jobs/colin-extract-refresh/src/subset/subset_pg_call_address_transpose.sql @@ -0,0 +1,11 @@ +-- Invoke the address-transpose orchestrator once at the end of a subset +-- load/refresh while the shared advisory lock is still held. +-- +-- IMPORTANT (DbSchemaCLI compatibility): +-- - Plain statements only; no DO $$ blocks, dollar quoting, or client scripting. +-- - The phase procedures COMMIT internally, so CALL must be top-level. +-- - This terminal result-bearing CALL returns one JSON value with phase counts, their total, and elapsed seconds. + +SET search_path TO TARGET_SCHEMA; + +CALL TARGET_SCHEMA.colin_address_transpose(NULL); diff --git a/jobs/colin-extract-refresh/src/subset/subset_pg_cleanup_boolean_stage.sql b/jobs/colin-extract-refresh/src/subset/subset_pg_cleanup_boolean_stage.sql new file mode 100644 index 0000000000..09320eecdd --- /dev/null +++ b/jobs/colin-extract-refresh/src/subset/subset_pg_cleanup_boolean_stage.sql @@ -0,0 +1,9 @@ +SET search_path TO TARGET_SCHEMA; + +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_corporation_bool_stage; +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_filing_bool_stage; +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_conv_event_bool_stage; +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_corp_involved_amalgamating_bool_stage; +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_corp_restriction_bool_stage; +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_share_struct_cls_bool_stage; +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_share_series_bool_stage; diff --git a/jobs/colin-extract-refresh/src/subset/subset_pg_prepare_boolean_stage.sql b/jobs/colin-extract-refresh/src/subset/subset_pg_prepare_boolean_stage.sql new file mode 100644 index 0000000000..c094e3e6f6 --- /dev/null +++ b/jobs/colin-extract-refresh/src/subset/subset_pg_prepare_boolean_stage.sql @@ -0,0 +1,58 @@ +SET search_path TO TARGET_SCHEMA; + +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_corporation_bool_stage; +CREATE TABLE TARGET_SCHEMA.subset_corporation_bool_stage ( + LIKE TARGET_SCHEMA.corporation INCLUDING DEFAULTS +); +ALTER TABLE TARGET_SCHEMA.subset_corporation_bool_stage + ALTER COLUMN send_ar_ind TYPE varchar(5); + +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_filing_bool_stage; +CREATE TABLE TARGET_SCHEMA.subset_filing_bool_stage ( + LIKE TARGET_SCHEMA.filing INCLUDING DEFAULTS +); +ALTER TABLE TARGET_SCHEMA.subset_filing_bool_stage + ALTER COLUMN arrangement_ind TYPE varchar(5); +ALTER TABLE TARGET_SCHEMA.subset_filing_bool_stage + ALTER COLUMN court_appr_ind TYPE varchar(5); + +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_conv_event_bool_stage; +CREATE TABLE TARGET_SCHEMA.subset_conv_event_bool_stage ( + LIKE TARGET_SCHEMA.conv_event INCLUDING DEFAULTS +); +ALTER TABLE TARGET_SCHEMA.subset_conv_event_bool_stage + ALTER COLUMN report_corp_ind TYPE varchar(5); + +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_corp_involved_amalgamating_bool_stage; +CREATE TABLE TARGET_SCHEMA.subset_corp_involved_amalgamating_bool_stage ( + LIKE TARGET_SCHEMA.corp_involved_amalgamating INCLUDING DEFAULTS +); +ALTER TABLE TARGET_SCHEMA.subset_corp_involved_amalgamating_bool_stage + ALTER COLUMN adopted_corp_ind TYPE varchar(5); + +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_corp_restriction_bool_stage; +CREATE TABLE TARGET_SCHEMA.subset_corp_restriction_bool_stage ( + LIKE TARGET_SCHEMA.corp_restriction INCLUDING DEFAULTS +); +ALTER TABLE TARGET_SCHEMA.subset_corp_restriction_bool_stage + ALTER COLUMN restriction_ind TYPE varchar(5); + +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_share_struct_cls_bool_stage; +CREATE TABLE TARGET_SCHEMA.subset_share_struct_cls_bool_stage ( + LIKE TARGET_SCHEMA.share_struct_cls INCLUDING DEFAULTS +); +ALTER TABLE TARGET_SCHEMA.subset_share_struct_cls_bool_stage + ALTER COLUMN max_share_ind TYPE varchar(5); +ALTER TABLE TARGET_SCHEMA.subset_share_struct_cls_bool_stage + ALTER COLUMN spec_rights_ind TYPE varchar(5); +ALTER TABLE TARGET_SCHEMA.subset_share_struct_cls_bool_stage + ALTER COLUMN par_value_ind TYPE varchar(5); + +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_share_series_bool_stage; +CREATE TABLE TARGET_SCHEMA.subset_share_series_bool_stage ( + LIKE TARGET_SCHEMA.share_series INCLUDING DEFAULTS +); +ALTER TABLE TARGET_SCHEMA.subset_share_series_bool_stage + ALTER COLUMN max_share_ind TYPE varchar(5); +ALTER TABLE TARGET_SCHEMA.subset_share_series_bool_stage + ALTER COLUMN spec_right_ind TYPE varchar(5); \ No newline at end of file diff --git a/jobs/colin-extract-refresh/src/subset/subset_transfer_chunk.sql b/jobs/colin-extract-refresh/src/subset/subset_transfer_chunk.sql index 05866e1cfc..c56a10f2ce 100644 --- a/jobs/colin-extract-refresh/src/subset/subset_transfer_chunk.sql +++ b/jobs/colin-extract-refresh/src/subset/subset_transfer_chunk.sql @@ -33,8 +33,9 @@ -- vset oracle_corp_num_predicate=c.CORP_NUM in ('1111585','1226175'); SET search_path TO TARGET_SCHEMA; +truncate table TARGET_SCHEMA.subset_corporation_bool_stage; -- corporation -transfer TARGET_SCHEMA.corporation from cprd using +transfer TARGET_SCHEMA.subset_corporation_bool_stage from cprd using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -87,6 +88,35 @@ select c.target_corp_num as CORP_NUM, from corporation_cte c left join last_ar la on la.corp_num = c.corp_num; +INSERT INTO TARGET_SCHEMA.corporation( + corp_num, + corp_frozen_type_cd, + corp_type_cd, + corp_password, + recognition_dts, + bn_9, + bn_15, + admin_email, + accession_num, + last_ar_filed_dt, + send_ar_ind, + last_ar_reminder_year +) +select corp_num, + corp_frozen_type_cd, + corp_type_cd, + corp_password, + recognition_dts, + bn_9, + bn_15, + admin_email, + accession_num, + last_ar_filed_dt, + send_ar_ind::boolean, + last_ar_reminder_year +from TARGET_SCHEMA.subset_corporation_bool_stage; + +truncate table TARGET_SCHEMA.subset_corporation_bool_stage; -- event transfer TARGET_SCHEMA.event from cprd using @@ -185,7 +215,9 @@ join corp_op_state cos on cos.state_typ_cd = cs.state_typ_cd; -- filing -transfer TARGET_SCHEMA.filing from cprd using +truncate table TARGET_SCHEMA.subset_filing_bool_stage; + +transfer TARGET_SCHEMA.subset_filing_bool_stage from cprd using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -230,6 +262,36 @@ from corporation_cte c join event e on e.corp_num = c.corp_num join filing f on f.event_id = e.event_id; +insert into TARGET_SCHEMA.filing( + event_id, + filing_type_cd, + effective_dt, + withdrawn_event_id, + ods_type_cd, + nr_num, + court_order_num, + change_dt, + period_end_dt, + arrangement_ind, + auth_sign_dt, + court_appr_ind +) +select event_id, + filing_type_cd, + effective_dt, + withdrawn_event_id, + ods_type_cd, + nr_num, + court_order_num, + change_dt, + period_end_dt, + arrangement_ind::boolean, + auth_sign_dt, + court_appr_ind::boolean +from TARGET_SCHEMA.subset_filing_bool_stage; + +truncate table TARGET_SCHEMA.subset_filing_bool_stage; + -- filing_user transfer TARGET_SCHEMA.filing_user from cprd using @@ -388,7 +450,7 @@ SET province = EXCLUDED.province, addr_line_3 = EXCLUDED.addr_line_3, city = EXCLUDED.city; -TRUNCATE TABLE subset_address_stage; +TRUNCATE TABLE TARGET_SCHEMA.subset_address_stage; -- office @@ -727,7 +789,9 @@ join CONT_OUT co on co.corp_num = c.corp_num; -- conv_event -transfer TARGET_SCHEMA.conv_event from cprd using +truncate table TARGET_SCHEMA.subset_conv_event_bool_stage; + +transfer TARGET_SCHEMA.subset_conv_event_bool_stage from cprd using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -764,6 +828,26 @@ from corporation_cte c join event e on e.corp_num = c.corp_num join CONV_EVENT ce on ce.event_id = e.event_id; +insert into TARGET_SCHEMA.conv_event ( + event_id, + effective_dt, + report_corp_ind, + activity_user_id, + activity_dt, + annual_file_dt, + accession_num, + remarks +) +select event_id, + effective_dt, + report_corp_ind::boolean, + activity_user_id, + activity_dt, + annual_file_dt, + accession_num, + remarks +from TARGET_SCHEMA.subset_conv_event_bool_stage; +truncate table TARGET_SCHEMA.subset_conv_event_bool_stage; -- conv_ledger transfer TARGET_SCHEMA.conv_ledger from cprd using @@ -797,7 +881,8 @@ join CONV_LEDGER cl on cl.event_id = e.event_id; -- corp_involved - amalgamaTING_businesses -transfer TARGET_SCHEMA.corp_involved_amalgamating from cprd using +truncate table TARGET_SCHEMA.subset_corp_involved_amalgamating_bool_stage; +transfer TARGET_SCHEMA.subset_corp_involved_amalgamating_bool_stage from cprd using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -863,7 +948,29 @@ join CORP_INVOLVED ci on ci.event_id = e.event_id join corporation c2 on c2.corp_num = ci.corp_num where f.filing_typ_cd in ('AMALH', 'AMALV', 'AMALR', 'AMLHU', 'AMLVU', 'AMLRU', 'AMLHC', 'AMLVC', 'AMLRC'); - +insert into TARGET_SCHEMA.corp_involved_amalgamating ( + event_id, + ted_corp_num, + ting_corp_num, + corp_involve_id, + can_jur_typ_cd, + adopted_corp_ind, + home_juri_num, + othr_juri_desc, + foreign_nme +) +select event_id, + ted_corp_num, + ting_corp_num, + corp_involve_id, + can_jur_typ_cd, + adopted_corp_ind::boolean, + home_juri_num, + othr_juri_desc, + foreign_nme +from TARGET_SCHEMA.subset_corp_involved_amalgamating_bool_stage; + +truncate table TARGET_SCHEMA.subset_corp_involved_amalgamating_bool_stage; -- corp_involved - continue_in_historical_xpro transfer TARGET_SCHEMA.corp_involved_cont_in from cprd using with corp_list as ( @@ -896,7 +1003,8 @@ where f.filing_typ_cd in ('CONTI', 'CONTU', 'CONTC') -- corp_restriction -transfer TARGET_SCHEMA.corp_restriction from cprd using +truncate table TARGET_SCHEMA.subset_corp_restriction_bool_stage; +transfer TARGET_SCHEMA.subset_corp_restriction_bool_stage from cprd using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -928,6 +1036,18 @@ select c.target_corp_num as CORP_NUM, from corporation_cte c join CORP_RESTRICTION cr on cr.corp_num = c.corp_num; +insert into TARGET_SCHEMA.corp_restriction ( + corp_num, + restriction_ind, + start_event_id, + end_event_id +) +select corp_num, + restriction_ind::boolean, + start_event_id, + end_event_id +from TARGET_SCHEMA.subset_corp_restriction_bool_stage; +truncate table TARGET_SCHEMA.subset_corp_restriction_bool_stage; -- correction transfer TARGET_SCHEMA.correction from cprd using @@ -1055,7 +1175,8 @@ join SHARE_STRUCT ss on ss.corp_num = c.corp_num; -- share_struct_cls -transfer TARGET_SCHEMA.share_struct_cls from cprd using +truncate table TARGET_SCHEMA.subset_share_struct_cls_bool_stage; +transfer TARGET_SCHEMA.subset_share_struct_cls_bool_stage from cprd using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -1102,9 +1223,36 @@ select c.target_corp_num as CORP_NUM, from corporation_cte c join SHARE_STRUCT_CLS ssc on ssc.corp_num = c.corp_num; +insert into TARGET_SCHEMA.share_struct_cls( + corp_num, + share_class_id, + class_nme, + currency_typ_cd, + max_share_ind, + share_quantity, + spec_rights_ind, + par_value_ind, + par_value_amt, + other_currency, + start_event_id +) +select corp_num, + share_class_id, + class_nme, + currency_typ_cd, + max_share_ind::boolean, + share_quantity, + spec_rights_ind::boolean, + par_value_ind::boolean, + par_value_amt, + other_currency, + start_event_id +from TARGET_SCHEMA.subset_share_struct_cls_bool_stage; +truncate table TARGET_SCHEMA.subset_share_struct_cls_bool_stage; -- share_series -transfer TARGET_SCHEMA.share_series from cprd using +truncate table TARGET_SCHEMA.subset_share_series_bool_stage; +transfer TARGET_SCHEMA.subset_share_series_bool_stage from cprd using with corp_list as ( select /*+ materialize */ c.corp_num from corporation c @@ -1144,6 +1292,26 @@ select c.target_corp_num as CORP_NUM, from corporation_cte c join SHARE_SERIES ss on ss.corp_num = c.corp_num; +insert into TARGET_SCHEMA.share_series( + corp_num, + share_class_id, + series_id, + max_share_ind, + share_quantity, + spec_right_ind, + series_nme, + start_event_id +) +select corp_num, + share_class_id, + series_id, + max_share_ind::boolean, + share_quantity, + spec_right_ind::boolean, + series_nme, + start_event_id +from TARGET_SCHEMA.subset_share_series_bool_stage; +truncate table TARGET_SCHEMA.subset_share_series_bool_stage; -- notification transfer TARGET_SCHEMA.notification from cprd using diff --git a/jobs/colin-extract-refresh/src/subset/subset_validate_staged_transfers.sql b/jobs/colin-extract-refresh/src/subset/subset_validate_staged_transfers.sql new file mode 100644 index 0000000000..4c4b607b9a --- /dev/null +++ b/jobs/colin-extract-refresh/src/subset/subset_validate_staged_transfers.sql @@ -0,0 +1,182 @@ +SET search_path TO TARGET_SCHEMA; + +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_validate_oracle_counts; +CREATE TABLE TARGET_SCHEMA.subset_validate_oracle_counts ( + corp_num varchar(20) NOT NULL, + table_name varchar(20) NOT NULL, + n bigint NOT NULL, + PRIMARY KEY (corp_num, table_name) +); + +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_validate_transfer_counts; +CREATE TABLE TARGET_SCHEMA.subset_validate_transfer_counts ( + corp_num varchar(20) NOT NULL, + table_name varchar(20) NOT NULL, + oracle_count bigint NOT NULL, + pg_count bigint NOT NULL, + delta bigint NOT NULL, + status varchar(16) NOT NULL, + PRIMARY KEY (corp_num, table_name) +); + +-- pull per corp oracle count for refreshed candidate +learn schema TARGET_SCHEMA; + +transfer TARGET_SCHEMA.subset_validate_oracle_counts from cprd using +WITH corp_list AS ( + SELECT c.corp_num + FROM corporation c + WHERE &oracle_corp_num_predicate + AND &oracle_corp_type_predicate + AND c.CORP_NUM NOT IN ('0460007', '1255957', '1186381') +), +corps AS ( + SELECT c.corp_num AS oracle_corp_num, + CASE + WHEN c.CORP_TYP_CD IN ('BC', 'ULC', 'CC') THEN 'BC' || c.CORP_NUM + ELSE c.CORP_NUM + END AS corp_num + FROM corporation c + JOIN corp_list cl ON cl.corp_num = c.corp_num +) +SELECT corp_num, table_name, n FROM ( + SELECT corp_num, 'corporation' AS table_name, COUNT(*) AS n + FROM corps + GROUP BY corp_num + + UNION ALL + SELECT c.corp_num, 'event', COUNT(*) + FROM corps c + JOIN event e ON e.corp_num = c.oracle_corp_num + WHERE e.event_typ_cd NOT IN ('BNUPD', 'ADDLEDGR') + GROUP BY c.corp_num + + UNION ALL + SELECT c.corp_num, 'filing', COUNT(*) + FROM corps c + JOIN event e ON e.corp_num = c.oracle_corp_num + JOIN filing f ON f.event_id = e.event_id + GROUP BY c.corp_num + + UNION ALL + SELECT c.corp_num, 'conv_event', COUNT(*) + FROM corps c + JOIN event e ON e.corp_num = c.oracle_corp_num + JOIN CONV_EVENT ce ON ce.event_id = e.event_id + GROUP BY c.corp_num + + UNION ALL + SELECT c.corp_num, 'corp_restriction', COUNT(*) + FROM corps c + JOIN CORP_RESTRICTION cr ON cr.corp_num = c.oracle_corp_num + GROUP BY c.corp_num + + UNION ALL + SELECT c.corp_num, 'share_struct_cls', COUNT(*) + FROM corps c + JOIN SHARE_STRUCT_CLS s ON s.corp_num = c.oracle_corp_num + GROUP BY c.corp_num + + UNION ALL + SELECT c.corp_num, 'share_series', COUNT(*) + FROM corps c + JOIN SHARE_SERIES s ON s.corp_num = c.oracle_corp_num + GROUP BY c.corp_num +) counts; + +-- comparison against extract for candidates + +INSERT INTO TARGET_SCHEMA.subset_validate_transfer_counts + (corp_num, table_name, oracle_count, pg_count, delta, status) +WITH corps AS ( + SELECT * FROM unnest(ARRAY[&corp_ids_in]) AS t(corp_num) +), +tables AS ( + SELECT * FROM ( + VALUES + ('corporation'), ('event'), ('filing'), ('conv_event'), ('corp_restriction'),('share_struct_cls'), ('share_series') + ) AS t(table_name) +), +pg_count AS ( + SELECT t.corp_num, 'corporation'::varchar AS table_name, COUNT(*)::bigint AS n + FROM TARGET_SCHEMA.corporation x + JOIN corps t ON t.corp_num = x.corp_num + GROUP BY t.corp_num + + UNION ALL + SELECT t.corp_num, 'event', COUNT(*) + FROM TARGET_SCHEMA.event x + JOIN corps t ON t.corp_num = x.corp_num + WHERE x.event_type_cd NOT IN ('BNUPD', 'ADDLEDGR') + GROUP BY t.corp_num + + UNION ALL + SELECT t.corp_num, 'filing', COUNT(*) + FROM TARGET_SCHEMA.filing x + JOIN TARGET_SCHEMA.event e ON e.event_id = x.event_id + JOIN corps t ON t.corp_num = e.corp_num + GROUP BY t.corp_num + + UNION ALL + SELECT t.corp_num, 'conv_event', COUNT(*) + FROM TARGET_SCHEMA.conv_event x + JOIN TARGET_SCHEMA.event e ON e.event_id = x.event_id + JOIN corps t ON t.corp_num = e.corp_num + GROUP BY t.corp_num + + UNION ALL + SELECT t.corp_num, 'corp_restriction', COUNT(*) + FROM TARGET_SCHEMA.corp_restriction x + JOIN corps t ON t.corp_num = x.corp_num + GROUP BY t.corp_num + + UNION ALL + SELECT t.corp_num, 'share_struct_cls', COUNT(*) + FROM TARGET_SCHEMA.share_struct_cls x + JOIN corps t ON t.corp_num = x.corp_num + GROUP BY t.corp_num + + UNION ALL + SELECT t.corp_num, 'share_series', COUNT(*) + FROM TARGET_SCHEMA.share_series x + JOIN corps t ON t.corp_num = x.corp_num + GROUP BY t.corp_num +), +compared AS ( + SELECT + c.corp_num, + t.table_name, + COALESCE(o.n, 0)::bigint AS oracle_count, + COALESCE(p.n, 0)::bigint AS pg_count, + (COALESCE(o.n, 0) - COALESCE(p.n, 0))::bigint AS delta, + CASE + WHEN COALESCE(o.n, 0) = COALESCE(p.n, 0) THEN 'OK' + ELSE 'MISMATCH' + END AS status + FROM corps c + CROSS JOIN tables t + LEFT JOIN TARGET_SCHEMA.subset_validate_oracle_counts o + ON o.corp_num = c.corp_num AND o.table_name = t.table_name + LEFT JOIN pg_count p + ON p.corp_num = c.corp_num AND p.table_name = t.table_name +) +SELECT corp_num, table_name, oracle_count, pg_count, delta, status +FROM compared; + +-- summary +SELECT + table_name, SUM(oracle_count) AS oracle_total, + SUM(pg_count) AS pg_total, + SUM(delta) AS delta_total, + SUM(CASE WHEN status = 'MISMATCH' THEN 1 ELSE 0 END) AS bad_corps +FROM TARGET_SCHEMA.subset_validate_transfer_counts +GROUP BY table_name +ORDER BY bad_corps DESC, table_name; + +-- -- discrepency +-- SELECT corp_num, table_name, oracle_count, pg_count, delta, status +-- FROM TARGET_SCHEMA.subset_validate_transfer_counts +-- WHERE status = 'MISMATCH' +-- ORDER BY table_name, corp_num; + +DROP TABLE IF EXISTS TARGET_SCHEMA.subset_validate_oracle_counts; \ No newline at end of file diff --git a/jobs/colin-extract-refresh/src/test_connectivity.py b/jobs/colin-extract-refresh/src/test_connectivity.py index 452e9d9577..a642e52b3f 100644 --- a/jobs/colin-extract-refresh/src/test_connectivity.py +++ b/jobs/colin-extract-refresh/src/test_connectivity.py @@ -1,12 +1,10 @@ import sys from checks.check_business import run_check as run_business_check -from checks.check_colin import run_check as run_colin_check def main() -> int: print("== running test_connectivity.py ==") run_business_check() - # run_colin_check() print("== done test_connectivity.py ==") return 0 diff --git a/legal-api/poetry.lock b/legal-api/poetry.lock index eb11c9ec9c..b4da2973a6 100644 --- a/legal-api/poetry.lock +++ b/legal-api/poetry.lock @@ -316,7 +316,7 @@ files = [ [[package]] name = "business-model" -version = "3.3.32" +version = "3.4.11" description = "" optional = false python-versions = ">=3.13,<3.14" @@ -336,14 +336,14 @@ pg8000 = ">=1.31.2,<2.0.0" pycountry = ">=24.6.1,<25.0.0" pydantic = ">=2.10.6,<3.0.0" pytz = ">=2025.1,<2026.0" -registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.73"} +registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.82"} sql-versioning = {git = "https://github.com/bcgov/lear.git", rev = "main", subdirectory = "python/common/sql-versioning-alt"} [package.source] type = "git" url = "https://github.com/bcgov/lear.git" reference = "main" -resolved_reference = "295ecb7a24620142f1a18af02fb191df4cafcc85" +resolved_reference = "71fbd531a0a89db74d9182922eed949bb8104b0b" subdirectory = "python/common/business-registry-model" [[package]] @@ -2265,22 +2265,6 @@ files = [ {file = "markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698"}, ] -[[package]] -name = "minio" -version = "7.0.2" -description = "MinIO Python SDK for Amazon S3 Compatible Cloud Storage" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "minio-7.0.2-py3-none-any.whl", hash = "sha256:0e69575a0b1b64bce9f158b52b5914953f902a21032966d070ab0d14ceef8962"}, - {file = "minio-7.0.2.tar.gz", hash = "sha256:f2f6022cfe4694d946972efef2a752f87d08cc030940faa50a640088772953c8"}, -] - -[package.dependencies] -certifi = "*" -urllib3 = "*" - [[package]] name = "multidict" version = "6.7.1" @@ -3352,7 +3336,7 @@ rpds-py = ">=0.7.0" [[package]] name = "registry_schemas" -version = "2.18.73" +version = "2.18.82" description = "A short description of the project" optional = false python-versions = ">=3.6" @@ -3370,8 +3354,8 @@ strict-rfc3339 = "*" [package.source] type = "git" url = "https://github.com/bcgov/business-schemas.git" -reference = "2.18.73" -resolved_reference = "e6999a46b6e38c52cc79f0e5a3c9fcbd0c473434" +reference = "2.18.82" +resolved_reference = "7dd37ea586adceff8f5b0d775a9c27eabd2d5135" [[package]] name = "reportlab" @@ -4436,4 +4420,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.13,<3.14" -content-hash = "38eb06e163c276a193e17597b91b0b3b3a58f445a1d060dfda9f4d68d813871e" +content-hash = "ae5d62d00a5b3878147fc84676d7da2a7a5399bc4e2d03bdbdedca7bd9eb32bb" diff --git a/legal-api/pyproject.toml b/legal-api/pyproject.toml index 4a25cfba36..571d525a7e 100644 --- a/legal-api/pyproject.toml +++ b/legal-api/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "legal-api" -version = "3.1.0" +version = "3.1.18" description = "" authors = [ {name = "thor",email = "1042854+thorwolpert@users.noreply.github.com"} @@ -25,7 +25,6 @@ dependencies = [ # FUTURE: look at removing "strict-rfc3339 (==0.7)", # FUTURE: look at removing restriction - "minio (==7.0.2)", "pypdf (>=6.12.1)", "reportlab (>=4.5.0)", # FUTURE: look at removing restriction diff --git a/legal-api/report-templates/certificateOfAmalgamation.html b/legal-api/report-templates/certificateOfAmalgamation.html index a274181194..7584318bb3 100644 --- a/legal-api/report-templates/certificateOfAmalgamation.html +++ b/legal-api/report-templates/certificateOfAmalgamation.html @@ -6,15 +6,24 @@ Certificate of Amalgamation - [[common/certificateStyle.html]] + [[amalgamation/certificateStyle.html]] {% if enable_sandbox %} [[common/certificateWatermark.html]] {% endif %} +
+ Incorporation Number: {{ business.identifier }} +
+
+ [[common/certificateSeal.html]] +
+
+
ELECTRONIC CERTIFICATE
+
+
-
Incorporation Number: {{ business.identifier }}
[[common/correctedOnCertificate.html]]
@@ -30,23 +39,40 @@
I hereby certify that the following corporations were amalgamated under the name
{{ business.legalName }}
-
on {{ effective_date_time }}
-
- {% for entity in amalgamatingBusinesses %} - {% if entity != amalgamatingBusinesses[0] %} - , and - {% endif %} - {{ entity.legalName }} ({{ entity.identifier }}) - {% endfor %} -
-
-
-
Issued under my hand at Victoria, British Columbia
-
on {{ business.formatted_founding_date }}
+
on {{ effective_date_time }}
+
+ {% for entity in amalgamatingBusinesses %} +
+ {% if entity != amalgamatingBusinesses[0] %} + , and + {% endif %} + + {% if entity.isBcCompany %} + {{ entity.legalName }} + , incorporation number {{ entity.identifier }} + + {% elif entity.isExtraprovincial %} + {{ entity.legalName }} + , a foreign corporation incorporated under the laws of + {{ entity.jurisdiction }} + + and registered as an extraprovincial company under the laws of + British Columbia with certificate number {{ entity.identifier }} + + + {% else %} + {{ entity.legalName }} + + , a foreign corporation incorporated under the laws of + + {{ entity.jurisdiction }} + {% endif %} +
+ {% endfor %}
+ [[amalgamation/certificateFooter.html]] - [[common/certificateFooter.html]] diff --git a/legal-api/report-templates/consentContinuationOut.html b/legal-api/report-templates/consentContinuationOut.html new file mode 100644 index 0000000000..6d574963c8 --- /dev/null +++ b/legal-api/report-templates/consentContinuationOut.html @@ -0,0 +1,46 @@ +[[macros.html]] + + + + Continue Out Application + + + [[common/style.html]] + {% if enable_sandbox %} + [[common/watermark.html]] + {% endif %} + + +
+
+ + + + + +
+ +
+ + [[common/businessDetails.html]] + +
+ CONTINUE OUT APPLICATION +
+
+
+
New jurisdiction
+
{{ jurisdiction }}
+
+
+ + diff --git a/legal-api/report-templates/template-parts/alteration-notice/statement.html b/legal-api/report-templates/template-parts/alteration-notice/statement.html index a20b5757d8..d3664b4ff1 100644 --- a/legal-api/report-templates/template-parts/alteration-notice/statement.html +++ b/legal-api/report-templates/template-parts/alteration-notice/statement.html @@ -16,7 +16,7 @@
BC Community Contribution Company Statement
This company is a community contribution company and, as such, has purposes beneficial to society. - This company is restricted, in accordance with Part 2.2 of the BCA, in its ability to pay dividends and to + This company is restricted, in accordance with Part 2.2 of the Business Corporations Act, in its ability to pay dividends and to distribute its assets on dissolution or otherwise.
diff --git a/legal-api/report-templates/template-parts/amalgamation/certificateFooter.html b/legal-api/report-templates/template-parts/amalgamation/certificateFooter.html new file mode 100644 index 0000000000..c29ec6f674 --- /dev/null +++ b/legal-api/report-templates/template-parts/amalgamation/certificateFooter.html @@ -0,0 +1,21 @@ +
+
+
+ +
+
Issued under my hand at
+
Victoria, British Columbia on
+
{{ business.formatted_founding_date }}
+
+ +
[[common/certificateRegistrarSignature.html]]
+
{{ registrarInfo.name }}
+
{{ registrarInfo.title }}
+
+
PROVINCE OF BRITISH COLUMBIA
+
CANADA
+
+ +
+
+
diff --git a/legal-api/report-templates/template-parts/amalgamation/certificateStyle.html b/legal-api/report-templates/template-parts/amalgamation/certificateStyle.html new file mode 100644 index 0000000000..07cdd4c562 --- /dev/null +++ b/legal-api/report-templates/template-parts/amalgamation/certificateStyle.html @@ -0,0 +1,308 @@ + diff --git a/legal-api/report-templates/template-parts/common/businessDetails.html b/legal-api/report-templates/template-parts/common/businessDetails.html index 0f5fd88d51..0c4febe248 100644 --- a/legal-api/report-templates/template-parts/common/businessDetails.html +++ b/legal-api/report-templates/template-parts/common/businessDetails.html @@ -120,6 +120,17 @@
{{recognition_date_time}}
{{filing_date_time}}
+ {% elif header.reportType == 'consentContinuationOut' %} + +
Incorporation Number:
+
Recognition Date and Time:
+
Filed Date and Time:
+ + +
{{business.identifier}}
+
{{recognition_date_time}}
+
{{filing_date_time}}
+ {% elif header.name in ['continuationIn', 'incorporationApplication', 'restoration'] %}
Incorporation Number:
diff --git a/legal-api/report-templates/template-parts/common/statement.html b/legal-api/report-templates/template-parts/common/statement.html index 3be1344ce3..cbba32d7de 100644 --- a/legal-api/report-templates/template-parts/common/statement.html +++ b/legal-api/report-templates/template-parts/common/statement.html @@ -16,7 +16,7 @@
BC Community Contribution Company Statement
This company is a community contribution company and, as such, has purposes beneficial to society. - This company is restricted, in accordance with Part 2.2 of the BCA, in its ability to pay dividends and to + This company is restricted, in accordance with Part 2.2 of the Business Corporations Act, in its ability to pay dividends and to distribute its assets on dissolution or otherwise.
diff --git a/legal-api/src/legal_api/config.py b/legal-api/src/legal_api/config.py index 69523f270d..df33896478 100644 --- a/legal-api/src/legal_api/config.py +++ b/legal-api/src/legal_api/config.py @@ -57,6 +57,10 @@ class _Config: # pylint: disable=too-few-public-methods REPORT_API_GOTENBERG_URL = os.getenv("REPORT_API_GOTENBERG_URL", "https://") COLIN_URL = f"{os.getenv('COLIN_API_URL', '')}{os.getenv('COLIN_API_VERSION', '')}" + try: + COLIN_TIMEOUT = int(os.getenv("COLIN_TIMEOUT", "20")) + except (TypeError, ValueError): + COLIN_TIMEOUT = 20 LEGAL_API_BASE_URL = f"{BUSINESS_API_GW_URL + BUSINESS_API_VERSION_2}/businesses" @@ -137,12 +141,6 @@ class _Config: # pylint: disable=too-few-public-methods # legislative timezone for future effective dating LEGISLATIVE_TIMEZONE = os.getenv("LEGISLATIVE_TIMEZONE", "America/Vancouver") - # Minio configuration values - MINIO_ENDPOINT = os.getenv("MINIO_ENDPOINT") - MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY") - MINIO_ACCESS_SECRET = os.getenv("MINIO_ACCESS_SECRET") - MINIO_BUCKET_BUSINESSES = os.getenv("MINIO_BUCKET_BUSINESSES", "businesses") - MINIO_SECURE = True # determines which year of NAICS data will be used to drive NAICS search NAICS_YEAR = int(os.getenv("NAICS_YEAR", "2022")) @@ -303,12 +301,6 @@ class TestConfig(_Config): # pylint: disable=too-few-public-methods 4H8UZcVFN95vEKxJiLRjAmj6g273pu9kK4ymXNEjWWJn -----END RSA PRIVATE KEY-----""" - # Minio variables - MINIO_ENDPOINT = "http://dummy-minio-url" - MINIO_ACCESS_KEY = "minio" - MINIO_ACCESS_SECRET = "minio123" - MINIO_BUCKET_BUSINESSES = "businesses" - MINIO_SECURE = False # determines which year of NAICS data will be used to drive NAICS search; # matches the test seed data loaded by business_model_migrations @@ -321,6 +313,8 @@ class TestConfig(_Config): # pylint: disable=too-few-public-methods PAYMENT_SVC_URL = "https://PAY_SVC_URL/api/v1/payment-requests" AUTH_SVC_URL = "https://AUTH_SVC_URL" ACCOUNT_SVC_AUTH_URL = "https://ACCOUNT_SVC_AUTH_URL" + COLIN_URL = "https://COLIN_API_URL/api/v1" + ACCOUNT_SVC_CLIENT_SECRET = None BUSINESS_SCHEMA_ID = os.getenv("BUSINESS_SCHEMA_ID", "TEST_BUSINESS_SCHEMA_ID") BUSINESS_CRED_DEF_ID = os.getenv("BUSINESS_CRED_DEF_ID", "TEST_BUSINESS_SCHEMA_ID") diff --git a/legal-api/src/legal_api/core/filing.py b/legal-api/src/legal_api/core/filing.py index 8eb407a144..31bc80c5f8 100644 --- a/legal-api/src/legal_api/core/filing.py +++ b/legal-api/src/legal_api/core/filing.py @@ -23,12 +23,12 @@ from flask import current_app, url_for from sqlalchemy import desc -from business_model.models import Business, Document, DocumentType, UserRoles +from business_model.models import Business, UserRoles from business_model.models import Filing as FilingStorage from legal_api.core.meta import FilingMeta from legal_api.reports.document_service import DocumentService from legal_api.services import VersionedBusinessDetailsService -from legal_api.services.authz import has_roles, is_competent_authority +from legal_api.services.authz import has_any_roles, is_competent_authority from .constants import REDACTED_STAFF_SUBMITTER @@ -368,7 +368,7 @@ def redact_submitter(submitter_roles: list, jwt: JwtManager) -> bool | None: with suppress(KeyError, TypeError): if (UserRoles.staff in submitter_roles or UserRoles.system in submitter_roles) \ - and not has_roles(jwt, [UserRoles.staff, ]): + and not has_any_roles(jwt, [UserRoles.staff, ]): return True return False @@ -439,13 +439,7 @@ def ledger(business_id: int, # noqa: PLR0913 ledger_filing["correctionLink"] = f"{base_url}/{business.identifier}/filings/{filing.parent_filing.id}" ledger_filing["correctionFilingStatus"] = filing.parent_filing.status - # add the collected meta_data - if filing.meta_data: - ledger_filing["data"] = filing.meta_data - - # orders - if filing.court_order_file_number or filing.order_details: - Filing._add_ledger_order(filing, ledger_filing) + Filing._add_meta_data(filing, ledger_filing) core_filing: Filing = Filing() # Filing.get_document_list needs a core Filing. core_filing._storage = filing # pylint: disable=protected-access @@ -474,92 +468,74 @@ def common_ledger_items(business_identifier: str, filing_storage: FilingStorage) } @staticmethod - def _add_ledger_order(filing: FilingStorage, ledger_filing: dict) -> dict: - court_order_data = {"fileNumber": filing.court_order_file_number} - if filing.court_order_date: - court_order_data["orderDate"] = filing.court_order_date - if filing.court_order_effect_of_order: - court_order_data["effectOfOrder"] = filing.court_order_effect_of_order - if filing.order_details: - court_order_data["orderDetails"] = filing.order_details - - if not ledger_filing.get("data"): - ledger_filing["data"] = {} - ledger_filing["data"]["order"] = court_order_data + def _add_meta_data(filing: FilingStorage, ledger_filing: dict): + if not (meta_data := copy.deepcopy(filing.meta_data)): + meta_data = {} - @staticmethod - def get_document_list(business, # noqa: PLR0912, PLR0915 NOSONAR(S3776) - filing, - jwt: JwtManager) -> dict | None: - """Return a list of documents for a particular filing.""" - no_output_filings = [ - Filing.FilingTypes.CONVERSION.value, - Filing.FilingTypes.PUTBACKOFF.value, - Filing.FilingTypes.PUTBACKON.value, - Filing.FilingTypes.REGISTRARSNOTATION.value, - Filing.FilingTypes.REGISTRARSORDER.value, - ] + court_order_data = {} + if court_order := meta_data.get("courtOrder"): + court_order.pop("files", None) # remove files if exists from court order + court_order_data = court_order + meta_data.pop("courtOrder") # remove courtOrder as it will be in ledger_filing["data"]["order"] - if not filing \ - or filing.status in ( - Filing.Status.PAPER_ONLY, - Filing.Status.DRAFT, - Filing.Status.PENDING, - Filing.Status.AWAITING_REVIEW, - Filing.Status.CHANGE_REQUESTED, - Filing.Status.APPROVED, - Filing.Status.REJECTED, - ): - return None + # None of the filings have both court_order.order_details and filing.details and UI depends on the orderDetails + # Future: would be ideal to sepatate the two and have both in the ledger_filing if they exist + if filing.details: + court_order_data["orderDetails"] = filing.details - base_url = current_app.config.get("BUSINESS_API_GW_URL") + if court_order_data: + meta_data["order"] = court_order_data + + if meta_data: + ledger_filing["data"] = meta_data + + @staticmethod + def _is_invalid_status_for_document_list(filing) -> bool: + return not filing or filing.status in ( + Filing.Status.PAPER_ONLY, + Filing.Status.DRAFT, + Filing.Status.PENDING, + Filing.Status.AWAITING_REVIEW, + Filing.Status.CHANGE_REQUESTED, + Filing.Status.APPROVED, + Filing.Status.REJECTED + ) + + @staticmethod + def _get_identifier_for_doc_list(business, filing) -> str: identifier = business.identifier if business else filing.storage.temp_reg if not identifier and filing.storage.withdrawn_filing_id: withdrawn_filing = Filing.find_by_id(filing.storage.withdrawn_filing_id) identifier = withdrawn_filing.storage.temp_reg + return identifier - doc_url = url_for("API2.get_documents", identifier=identifier, filing_id=filing.id, legal_filing_name=None) - documents = {"documents": {}} - # for paper_only filings return and empty documents list - if filing.storage and filing.storage.paper_only: - return documents - - if filing.storage and filing.storage.filing_type in no_output_filings: - return documents - - user_is_ca = is_competent_authority(jwt) - - # return a receipt for filings completed in our system (but not for ca users - # see https://github.com/bcgov/entity/issues/21881 - if filing.storage and filing.storage.payment_completion_date: - if filing.filing_type == "courtOrder" and \ - (filing.storage.documents.filter( - Document.type == DocumentType.COURT_ORDER.value).one_or_none()): - documents["documents"]["uploadedCourtOrder"] = f"{base_url}{doc_url}/uploadedCourtOrder" - documents["documents"]["receipt"] = f"{base_url}{doc_url}/receipt" - elif filing.storage and filing.storage.source == filing.storage.Source.COLIN.value: + @staticmethod + def _add_receipt(documents, filing, base_url, doc_url): + if ( + filing.storage.payment_completion_date or + filing.storage.source == filing.storage.Source.COLIN.value + ): documents["documents"]["receipt"] = f"{base_url}{doc_url}/receipt" - filing_sub_type = filing.storage.filing_sub_type - - receipt_only_filings = [ - Filing.FilingTypes.CHANGEOFRECEIVERS.value, - ] - + @staticmethod + def _has_no_outputs_except_receipt(filing) -> bool: + receipt_only_filings = [Filing.FilingTypes.CHANGEOFRECEIVERS.value] receipt_only_sub_filings = [ (Filing.FilingTypes.CHANGEOFLIQUIDATORS.value, "liquidationReport"), + (Filing.FilingTypes.DISSOLUTION.value, "delay") ] - no_outputs_except_receipt = ( - filing.filing_type in receipt_only_filings - or (filing.filing_type, filing_sub_type) in receipt_only_sub_filings + return ( + filing.filing_type in receipt_only_filings or + (filing.filing_type, filing.storage.filing_sub_type) in receipt_only_sub_filings ) - no_legal_filings_in_paid_withdrawn_status = no_outputs_except_receipt or filing.filing_type in [ + @staticmethod + def _has_no_legal_filings_in_paid_or_withdrawn_status(filing, business) -> bool: + no_legal_filings = [ Filing.FilingTypes.AMALGAMATIONOUT.value, Filing.FilingTypes.REGISTRATION.value, Filing.FilingTypes.CONSENTAMALGAMATIONOUT.value, - Filing.FilingTypes.CONSENTCONTINUATIONOUT.value, Filing.FilingTypes.COURTORDER.value, Filing.FilingTypes.CONTINUATIONOUT.value, Filing.FilingTypes.AGMEXTENSION.value, @@ -568,90 +544,146 @@ def get_document_list(business, # noqa: PLR0912, PLR0915 NOSONAR(S3776) Filing.FilingTypes.CHANGEOFLIQUIDATORS.value, Filing.FilingTypes.CHANGEOFOFFICERS.value ] - no_outputs_except_receipt_dissolution = filing.storage.filing_sub_type == "delay" - no_legal_filings_in_paid_withdrawn_status_dissolution = ( - filing.filing_type == Filing.FilingTypes.DISSOLUTION.value - and (business.legal_type in [Business.LegalTypes.SOLE_PROP.value, - Business.LegalTypes.PARTNERSHIP.value] - or no_outputs_except_receipt_dissolution + return ( + filing.filing_type in no_legal_filings or + ( + filing.filing_type == Filing.FilingTypes.DISSOLUTION.value and + business.legal_type in ( + Business.LegalTypes.SOLE_PROP.value, + Business.LegalTypes.PARTNERSHIP.value + ) ) ) - if ((filing.status in (Filing.Status.PAID, Filing.Status.WITHDRAWN) or - (filing.status == Filing.Status.COMPLETED and - filing.filing_type == Filing.FilingTypes.NOTICEOFWITHDRAWAL.value)) - and not (no_legal_filings_in_paid_withdrawn_status - or no_legal_filings_in_paid_withdrawn_status_dissolution) + + @staticmethod + def _populate_completed_filing_documents( # noqa: PLR0913 + documents, + filing, + business, + jwt, + base_url, + doc_url, + legal_filings + ): + legal_filings_copy = copy.deepcopy(legal_filings) + if ( + filing.filing_type == Filing.FilingTypes.SPECIALRESOLUTION.value and + business.legal_type == Business.LegalTypes.COOP.value ): - documents["documents"]["legalFilings"] = \ - [{filing.filing_type: f"{base_url}{doc_url}/{filing.filing_type}"}, ] + documents["documents"]["specialResolutionApplication"] = f"{base_url}{doc_url}/specialResolutionApplication" + if Filing.FilingTypes.CHANGEOFNAME.value in legal_filings: + legal_filings_copy.remove(Filing.FilingTypes.CHANGEOFNAME.value) + if Filing.FilingTypes.ALTERATION.value in legal_filings: + legal_filings_copy.remove(Filing.FilingTypes.ALTERATION.value) + + no_legal_filings = [ + Filing.FilingTypes.AMALGAMATIONOUT.value, + Filing.FilingTypes.CONSENTAMALGAMATIONOUT.value, + Filing.FilingTypes.CONTINUATIONOUT.value, + Filing.FilingTypes.COURTORDER.value, + Filing.FilingTypes.AGMEXTENSION.value, + Filing.FilingTypes.AGMLOCATIONCHANGE.value, + Filing.FilingTypes.TRANSPARENCY_REGISTER.value, + Filing.FilingTypes.CHANGEOFOFFICERS.value + ] + if filing.filing_type not in no_legal_filings: + documents["documents"]["legalFilings"] = [ + {doc: f"{base_url}{doc_url}/{doc}"} + for doc in legal_filings_copy + ] + + if ( + filing.storage.transaction_id and + (business_rev := VersionedBusinessDetailsService.get_business_revision_obj(filing.storage, business.id)) + ): + business = business_rev + + adds = [FilingMeta.get_all_outputs(business.legal_type, doc) for doc in legal_filings] + additional = {item for sublist in adds for item in sublist} + FilingMeta.alter_outputs(filing.storage, business, additional) + for doc in additional: + documents["documents"][doc] = f"{base_url}{doc_url}/{doc}" + + is_staff_or_system = has_any_roles(jwt, [UserRoles.staff, UserRoles.system]) + static_invisible = ( + filing.storage.filing_type == Filing.FilingTypes.CONTINUATIONIN.value and + not is_staff_or_system + ) + if ( + not static_invisible and + (static_docs := FilingMeta.get_static_documents(filing.storage, f"{base_url}{doc_url}/static")) + ): + documents["documents"]["staticDocuments"] = static_docs + + @staticmethod + def get_document_list(business, filing, jwt: JwtManager) -> dict | None: # NOSONAR(S3776) + """Return a list of documents for a particular filing.""" + if Filing._is_invalid_status_for_document_list(filing): + return None + + base_url = current_app.config.get("BUSINESS_API_GW_URL") + identifier = Filing._get_identifier_for_doc_list(business, filing) + doc_url = url_for("API2.get_documents", identifier=identifier, filing_id=filing.id, legal_filing_name=None) + + documents = {"documents": {}} + if filing.storage and filing.storage.paper_only: + return documents + + no_output_filings = [ + Filing.FilingTypes.CONVERSION.value, + Filing.FilingTypes.PUTBACKOFF.value, + Filing.FilingTypes.PUTBACKON.value, + Filing.FilingTypes.REGISTRARSNOTATION.value, + Filing.FilingTypes.REGISTRARSORDER.value + ] + if filing.storage and filing.storage.filing_type in no_output_filings: + return documents + + user_is_ca = is_competent_authority(jwt) + Filing._add_receipt(documents, filing, base_url, doc_url) + + only_receipt = Filing._has_no_outputs_except_receipt(filing) + no_legal_filings_before_completion = Filing._has_no_legal_filings_in_paid_or_withdrawn_status(filing, business) + + is_paid_or_withdrawn = filing.status in (Filing.Status.PAID, Filing.Status.WITHDRAWN) + is_completed_notice_of_withdrawal = ( + filing.status == Filing.Status.COMPLETED and + filing.filing_type == Filing.FilingTypes.NOTICEOFWITHDRAWAL.value + ) + + if ( + ( + is_paid_or_withdrawn and + not (only_receipt or no_legal_filings_before_completion) + ) or + is_completed_notice_of_withdrawal + ): + documents["documents"]["legalFilings"] = [ + {filing.filing_type: f"{base_url}{doc_url}/{filing.filing_type}"} + ] if user_is_ca: del documents["documents"]["receipt"] return documents legal_filings = filing.storage.meta_data.get("legalFilings") if filing.storage.meta_data else None - if not legal_filings and filing.storage and filing.storage.source == filing.storage.Source.COLIN.value: + if not legal_filings and filing.storage.source == filing.storage.Source.COLIN.value: legal_filings = [filing.filing_type] + if ( filing.status in (Filing.Status.COMPLETED, Filing.Status.CORRECTED) and - filing.storage.meta_data and - legal_filings - and not (no_outputs_except_receipt or no_outputs_except_receipt_dissolution) + filing.storage.meta_data and legal_filings and + not only_receipt ): - legal_filings_copy = copy.deepcopy(legal_filings) - if ( - filing.filing_type == Filing.FilingTypes.SPECIALRESOLUTION.value and - business.legal_type == Business.LegalTypes.COOP.value - ): - # add special resolution application output - documents["documents"]["specialResolutionApplication"] = \ - f"{base_url}{doc_url}/specialResolutionApplication" - if Filing.FilingTypes.CHANGEOFNAME.value in legal_filings: - # suppress change of name output for MVP since the design is outdated. - legal_filings_copy.remove(Filing.FilingTypes.CHANGEOFNAME.value) - if Filing.FilingTypes.ALTERATION.value in legal_filings: - # suppress alteration output for MVP since the design is outdated. - legal_filings_copy.remove(Filing.FilingTypes.ALTERATION.value) - - no_legal_filings = [ - Filing.FilingTypes.AMALGAMATIONOUT.value, - Filing.FilingTypes.CONSENTAMALGAMATIONOUT.value, - Filing.FilingTypes.CONSENTCONTINUATIONOUT.value, - Filing.FilingTypes.CONTINUATIONOUT.value, - Filing.FilingTypes.COURTORDER.value, - Filing.FilingTypes.AGMEXTENSION.value, - Filing.FilingTypes.AGMLOCATIONCHANGE.value, - Filing.FilingTypes.TRANSPARENCY_REGISTER.value, - Filing.FilingTypes.CHANGEOFOFFICERS.value - ] - if filing.filing_type not in no_legal_filings: - documents["documents"]["legalFilings"] = \ - [{doc: f"{base_url}{doc_url}/{doc}"} for doc in legal_filings_copy] - - # get extra outputs - if filing.storage.transaction_id and \ - (bus_rev_temp := VersionedBusinessDetailsService.get_business_revision_obj( - filing.storage, business.id)): - business = bus_rev_temp - - adds = [FilingMeta.get_all_outputs(business.legal_type, doc) for doc in legal_filings] - additional = {item for sublist in adds for item in sublist} - FilingMeta.alter_outputs(filing.storage, business, additional) - for doc in additional: - documents["documents"][doc] = f"{base_url}{doc_url}/{doc}" - - # continuationOut uploaded documents are visible to clients as well as staff - # (see https://github.com/bcgov/entity/issues/33788); all other static - # documents (e.g. continuationIn affidavit/authorization files) remain staff-only. - static_documents_visible = ( - has_roles(jwt, [UserRoles.staff]) or - filing.storage.filing_type == Filing.FilingTypes.CONTINUATIONOUT.value + Filing._populate_completed_filing_documents( + documents, + filing, + business, + jwt, + base_url, + doc_url, + legal_filings ) - if ( - static_documents_visible and - (static_docs := FilingMeta.get_static_documents(filing.storage, f"{base_url}{doc_url}/static")) - ): - documents["documents"]["staticDocuments"] = static_docs if user_is_ca: del documents["documents"]["receipt"] - return documents \ No newline at end of file + return documents diff --git a/legal-api/src/legal_api/core/meta/filing.py b/legal-api/src/legal_api/core/meta/filing.py index ade85564b5..105ef0f536 100644 --- a/legal-api/src/legal_api/core/meta/filing.py +++ b/legal-api/src/legal_api/core/meta/filing.py @@ -1051,22 +1051,22 @@ def alter_outputs_dissolution(filing, outputs): if filing.filing_type == "dissolution": # Suppress Certificate of Dissolution for Admin Dissolution if filing.filing_sub_type == "administrative": - outputs.remove("certificateOfDissolution") + outputs.discard("certificateOfDissolution") # Suppress Certified Memorandum and Certified Rules for Coop Voluntary Dissolution if filing.filing_sub_type == "voluntary" and filing.json_legal_type == Business.LegalTypes.COOP: - outputs.remove("certifiedRules") - outputs.remove("certifiedMemorandum") + outputs.discard("certifiedRules") + outputs.discard("certifiedMemorandum") return outputs @staticmethod def alter_outputs_special_resolution(filing, outputs): """Handle output file list modification for special resolution.""" if filing.filing_type == "specialResolution": - outputs.remove("certifiedMemorandum") + outputs.discard("certifiedMemorandum") if "changeOfName" in filing.meta_data.get("legalFilings", []): outputs.add("certificateOfNameChange") if not filing.meta_data.get("alteration", {}).get("uploadNewRules"): - outputs.remove("certifiedRules") + outputs.discard("certifiedRules") return outputs @staticmethod @@ -1074,28 +1074,53 @@ def get_static_documents(filing, url_prefix): """Get static documents.""" outputs = [] if filing.filing_type == "continuationIn": - continuation_in = filing.meta_data.get("continuationIn", {}) - if file_key := continuation_in.get("affidavitFileKey"): + FilingMeta._get_continuation_in_static_documents(filing, url_prefix, outputs) + elif filing.filing_type == "continuationOut": + FilingMeta._get_continuation_out_static_documents(filing, url_prefix, outputs) + elif filing.filing_type == "courtOrder": + FilingMeta._get_court_order_static_documents(filing, url_prefix, outputs) + return outputs + + @staticmethod + def _get_continuation_in_static_documents(filing, url_prefix, outputs): + continuation_in = filing.meta_data.get("continuationIn", {}) + if file_key := continuation_in.get("affidavitFileKey"): + outputs.append({ + "name": "Unlimited Liability Corporation Information", + "url": f"{url_prefix}/{file_key}" + }) + if authorization_files := continuation_in.get("authorizationFiles"): + for file in authorization_files: + file_key = file.get("fileKey") outputs.append({ - "name": "Unlimited Liability Corporation Information", + "name": file.get("fileName"), + "url": f"{url_prefix}/{file_key}" + }) + + @staticmethod + def _get_continuation_out_static_documents(filing, url_prefix, outputs): + continuation_out = filing.meta_data.get("continuationOut", {}) + for file in continuation_out.get("uploadedDocuments", []): + if file_key := file.get("fileKey"): + outputs.append({ + "name": file.get("fileName"), + "url": f"{url_prefix}/{file_key}" + }) + + @staticmethod + def _get_court_order_static_documents(filing, url_prefix, outputs): + court_order = filing.meta_data.get("courtOrder", {}) + if files := court_order.get("files"): + for file in files: + file_key = file.get("fileKey") + file_name = file.get("fileName") + if file_name.lower().endswith(".pdf"): + # This may not be required. Doing this to align with the current behavior + file_name = file_name[:-4] + outputs.append({ + "name": file_name, "url": f"{url_prefix}/{file_key}" }) - if authorization_files := continuation_in.get("authorizationFiles"): - for file in authorization_files: - file_key = file.get("fileKey") - outputs.append({ - "name": file.get("fileName"), - "url": f"{url_prefix}/{file_key}" - }) - elif filing.filing_type == "continuationOut": - continuation_out = filing.meta_data.get("continuationOut", {}) - for file in continuation_out.get("uploadedDocuments", []): - if file_key := file.get("fileKey"): - outputs.append({ - "name": file.get("fileName"), - "url": f"{url_prefix}/{file_key}" - }) - return outputs @staticmethod def get_display_name(legal_type: str, filing_type: str, filing_sub_type: str | None = None) -> str: diff --git a/legal-api/src/legal_api/reports/document_service.py b/legal-api/src/legal_api/reports/document_service.py index 62687909c0..38ad6f3d70 100644 --- a/legal-api/src/legal_api/reports/document_service.py +++ b/legal-api/src/legal_api/reports/document_service.py @@ -34,34 +34,48 @@ DRS_DOCUMENT_PARAMS: str = "?documentClass={document_class}&drsId={drs_id}" # Used for audit trail and db queries. BUSINESS_API_ACCOUNT_ID: str = "business-api" +# Each output key may map to more than one DRS documentClass-documentType combination. FILING_DOCUMENTS = { - "certifiedRules": { + "certifiedRules": [{ "documentType": "COOP_RULES", "documentClass": "COOP" - }, - "certifiedMemorandum": { + }], + "certifiedMemorandum": [{ "documentType": "COOP_MEMORANDUM", "documentClass": "COOP" - }, - "affidavit": { - "documentType": "CORP_AFFIDAVIT", # "COSD", + }], + "affidavit": [{ + "documentType": "COSD", + "documentClass": "COOP" + }, { + "documentType": "CORP_AFFIDAVIT", "documentClass": "CORP" - }, - "uploadedCourtOrder": { - "documentType": "CRT", + }], + "continuationIn": [{ + "documentType": "CNTA", "documentClass": "CORP" - } + }], + "continuationOut": [{ + "documentType": "CNTO", + "documentClass": "CORP" + }] } +# Map to DRS document based on the default static document name set when the client does not submit a filename. +# Included for migrated minio docs where submissions with no filename were allowed. STATIC_DOCUMENTS = { "Unlimited Liability Corporation Information": { - "documentType": "DIRECTOR_AFFIDAVIT", - "documentClass": "CORP" + "documentType": "DIRECTOR_AFFIDAVIT" + }, + "Court Order": { + "documentType": "CRTO" } } APP_JSON = "application/json" APP_PDF = "application/pdf" DOC_PATH = "/documents" BEARER = "Bearer " +DS_ID_PREFIX = "DS" +DS_KEY_TOKENS = 2 class ReportTypes(BaseEnum): @@ -276,7 +290,7 @@ def update_document_list(self, drs_docs: list, document_list: list, filing: Fili doc_list = self._update_legal_filing(doc_list, query_params) elif report_type.startswith("FILING"): # Additional filings - doc_list = self._update_additional_filing(doc_list, query_params) + doc_list = self._update_additional_filing(doc_list, query_params, report_type) elif report_type == "CERT": doc_list = self._update_certificate(doc_list, query_params) elif doc.get("documentClass"): @@ -320,7 +334,7 @@ def update_filing_documents(self, drs_docs: list, filing_docs: list, filing: Fil doc_list = self._update_legal_filing(doc_list, query_params) elif report_type.startswith("FILING"): # Additional filings - doc_list = self._update_additional_filing(doc_list, query_params) + doc_list = self._update_additional_filing(doc_list, query_params, report_type) elif report_type == "CERT": doc_list = self._update_certificate(doc_list, query_params) elif doc.get("documentClass"): @@ -340,7 +354,8 @@ def get_filing_report(self, drs_id: str, report_type: str): headers = self._get_request_headers(BUSINESS_API_ACCOUNT_ID, APP_PDF) url: str = self.url.replace(DOC_PATH, "") get_url = GET_REPORT_PATH - if report_type in [ReportTypes.FILING.value, ReportTypes.FILING_2.value, ReportTypes.NOA.value]: + if report_type in [ReportTypes.FILING.value, ReportTypes.FILING_2.value, ReportTypes.FILING_4.value, + ReportTypes.NOA.value]: get_url = GET_REPORT_CERTIFIED_PATH get_url = get_url.format(url=url, product=self.product_code, drsId=drs_id) response = requests.get(url=get_url, headers=headers) @@ -500,10 +515,10 @@ def replace_filing_report( return response2 return report_response - def _update_static_document(self, doc: dict, doc_list: list) -> list: """ Update static document urls in the document_list if a DRS match is found. + For static documents try obtaining download URL params from the file key first (DRS key). docs: The DRS document information to match on. doc_list: The business list of reports/documents for the filing. @@ -516,23 +531,73 @@ def _update_static_document(self, doc: dict, doc_list: list) -> list: for key in doc_list: if key == "staticDocuments": for static_doc in doc_list.get("staticDocuments"): - name: str = static_doc.get("name") - if (name and name == doc.get("name")) or ( - name and STATIC_DOCUMENTS.get(name) and - doc.get("documentClass") == STATIC_DOCUMENTS[name].get("documentClass") and - doc.get("documentType") == STATIC_DOCUMENTS[name].get("documentType") - ): + if self.is_static_doc_match(doc, static_doc): static_doc["url"] = static_doc["url"] + query_params break - elif ( - FILING_DOCUMENTS.get(key) and - doc.get("documentClass") == FILING_DOCUMENTS[key].get("documentClass") and - doc.get("documentType") == FILING_DOCUMENTS[key].get("documentType") + elif any( + doc.get("documentClass") == combo.get("documentClass") and + doc.get("documentType") == combo.get("documentType") + for combo in FILING_DOCUMENTS.get(key, []) ): doc_list[key] = doc_list[key] + query_params break return doc_list + def get_url_drs_id(self, url: str) -> str: + """ + Try to extract a DRS ID from the static document url (either minio or DRS). + + url: URL to use. + return: DRS ID if url ends with DRS file key. + """ + file_key: str = str(url).split("/")[-1] + if file_key: + tokens = file_key.split("-") + if ( + len(tokens) == DS_KEY_TOKENS and + tokens[0] in ("COOP", "FIRM", "CORP") and + str(tokens[1]).startswith(DS_ID_PREFIX) + ): + return tokens[1] + return "" + + + def is_static_doc_match(self, doc: dict, static_doc: dict) -> bool: + """ + Match the current DRS document with a ledger filing static document entry. + Try obtaining download URL params from the file key first (DRS key). + + doc: Current DRS document to match with static document. + doc_list: The business list of reports/documents for the filing. + return: True if the DRS doc matches the static document information. + """ + # Name is the ledger link name and should always exist, otherwise there is a defect somewhere else. + name: str = static_doc.get("name", "") + if not name or str(static_doc.get("url")).find("documentClass=") > 0: + return False + drs_id = self.get_url_drs_id(static_doc.get("url", "")) + if drs_id: + return drs_id == doc.get("identifier") + doc_name: str = doc.get("name", "") + if (doc_name): + if name.removesuffix(".pdf") == doc_name.removesuffix(".pdf"): + return True + if ( + STATIC_DOCUMENTS.get(name) and + doc.get("documentType") == STATIC_DOCUMENTS[name].get("documentType") + ): + return True + # Try partial matching static doc name with default name. + for key, value in STATIC_DOCUMENTS.items(): + if ( + name.startswith(str(key)) and + (doc_name.startswith(str(key)) or doc_name == "") and + value.get("documentType") == doc.get("documentType") + ): + return True + return False + + def _get_request_headers(self, account_id: str, accept_mime_type = None) -> dict: """ Get request headers for the DRS api call. @@ -559,11 +624,20 @@ def _update_legal_filing(self, doc_list: dict, drs_params: str) -> dict: legal_filing[filing_key] = legal_filing[filing_key] + drs_params return doc_list - def _update_additional_filing(self, doc_list: dict, drs_params: str) -> dict: + def _update_additional_filing(self, doc_list: dict, drs_params: str, report_type: str | None = None) -> dict: """Conditionally add the DRS params to the additional filing document url.""" + # Local import: report.py imports this module. + from legal_api.reports.report import ReportMeta + if doc_list.get("specialResolutionApplication"): # Not in configuration doc_list["specialResolutionApplication"] = doc_list["specialResolutionApplication"] + drs_params return doc_list + for legal_filing in (doc_list.get("legalFilings") or [])[1:]: + # A special resolution accompanying another filing is stored as FILING-2 (#34299); decorate + # its legal filing entry so the fallback below cannot attach the params to another output. + if "specialResolution" in legal_filing: + legal_filing["specialResolution"] = legal_filing["specialResolution"] + drs_params + return doc_list if doc_list.get("legalFilings"): legal_filing = doc_list["legalFilings"][0] filing_key = next(iter(legal_filing.keys())) @@ -576,6 +650,11 @@ def _update_additional_filing(self, doc_list: dict, drs_params: str) -> dict: not str(output).startswith("certificate") and doc_list.get(output) ): + # A DRS record only maps to an output configured with the same + # report type (e.g. skip legacy FILING-2 letters of consent). + output_report_type = ReportMeta.reports.get(output, {}).get("reportType") + if report_type and output_report_type and output_report_type != report_type: + continue doc_list[output] = doc_list[output] + drs_params break return doc_list diff --git a/legal-api/src/legal_api/reports/report.py b/legal-api/src/legal_api/reports/report.py index deef60e755..63d12aa1f2 100644 --- a/legal-api/src/legal_api/reports/report.py +++ b/legal-api/src/legal_api/reports/report.py @@ -13,6 +13,7 @@ import copy import json import os +import re from contextlib import suppress from http import HTTPStatus from pathlib import Path @@ -76,11 +77,17 @@ def get_pdf(self, report_type=None, regenerate: bool=False): def _get_static_report(self): document_type = ReportMeta.static_reports[self._report_key]["documentType"] document: Document = self._filing.documents.filter(Document.type == document_type).first() - from legal_api.services import MinioService - response = MinioService.get_file(document.file_key) + # DRS-backed keys are "{documentClass}-{documentServiceId}", e.g. "COOP-DS0000101951"; + if not (match := re.match(r"^([A-Z]+)-(DS\d+)$", document.file_key or "")): + return current_app.response_class(status=HTTPStatus.NOT_FOUND) + # the DRS applies the certified copy stamp itself for configured combinations + # (e.g. COOP-COSD), so DRS-served documents must not be stamped again here + from legal_api.services import doc_service + drs_response = doc_service.get_document(match.group(2), match.group(1), doc_binary=True) + document_data, status = drs_response.content, drs_response.status_code return current_app.response_class( - response=response.data, - status=response.status, + response=document_data, + status=status, mimetype="application/pdf" ) @@ -93,6 +100,18 @@ def _get_report(self, regenerate: bool = False): report_meta: dict = ReportMeta.reports.get(self._report_key) if not report_meta: report_meta = ReportMeta.reports.get("default") + if self._report_key == "specialResolution" and self._filing.filing_type != "specialResolution": + # A special resolution accompanying another filing (e.g. a coop dissolution) must not share + # the FILING report type with that filing's own report, or the DRS lookup below serves + # whichever of the two documents was stored first (#34299). + report_meta = {**report_meta, "reportType": ReportTypes.FILING_2.value} + if self._filing.filing_type == "annualReport" and \ + (accompanying_type := {"changeOfDirectors": ReportTypes.FILING_2.value, + "changeOfAddress": ReportTypes.FILING_4.value}.get(self._report_key)): + # A coop annual report can bundle director and address changes as additional legal filings; + # each output must have its own report type or the DRS lookup below serves whichever of the + # documents was stored first (#34457). + report_meta = {**report_meta, "reportType": accompanying_type} report_type = report_meta.get("reportType") if business_identifier and not regenerate: # Skip if regenerating and replacing DRS doc. document, status = self._document_service.get_filing_report_by_filing_id( @@ -185,6 +204,8 @@ def _substitute_template_parts(template_code): """ template_path = current_app.config.get("REPORT_TEMPLATE_PATH") template_parts = [ + "amalgamation/certificateFooter", + "amalgamation/certificateStyle", "amalgamation/amalgamatingCorp", "amalgamation/amalgamationName", "amalgamation/amalgamationStmt", @@ -346,7 +367,7 @@ def _format_filing_json(self, filing): # noqa: PLR0912 self._format_certificate_of_restoration_data(filing) elif self._report_key == "restoration": self._format_restoration_data(filing) - elif self._report_key in {"letterOfConsent", "letterOfConsentAmalgamationOut"}: + elif self._report_key in {"letterOfConsent", "letterOfConsentAmalgamationOut", "consentContinuationOut"}: self._format_consent_continuation_amalgamation_out_data(filing) elif self._report_key == "correction": self._format_correction_data(filing) @@ -411,10 +432,16 @@ def _set_completing_party(self, filing): ) if is_corp_incorp and incorp_compparty_stmnt_enabled: - if self._filing.submitter_roles in [UserRoles.staff]: + # staff and API gateway users supply the completing party name via header certifiedBy; + # for API users the token resolves to the account name, not a user name, so it can't be + # sourced from the submitter (API users are identified by the jwt loginSource, not a role). + api_login_source = "API_GW" # jwt loginSource of an API gateway user + submitter = self._filing.filing_submitter + if (self._filing.submitter_roles == UserRoles.staff + or (submitter and submitter.login_source == api_login_source)): filing["header"]["certifiedBy"] = self._filing.filing_json["filing"]["header"].get("certifiedBy") else: - filing["header"]["certifiedBy"] = self._filing.filing_submitter.display_name + filing["header"]["certifiedBy"] = submitter.display_name if not filing["header"]["certifiedBy"]: raise BusinessException( @@ -1496,7 +1523,11 @@ def _format_special_resolution(self, filing): display_name = FILINGS.get(self._filing.filing_type, {}).get("displayName") if isinstance(display_name, dict): display_name = display_name.get(self._business.legal_type) - filing_source = "specialResolution" if self._filing.filing_type == "specialResolution" else "correction" + # coop dissolutions carry a specialResolution section alongside the dissolution filing, so + # source the resolution from it rather than defaulting to the (empty) correction section + filing_source = "specialResolution" \ + if self._filing.filing_type == "specialResolution" or filing.get("specialResolution") \ + else "correction" filing["header"]["displayName"] = display_name resolution_date_str = filing.get(filing_source, {}).get("resolutionDate", None) signing_date_str = filing.get(filing_source, {}).get("signingDate", None) @@ -1949,6 +1980,11 @@ class ReportMeta: # pylint: disable=too-few-public-methods "fileName": "letterOfConsentAmalgamationOut", "reportType": ReportTypes.FILING_2.value # change to filing-3 to omit stamp }, + "consentContinuationOut": { + "filingDescription": "Continue Out Application", + "fileName": "consentContinuationOut", + "reportType": ReportTypes.FILING.value + }, "letterOfAgmExtension": { "filingDescription": "Letter Of AGM Extension", "fileName": "letterOfAgmExtension", @@ -2004,8 +2040,5 @@ class ReportMeta: # pylint: disable=too-few-public-methods }, "affidavit": { "documentType": "affidavit" - }, - "uploadedCourtOrder": { - "documentType": "court_order" } } diff --git a/legal-api/src/legal_api/reports/utils.py b/legal-api/src/legal_api/reports/utils.py index 129dc1e8f6..e8e8eede19 100644 --- a/legal-api/src/legal_api/reports/utils.py +++ b/legal-api/src/legal_api/reports/utils.py @@ -65,6 +65,9 @@ def get_formatted_amalg_business_data( ting_business: Business | None = None ): """Return the amalgamation business data for the report output.""" + is_bc_company = False + is_extraprovincial = False + if foreign_name: # Set identifier to 'N/A' for foreign businesses (we are showing the 'Number in BC' in the output) display_identifier = "N/A" @@ -73,15 +76,14 @@ def get_formatted_amalg_business_data( region_code = foreign_region_code # FUTURE: rework this once expros are in lear # Check if this is an expro - if (identifier - and identifier.startswith("A") - and (colin_resp := ColinService.query_business(identifier)) - and colin_resp.status_code == HTTPStatus.OK - ): - # this is an expro so set the identifier (it is the BC expro identifier) - display_identifier = identifier - # overwrite the region_code if jurisdiction is available in the response - region_code = colin_resp.json().get("business", {}).get("jurisdiction") + if identifier and identifier.startswith("A"): + colin_json, colin_status = ColinService.query_business(identifier) + if colin_json is not None and colin_status == HTTPStatus.OK: + # this is an expro so set the identifier (it is the BC expro identifier) + display_identifier = identifier + is_extraprovincial = True + # overwrite the region_code if jurisdiction is available in the response + region_code = colin_json.get("business", {}).get("jurisdiction") else: if not ting_business: @@ -93,11 +95,14 @@ def get_formatted_amalg_business_data( business_legal_name = ting_business.legal_name country_code = "CA" region_code = "BC" + is_bc_company = True jurisdiction = get_amalg_formatted_jurisdiction(identifier, country_code, region_code) return { "legalName": business_legal_name or "N/A", "identifier": display_identifier or "N/A", - "jurisdiction": jurisdiction or "N/A" - } \ No newline at end of file + "jurisdiction": jurisdiction or "N/A", + "isBcCompany": is_bc_company, + "isExtraprovincial": is_extraprovincial + } diff --git a/legal-api/src/legal_api/resources/v2/business/__init__.py b/legal-api/src/legal_api/resources/v2/business/__init__.py index 3e807b6303..d80ec7d211 100644 --- a/legal-api/src/legal_api/resources/v2/business/__init__.py +++ b/legal-api/src/legal_api/resources/v2/business/__init__.py @@ -27,6 +27,8 @@ from .business_parties import get_parties from .business_resolutions import get_resolutions from .business_account_settings import get_business_account_settings, update_business_account_settings +from .business_extended import get_extended_data +from .business_court_orders import get_court_orders from .business_share_classes import get_share_class from .business_tasks import get_tasks from .colin_sync import ( diff --git a/legal-api/src/legal_api/resources/v2/business/business.py b/legal-api/src/legal_api/resources/v2/business/business.py index c4fbdf2923..8601127e72 100644 --- a/legal-api/src/legal_api/resources/v2/business/business.py +++ b/legal-api/src/legal_api/resources/v2/business/business.py @@ -31,6 +31,7 @@ SYSTEM_ROLE, RegistrationBootstrapService, check_warnings, + colin, flags, ) from legal_api.services.authz import authorized, get_allowable_actions, get_allowed, get_could_files @@ -136,6 +137,30 @@ def get_businesses_public(identifier: str, slim = False): return jsonify(business=business_json) +@bp.route("//colin-snapshot", methods=["GET"]) +@cross_origin() +@jwt.requires_auth +def get_businesses_colin_snapshot(identifier: str): + """Return the snapshot data of a COLIN business not yet managed in LEAR.""" + if not authorized(identifier, jwt, action=["view"]): + return jsonify({"message": + f"You are not authorized to view business {identifier}."}), \ + HTTPStatus.UNAUTHORIZED + + if Business.find_by_identifier(identifier): + # a business in LEAR is managed here - its COLIN data may be stale + return jsonify({"message": f"{identifier} is managed in the Business Registry."}), HTTPStatus.NOT_FOUND + + snapshot_json, status_code = colin.get_snapshot(identifier) + if status_code == HTTPStatus.NOT_FOUND: + return jsonify({"message": f"{identifier} not found"}), HTTPStatus.NOT_FOUND + if snapshot_json is None or status_code != HTTPStatus.OK: + return jsonify({"message": f"Unable to retrieve {identifier} data from COLIN."}), \ + HTTPStatus.INTERNAL_SERVER_ERROR + + return jsonify(snapshot_json), HTTPStatus.OK + + @bp.route("", methods=["POST"]) @cross_origin() @jwt.requires_auth diff --git a/legal-api/src/legal_api/resources/v2/business/business_court_orders.py b/legal-api/src/legal_api/resources/v2/business/business_court_orders.py new file mode 100644 index 0000000000..b9d28fa400 --- /dev/null +++ b/legal-api/src/legal_api/resources/v2/business/business_court_orders.py @@ -0,0 +1,87 @@ +# Copyright © 2020 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Retrieve the court orders for the entity.""" +from http import HTTPStatus + +from flask import current_app, jsonify, url_for +from flask_cors import cross_origin + +from business_model.models import Business, CourtOrder, Document, DocumentType, Filing +from legal_api.services import authorized +from legal_api.utils.auth import jwt + +from .bp import bp + + +@bp.route("//court-orders", methods=["GET", "OPTIONS"]) +@bp.route("//court-orders/", methods=["GET", "OPTIONS"]) +@cross_origin() +@jwt.requires_auth +def get_court_orders(identifier, court_order_id=None): + """Return a JSON of the court orders.""" + business = Business.find_by_identifier(identifier) + + if not business: + return jsonify({"message": f"{identifier} not found"}), HTTPStatus.NOT_FOUND + + # check authorization + if not authorized(identifier, jwt, action=["view"]): + return jsonify({"message": + f"You are not authorized to view court orders for {identifier}."}), \ + HTTPStatus.UNAUTHORIZED + + # return the matching court order + if court_order_id: + court_order, code = _get_court_order(business, court_order_id) + return jsonify(court_order), code + + court_orders_list = CourtOrder.get_json_with_filing_type(business.id) + return jsonify({ + "courtOrders": court_orders_list + }), HTTPStatus.OK + + +def _get_court_order(business, court_order_id=None): + if court_order := CourtOrder.get_by_id(court_order_id): + court_order_json = court_order.json + filing = Filing.find_by_id(court_order.filing_id) + if filing.filing_type == "courtOrder": + _include_court_order_files(court_order_json, filing, business) + + return {"courtOrder": court_order_json}, HTTPStatus.OK + + return {"message": f"{business.identifier} court order not found"}, HTTPStatus.NOT_FOUND + + +def _include_court_order_files(court_order_json, filing, business): + documents = Document.find_all_by(filing.id, DocumentType.COURT_ORDER.value) + if documents: + base_url = current_app.config.get("BUSINESS_API_GW_URL") + doc_url = url_for( + "API2.get_documents", + identifier=business.identifier, + filing_id=filing.id, + legal_filing_name=None + ) + files = [] + for doc in documents: + file_name = doc.file_name + if not file_name: + file_name = f"Court Order {court_order_json.get('fileNumber')}" + files.append({ + "fileName": file_name, + "fileKey": doc.file_key, + "url": f"{base_url}{doc_url}/static/{doc.file_key}", + }) + court_order_json["files"] = files diff --git a/legal-api/src/legal_api/resources/v2/business/business_extended.py b/legal-api/src/legal_api/resources/v2/business/business_extended.py new file mode 100644 index 0000000000..d6ed35152a --- /dev/null +++ b/legal-api/src/legal_api/resources/v2/business/business_extended.py @@ -0,0 +1,200 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Retrieve the extended data for the entity.""" +from http import HTTPStatus + +from flask import jsonify, request +from flask_cors import cross_origin + +from business_model.models import Business, Filing, Jurisdiction +from business_model.utils.legislation_datetime import LegislationDatetime +from legal_api.core import Filing as CoreFiling +from legal_api.services import authorized +from legal_api.utils.auth import jwt + +from .bp import bp + + +@bp.route("//extended/", methods=["GET", "OPTIONS"]) +@bp.route("//extended", methods=["GET", "OPTIONS"]) +@cross_origin() +@jwt.requires_auth +def get_extended_data(identifier, filing_type=None): + """Return a JSON of the extended data for a filing type.""" + business = Business.find_by_identifier(identifier) + if not business: + return jsonify({"message": f"{identifier} not found"}), HTTPStatus.NOT_FOUND + + # check authorization + if not authorized(identifier, jwt, action=["view"]): + return jsonify({"message": f"You are not authorized to view extended data for {identifier}."}), \ + HTTPStatus.UNAUTHORIZED + + for_correction = str(request.args.get("forCorrection", None)).lower() == "true" + + if filing_type: + return _get_extended_filing_data(business, filing_type, for_correction) + else: + return _get_all_extended_data(business, for_correction) + + +def _get_extended_filing_data(business: Business, filing_type: str, for_correction: bool = False): + supported_filing_types = [ + CoreFiling.FilingTypes.AMALGAMATIONAPPLICATION, + CoreFiling.FilingTypes.AMALGAMATIONOUT, + CoreFiling.FilingTypes.CONTINUATIONIN, + CoreFiling.FilingTypes.CONTINUATIONOUT, + ] + if filing_type not in supported_filing_types: + return jsonify({"message": f"{filing_type} not supported"}), HTTPStatus.BAD_REQUEST + + if filing_type == CoreFiling.FilingTypes.CONTINUATIONIN: + return _get_continuation_in_data(business) + elif filing_type == CoreFiling.FilingTypes.CONTINUATIONOUT: + return _get_continuation_out_data(business) + elif filing_type == CoreFiling.FilingTypes.AMALGAMATIONOUT: + return _get_amalgamation_out_data(business) + elif filing_type == CoreFiling.FilingTypes.AMALGAMATIONAPPLICATION: + return _get_amalgamation_application_data(business, for_correction) + + +def _get_all_extended_data(business: Business, for_correction): + continuation_in, continuation_in_status = _get_continuation_in_data(business) + continuation_out, continuation_out_status = _get_continuation_out_data(business) + amalgamation_out, amalgamation_out_status = _get_amalgamation_out_data(business) + amalgamation, amalgamation_status = _get_amalgamation_application_data(business, for_correction) + + return { + **(continuation_in if continuation_in_status == HTTPStatus.OK else {}), + **(amalgamation if amalgamation_status == HTTPStatus.OK else {}), + **(continuation_out if continuation_out_status == HTTPStatus.OK else {}), + **(amalgamation_out if amalgamation_out_status == HTTPStatus.OK else {}), + }, HTTPStatus.OK + + +def _get_continuation_in_data(business: Business): + jurisdiction = Jurisdiction.get_continuation_in_jurisdiction(business.id) + if not jurisdiction: + return jsonify({ + "message": f"Could not find continuationIn data for {business.identifier}" + }), HTTPStatus.NOT_FOUND + + incorporation_date = None + if jurisdiction.incorporation_date: + incorporation_date = LegislationDatetime.as_legislation_timezone(jurisdiction.incorporation_date) + incorporation_date = incorporation_date.date().isoformat() + continuation_in = { + "country": jurisdiction.country, + "region": jurisdiction.region, + "identifier": jurisdiction.identifier, + "legalName": jurisdiction.legal_name, + "incorporationDate": incorporation_date + } + if jurisdiction.expro_identifier or jurisdiction.expro_legal_name: + continuation_in["expro"] = { + "identifier": jurisdiction.expro_identifier, + "legalName": jurisdiction.expro_legal_name + } + return { + "continuationIn": continuation_in + }, HTTPStatus.OK + + +def _get_continuation_out_data(business: Business): + if not ( + business.state == Business.State.HISTORICAL and + ( + (filing := Filing.find_by_id(business.state_filing_id)) and + filing.filing_type == CoreFiling.FilingTypes.CONTINUATIONOUT + ) + ): + return jsonify({ + "message": f"Could not find continuationOut data for {business.identifier}" + }), HTTPStatus.NOT_FOUND + + return { + "continuationOut": { + "date": LegislationDatetime.format_as_legislation_date(business.continuation_out_date), + "country": business.jurisdiction, + "region": business.foreign_jurisdiction_region, + "legalName": business.foreign_legal_name + } + }, HTTPStatus.OK + + +def _get_amalgamation_out_data(business: Business): + if not ( + business.state == Business.State.HISTORICAL and + ( + (filing := Filing.find_by_id(business.state_filing_id)) and + filing.filing_type == CoreFiling.FilingTypes.AMALGAMATIONOUT + ) + ): + return jsonify({ + "message": f"Could not find amalgamationOut data for {business.identifier}" + }), HTTPStatus.NOT_FOUND + + return { + "amalgamationOut": { + "date": LegislationDatetime.format_as_legislation_date(business.amalgamation_out_date), + "country": business.jurisdiction, + "region": business.foreign_jurisdiction_region, + "legalName": business.foreign_legal_name + } + }, HTTPStatus.OK + + +def _get_amalgamation_application_data(business: Business, for_correction: bool = False): + amalgamation = business.amalgamation.first() + if not amalgamation: + return jsonify({ + "message": f"Could not find amalgamation data for {business.identifier}" + }), HTTPStatus.NOT_FOUND + + amalgamating_businesses = [] + for ting in amalgamation.amalgamating_businesses: + ting_info = { + "id": ting.id, + "role": ting.role.name + } + if ting.business_id: + ting_business = Business.find_by_internal_id(ting.business_id) + ting_info.update({ + "identifier": ting_business.identifier + }) + if for_correction: + ting_info.update({ + "legalName": ting_business.legal_name, + "legalType": ting_business.legal_type + }) + mailing_address = ting_business.mailing_address.one_or_none() + if mailing_address: + ting_info["mailingAddress"] = mailing_address.json + else: + ting_info.update({ + "identifier": ting.foreign_identifier, + "foreignJurisdiction": { + "country": ting.foreign_jurisdiction, + "region": ting.foreign_jurisdiction_region + }, + "legalName": ting.foreign_name + }) + amalgamating_businesses.append(ting_info) + + return { + "amalgamation": { + "amalgamatingBusinesses": amalgamating_businesses, + "courtApproval": amalgamation.court_approval + } + }, HTTPStatus.OK diff --git a/legal-api/src/legal_api/resources/v2/business/business_filings/business_documents.py b/legal-api/src/legal_api/resources/v2/business/business_filings/business_documents.py index 770d123577..7cd095e3a3 100644 --- a/legal-api/src/legal_api/resources/v2/business/business_filings/business_documents.py +++ b/legal-api/src/legal_api/resources/v2/business/business_filings/business_documents.py @@ -15,6 +15,7 @@ Provides all the search and retrieval from the business entity documents. """ +import re from http import HTTPStatus from typing import Final @@ -25,6 +26,7 @@ from flask_pydantic import validate as pydantic_validate from pydantic import BaseModel +from business_account import AccountService from business_common.utils.legislation_datetime import LegislationDatetime from business_model.models import Business, Document, UserRoles from business_model.models import Filing as FilingModel @@ -33,7 +35,9 @@ from legal_api.reports import get_pdf from legal_api.reports.document_service import DocumentService from legal_api.resources.v2.business.bp import bp -from legal_api.services import MinioService, authorized +from legal_api.services import authorized +from legal_api.services import doc_service as client_doc_service +from legal_api.services.request_context import add_account_linking_key_header from legal_api.utils.auth import jwt from legal_api.utils.util import cors_preflight @@ -115,17 +119,18 @@ def get_documents(identifier: str, # noqa: PLR0911, PLR0912 if legal_filing_name: if legal_filing_name.lower().startswith("receipt"): - return _get_receipt(business, filing, jwt.get_token_auth_header()) + return _get_receipt(business, filing) return get_pdf(filing.storage, legal_filing_name) elif file_key and (document := Document.find_by_file_key(file_key)): - if document.filing_id == filing.id: # make sure the file belongs to this filing - response = MinioService.get_file(document.file_key) + if document.filing_id == filing.id and (match := re.match(r"^([A-Z]+)-(DS\d+)$", document.file_key)): # make sure the file belongs to this filing + drs_response = client_doc_service.get_document(match.group(2), match.group(1), doc_binary=True) return current_app.response_class( - response=response.data, - status=response.status, + response=drs_response.content, + status=drs_response.status_code, mimetype=APP_PDF ) + return {}, HTTPStatus.NOT_FOUND @@ -202,7 +207,7 @@ def _get_document_list(business: Business, filing: Filing): return jsonify(document_list), HTTPStatus.OK -def _get_receipt(business: Business, filing: Filing, token): +def _get_receipt(business: Business, filing: Filing): """Get the receipt for the filing.""" if filing.status not in ( Filing.Status.COMPLETED, @@ -219,8 +224,9 @@ def _get_receipt(business: Business, filing: Filing, token): filing.filing_type == "noticeOfWithdrawal" ): effective_date = LegislationDatetime.format_as_report_string(filing.storage.effective_date) - - headers = {"Authorization": "Bearer " + token} + service_token = AccountService.get_bearer_token() + headers = {"Authorization": "Bearer " + service_token} + add_account_linking_key_header(headers) corp_name = _get_corp_name(business, filing.storage) @@ -329,7 +335,6 @@ def _regenerate_documents(business: Business, filing: Filing, query: RegenerateQ docs.pop("receipt", None) docs.pop("staticDocuments", None) - docs.pop("uploadedCourtOrder", None) doc_keys.extend(docs.keys()) for doc_name in doc_keys: diff --git a/legal-api/src/legal_api/resources/v2/business/business_filings/business_filings.py b/legal-api/src/legal_api/resources/v2/business/business_filings/business_filings.py index 51786564a3..9366115dae 100644 --- a/legal-api/src/legal_api/resources/v2/business/business_filings/business_filings.py +++ b/legal-api/src/legal_api/resources/v2/business/business_filings/business_filings.py @@ -16,6 +16,7 @@ Provides all the search and retrieval from the business entity datastore. """ import copy +import re from contextlib import suppress from datetime import UTC from datetime import datetime as _datetime @@ -39,6 +40,7 @@ from business_model.models import ( Address, Business, + Document, Filing, OfficeType, RegistrationBootstrap, @@ -59,9 +61,9 @@ from legal_api.services import ( STAFF_ROLE, SYSTEM_ROLE, - MinioService, RegistrationBootstrapService, authorized, + doc_service, flags, namex, ) @@ -69,6 +71,7 @@ from legal_api.services.event_publisher import publish_to_queue from legal_api.services.filings import validate from legal_api.services.permissions import PermissionService +from legal_api.services.request_context import add_account_linking_key_header from legal_api.services.utils import get_str from legal_api.utils.auth import jwt @@ -129,7 +132,7 @@ def saving_filings(body: FilingModel, # noqa: PLR0911, PLR0912 business, filing = ListFilingResource.get_business_and_filing(identifier, filing_id) # basic checks - err_msg, err_code = ListFilingResource.put_basic_checks(identifier, filing, request, business) + err_msg, err_code = ListFilingResource.put_basic_checks(identifier, filing, request, business, query) if err_msg: return jsonify({"errors": [err_msg, ]}), err_code @@ -222,7 +225,7 @@ def delete_filings(identifier, filing_id=None): filing.delete() with suppress(Exception): - ListFilingResource.delete_from_minio(filing_type, filing_json) + ListFilingResource.delete_uploaded_documents(filing_type, filing_json) if identifier.startswith("T") and filing.filing_type != Filing.FILINGS["noticeOfWithdrawal"]["name"]: bootstrap = RegistrationBootstrap.find_by_identifier(identifier) @@ -268,6 +271,7 @@ def patch_filings(identifier, filing_id=None): payment_svc_url = "{}/{}".format(current_app.config.get("PAYMENT_SVC_URL"), filing.payment_token) token = jwt.get_token_auth_header() headers = {"Authorization": "Bearer " + token} + add_account_linking_key_header(headers) rv = requests.delete(url=payment_svc_url, headers=headers, timeout=20.0) if rv.status_code in (HTTPStatus.OK, HTTPStatus.ACCEPTED): filing.reset_filing_to_draft() @@ -326,6 +330,9 @@ def get_single_filing_public_json(filing_id: int): if filing.meta_data and (expiry_date := filing.meta_data.get(filing_name, {}).get("expiryDate")): filing_json[filing_name]["expiryDate"] = expiry_date + if filing.meta_data and (dissolution_date := filing.meta_data.get(filing_name, {}).get("dissolutionDate")): + filing_json[filing_name]["dissolutionDate"] = dissolution_date + return jsonify({"filing": filing_json}) @@ -384,6 +391,7 @@ def get_payment_update(filing_dict: dict): "Authorization": f"Bearer {jwt.get_token_auth_header()}", "Content-Type": "application/json" } + add_account_linking_key_header(headers) payment_svc_url = current_app.config.get("PAYMENT_SVC_URL") if payment_token := filing_dict.get("filing", {}).get("header", {}).get("paymentToken"): @@ -533,7 +541,7 @@ def get_notice_of_withdrawal(filing_id: str | None = None): return filing @staticmethod - def put_basic_checks(identifier, filing, client_request, business) -> tuple[dict, int]: + def put_basic_checks(identifier, filing, client_request, business, query) -> tuple[dict, int]: # noqa: PLR0911 """Perform basic checks to ensure put can do something.""" json_input = client_request.get_json(silent=True) if not json_input: @@ -550,6 +558,10 @@ def put_basic_checks(identifier, filing, client_request, business) -> tuple[dict if not filing_type: return ({"message": "filing/header/name is a required property"}, HTTPStatus.BAD_REQUEST) + filing_business_identifier = json_input.get("filing", {}).get("business", {}).get("identifier") + if not query.draft and filing_business_identifier != identifier: + return ({"message": "filing/business/identifier does not equal the identifier in the request path."}, HTTPStatus.BAD_REQUEST) + if filing_type not in [*CoreFiling.NEW_BUSINESS_FILING_TYPES, CoreFiling.FilingTypes.NOTICEOFWITHDRAWAL] \ and business is None: return ({"message": "A valid business is required."}, HTTPStatus.BAD_REQUEST) @@ -1050,6 +1062,7 @@ def create_invoice(business: Business, # noqa: PLR0912, PLR0915 token = user_jwt.get_token_auth_header() headers = {"Authorization": "Bearer " + token, "Content-Type": "application/json"} + add_account_linking_key_header(headers) current_app.logger.debug(f"Pay api call - url: {payment_svc_url}, payload: {payload}") rv = requests.post(url=payment_svc_url, json=payload, @@ -1106,8 +1119,18 @@ def is_future_effective_filing(filing_json: dict) -> bool: return is_future_effective @staticmethod - def delete_from_minio(filing_type: str, filing_json: dict): - """Delete file from minio.""" + def delete_uploaded_file(file_key: str): + """Delete an uploaded file from the DRS based on the file key shape. + + DRS-backed keys are "{documentClass}-{documentServiceId}", e.g. "COOP-DS0000101951"; + """ + if re.match(r"^([A-Z]+)-(DS\d+)$", file_key): + doc_service.delete_document(Document(file_key=file_key)) + + + @staticmethod + def delete_uploaded_documents(filing_type: str, filing_json: dict): + """Delete the filing's uploaded files from the DRS.""" if (filing_type == Filing.FILINGS["incorporationApplication"].get("name") and (cooperative := filing_json .get("filing", {}) @@ -1118,38 +1141,38 @@ def delete_from_minio(filing_type: str, filing_json: dict): .get("filing", {}) .get("alteration", {}))): if rules_file_key := cooperative.get("rulesFileKey", None): - MinioService.delete_file(rules_file_key) + ListFilingResource.delete_uploaded_file(rules_file_key) if memorandum_file_key := cooperative.get("memorandumFileKey", None): - MinioService.delete_file(memorandum_file_key) + ListFilingResource.delete_uploaded_file(memorandum_file_key) elif filing_type == Filing.FILINGS["dissolution"].get("name") \ and (affidavit_file_key := filing_json .get("filing", {}) .get("dissolution", {}) .get("affidavitFileKey", None)): - MinioService.delete_file(affidavit_file_key) + ListFilingResource.delete_uploaded_file(affidavit_file_key) elif filing_type == Filing.FILINGS["courtOrder"].get("name") \ and (file_key := filing_json .get("filing", {}) .get("courtOrder", {}) .get("fileKey", None)): - MinioService.delete_file(file_key) + ListFilingResource.delete_uploaded_file(file_key) elif filing_type == Filing.FILINGS["continuationIn"].get("name"): ListFilingResource.delete_continuation_in_files(filing_json) @staticmethod def delete_continuation_in_files(filing_json: dict): - """Delete continuation in files from minio.""" + """Delete continuation in files from DRS.""" continuation_in = filing_json.get("filing", {}).get("continuationIn", {}) # Delete affidavit file if affidavit_file_key := continuation_in.get("foreignJurisdiction", {}).get("affidavitFileKey", None): - MinioService.delete_file(affidavit_file_key) + ListFilingResource.delete_uploaded_file(affidavit_file_key) # Delete authorization file(s) authorization_files = continuation_in.get("authorization", {}).get("files", []) for file in authorization_files: if auth_file_key := file.get("fileKey", None): - MinioService.delete_file(auth_file_key) + ListFilingResource.delete_uploaded_file(auth_file_key) @staticmethod def details_for_invoice(business_identifier: str, corp_type: str): @@ -1167,6 +1190,9 @@ def details_for_invoice(business_identifier: str, corp_type: str): @staticmethod def modify_filing_json(filing_json, filing: Filing = None): """Modify filing json values if needed.""" + # Trim leading/trailing whitespace from all string fields + ListFilingResource._trim_strings(filing_json) + filing_type = filing_json["filing"]["header"]["name"] if (filing_type == Filing.FILINGS["continuationIn"].get("name") and filing and filing.status == Filing.Status.APPROVED.value @@ -1183,6 +1209,23 @@ def modify_filing_json(filing_json, filing: Filing = None): filing_json["filing"][filing_type]["foreignJurisdiction"] = \ filing.filing_json["filing"][filing_type]["foreignJurisdiction"] + @staticmethod + def _trim_strings(node): + """Recursively trim leading/trailing whitespace from all string values in place.""" + if isinstance(node, dict): + for key, value in node.items(): + if isinstance(value, str): + node[key] = value.strip() + elif isinstance(value, dict | list): + ListFilingResource._trim_strings(value) + elif isinstance(node, list): + for i, value in enumerate(node): + if isinstance(value, str): + node[i] = value.strip() + elif isinstance(value, dict | list): + ListFilingResource._trim_strings(value) + return node + @staticmethod def submit_filing_for_review(business: Business | RegistrationBootstrap, filing: Filing): """Submit filing for review.""" diff --git a/legal-api/src/legal_api/resources/v2/business/business_tasks.py b/legal-api/src/legal_api/resources/v2/business/business_tasks.py index 17f85d3ea7..2e28b1bdb1 100644 --- a/legal-api/src/legal_api/resources/v2/business/business_tasks.py +++ b/legal-api/src/legal_api/resources/v2/business/business_tasks.py @@ -28,6 +28,7 @@ from business_common.utils.legislation_datetime import LegislationDatetime from business_model.models import Business, Filing from legal_api.services import check_warnings, namex +from legal_api.services.request_context import add_account_linking_key_header from legal_api.services.warnings.business.business_checks import BusinessWarningCodes, WarningType from legal_api.utils.auth import jwt @@ -130,6 +131,7 @@ def construct_task_list(business: Business): # noqa: PLR0915 "Authorization": f"Bearer {jwt.get_token_auth_header()}", "Content-Type": "application/json" } + add_account_linking_key_header(headers) pay_response = requests.get( url=f'{current_app.config.get("PAYMENT_SVC_URL")}/{filing.payment_token}', headers=headers diff --git a/legal-api/src/legal_api/resources/v2/document.py b/legal-api/src/legal_api/resources/v2/document.py index 3ef24d75fc..09fa70f0c8 100644 --- a/legal-api/src/legal_api/resources/v2/document.py +++ b/legal-api/src/legal_api/resources/v2/document.py @@ -11,26 +11,81 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Module for handling Minio document operations.""" +"""Module for maintaining client documents.""" from http import HTTPStatus -from flask import Blueprint, current_app, jsonify +from flask import Blueprint, current_app, jsonify, request from flask_cors import cross_origin from business_model.models import Document, Filing -from legal_api.services.minio import MinioService +from legal_api.services import doc_service from legal_api.utils.auth import jwt bp = Blueprint("DOCUMENTS2", __name__, url_prefix="/api/v2/documents") +DOCUMENT_CLASS_DEFAULT = "CORP" +DOCUMENT_TYPE_DEFAULT = "COSD" +HEADER_ACCEPT = "Accept" +PARAM_FILENAME = "filename" +PARAM_FILEDATE = "filingDate" +PARAM_IDENTIFIER = "businessIdentifier" +PARAM_FILING_ID = "filingId" +DRS_PROPERTY_MAPPING = { + PARAM_FILENAME: "consumerFilename", + PARAM_FILEDATE: "consumerFilingDate", + PARAM_IDENTIFIER: "consumerIdentifier", + PARAM_FILING_ID: "consumerReferenceId" +} +DRS_MAPPING_DOC_CLASS = { + "CP": "COOP", + "GP": "FIRM", + "SP": "FIRM", + "DEFAULT": "CORP" +} +DRS_MAPPING_DOC_TYPE = { + "continuationIn": { + "documentTypes": { + "authorization_file": "CNTA", + "director_affidavit": "DIRECTOR_AFFIDAVIT" + } + }, + "continuationOut": { + "documentTypes": { + "continuation_out": "CNTO" + } + }, + "correction": { + "documentTypes": { + "coop_memorandum": "COOP_MEMORANDUM", + "coop_rules": "COOP_RULES" + } + }, + "courtOrder": { + "documentTypes": { + "court_order": "CRTO" + } + }, + "dissolution": { + "documentTypes": { + "affidavit": "COSD" + } + }, + "incorporationApplication": { + "documentTypes": { + "coop_memorandum": "COOP_MEMORANDUM", + "coop_rules": "COOP_RULES" + } + }, + "specialResolution": { + "documentTypes": { + "coop_memorandum": "COOP_MEMORANDUM", + "coop_rules": "COOP_RULES" + } + }, +} + -@bp.route("//signatures", methods=["GET"]) -@cross_origin() -@jwt.requires_auth -def get_signatures(file_name: str): - """Return a pre-signed URL for the new document.""" - return MinioService.create_signed_put_url(file_name), HTTPStatus.OK def is_draft_filing(file_key: str) -> bool: @@ -42,15 +97,85 @@ def is_draft_filing(file_key: str) -> bool: return filing and filing.status == Filing.Status.DRAFT.value -@bp.route("/", methods=["DELETE"]) + + + +@bp.route("/client///", methods=["POST"]) @cross_origin() @jwt.requires_auth -def delete_minio_document(document_key): - """Delete Minio document based on the provided document key and if it is a draft filing.""" +def create_client_document(filing_type, entity_type, document_type): + """ + Create a new document record where the payload is the document binary data. Additional information associated with + the document is submitted with following request parameters: + filename: The document file name. + businessIdentifier: The business identifier for change filings where the business exists: include if available. + + The combination of filing_type, entity_type, and document_type is used to derive a DRS document class and document + type. If no mapping is found the default values of DOCUMENT_CLASS_DEFAULT and DOCUMENT_TYPE_DEFAULT are used. + + The response JSON key is the unique identifier for the document record. The caller is expected to store + this value in the documents table file_key column. + + filing_type: One of the filing types that supports the submission of client documents. For example: + courtOrder, continuationOut, continuationIn, correction, dissolution, incorporationApplication, + specialResolution. + entity_type: One of the entity/legal business types. For example: + BC, BEN, C, CC, CP, CUL, GP, SP, ULC, CBEN, CCC + document_type: One of the business document types stored in the documents table. For example: + affidavit, court_order, continuation_out, authorization_file, director_affidavit, + coop_memorandum, coop_rules + return: Flask response. The response JSON payload key is the unique identifier for the document record. + Status codes: + CREATED: success + INTERNAL_SERVER_ERROR: default when any exception. + Any response code returned from the DRS API. + """ + try: + doc_info: dict = build_create_info(filing_type, entity_type, document_type, request.args) + response = doc_service.create_document(doc_info, request.get_data()) + response_json = doc_service.get_content(response) + if response.ok: + file_key = doc_info.get("documentClass") + "-" + response_json.get("documentServiceId") + current_app.logger.info(f"create_document successful file_key={file_key}") + response_json["key"] = file_key + return jsonify(response_json), HTTPStatus.CREATED + + current_app.logger.error(f"Error creating document record: {response_json}") + return jsonify( + message="Error creating document record." + ), response.status_code + except Exception as e: + current_app.logger.error(f"Error creating document record: {e}") + return jsonify( + message="Error creating document record." + ), HTTPStatus.INTERNAL_SERVER_ERROR + + +@bp.route("/client/", methods=["DELETE"]) +@cross_origin() +@jwt.requires_auth +def delete_client_document(document_key): + """ + Delete client document based on the provided document key and if it is a draft filing. + + document_key: DRS document class-DRS ID returned by a previously successful POST request. + return: Flask response. Status codes: + OK: success + FORBIDDEN: Filing is not in a state where an update is allowed. + INTERNAL_SERVER_ERROR: default when any exception. + Any response code returned from the DRS API, typically NOT_FOUND. + """ try: if is_draft_filing(document_key): - MinioService.delete_file(document_key) - return jsonify({"message": f"File {document_key} deleted successfully."}), HTTPStatus.OK + bus_doc: Document = Document(file_key=document_key) + response = doc_service.delete_document(bus_doc) + if response.ok: + return jsonify({"message": f"Document {document_key} deleted successfully."}), HTTPStatus.OK + api_error = doc_service.get_content(response) + current_app.logger.error(f"Error deleting file {document_key}: {api_error}") + return jsonify( + message=f"Error deleting file {document_key}." + ), response.status_code return jsonify({"message": "Filing is not a draft."}), HTTPStatus.FORBIDDEN except Exception as e: current_app.logger.error(f"Error deleting file {document_key}: {e}") @@ -59,20 +184,173 @@ def delete_minio_document(document_key): ), HTTPStatus.INTERNAL_SERVER_ERROR -@bp.route("/", methods=["GET"]) +@bp.route("/client/", methods=["GET"]) @cross_origin() @jwt.requires_auth -def get_minio_document(document_key: str): - """Get the document from Minio.""" +def get_client_document(document_key): + """ + Get a client document based on the provided document key returned by a previously successful POST request. + By default the document binary data representing the document itself is returned. If the request includes + and Accept=application/json header then the DRS response is JSON with a storage download URL for the + document accessed by the downloadURL property. + + document_key: DRS document class-DRS ID returned by a previously successful POST request. + return: Flask response. Status codes: + OK: success + Any response code returned from the DRS API, typically NOT_FOUND. + INTERNAL_SERVER_ERROR: default when any exception. + """ try: - response = MinioService.get_file(document_key) - return current_app.response_class( - response=response.data, - status=response.status, - mimetype="application/pdf" - ) + doc_class = get_drs_class_from_key(document_key) + drs_id = get_drs_id_from_key(document_key) + accept = request.headers.get(HEADER_ACCEPT) + doc_binary = accept is None or accept != doc_service.APP_JSON + response = doc_service.get_document(drs_id, doc_class, doc_binary) + if response.ok: + if doc_binary: + return current_app.response_class( + response=response.content, + status=response.status_code, + mimetype="application/pdf" + ) + return jsonify(doc_service.get_content(response)), HTTPStatus.OK + api_error = doc_service.get_content(response) + current_app.logger.error(f"Error getting file {document_key}: {api_error}") + return jsonify( + message=f"Error getting file {document_key}." + ), response.status_code except Exception as e: current_app.logger.error(f"Error getting file {document_key}: {e}") return jsonify( message=f"Error getting file {document_key}." ), HTTPStatus.INTERNAL_SERVER_ERROR + + +@bp.route("/client/", methods=["PUT"]) +@cross_origin() +@jwt.requires_auth +def replace_client_document(document_key): + """ + Replace a client document by document key returned from a previously successful POST request. + The filing must be in a draft/incomplete state. + + document_key: DRS document class-DRS ID returned by a previously successful POST request. + return: Flask response. Status codes: + OK: success + FORBIDDEN: Filing is not in a state where an update is allowed. + INTERNAL_SERVER_ERROR: default when any exception. + Any response code returned from the DRS API. + """ + try: + if is_draft_filing(document_key): + bus_doc: Document = Document(file_key=document_key) + response = doc_service.add_replace_document(bus_doc, request.get_data()) + if response.ok: + return jsonify({"message": f"Document {document_key} replaced successfully."}), HTTPStatus.OK + api_error = doc_service.get_content(response) + current_app.logger.error(f"Error replacing file {document_key}: {api_error}") + return jsonify( + message=f"Error replacing file {document_key}." + ), response.status_code + return jsonify({"message": "Cannot replace when filing is not a draft."}), HTTPStatus.FORBIDDEN + except Exception as e: + current_app.logger.error(f"Error replacing file {document_key}: {e}") + return jsonify( + message=f"Error adding file {document_key}." + ), HTTPStatus.INTERNAL_SERVER_ERROR + + +@bp.route("/client/", methods=["PATCH"]) +@cross_origin() +@jwt.requires_auth +def update_client_document_info(document_key): + """ + Update information associated with a client document by document key returned from a previously successful POST + request. Typically update after a filing completes with the business identifier, filing date, and filing identifier + (filings table record id). The expected request payload typically includes: + filingId: the filings record ID. + filingDate: the filings record filing date/timestamp in the ISO format. + businessIdentifier: the businesses record identifier. + + document_key: DRS document class-DRS ID returned by a previously successful POST request. + return: Flask response. Status codes: + OK: success + INTERNAL_SERVER_ERROR: default when any exception. + Any response code returned from the DRS API. + """ + try: + bus_doc: Document = Document(file_key=document_key) + request_json: dict = request.get_json() + update_payload: dict = {} + if request_json.get(PARAM_FILEDATE): + update_payload[DRS_PROPERTY_MAPPING.get(PARAM_FILEDATE)] = request_json.get(PARAM_FILEDATE) + if request_json.get(PARAM_IDENTIFIER): + update_payload[DRS_PROPERTY_MAPPING.get(PARAM_IDENTIFIER)] = request_json.get(PARAM_IDENTIFIER) + if request_json.get(PARAM_FILING_ID): + update_payload[DRS_PROPERTY_MAPPING.get(PARAM_FILING_ID)] = request_json.get(PARAM_FILING_ID) + if request_json.get(PARAM_FILENAME): + update_payload[DRS_PROPERTY_MAPPING.get(PARAM_FILENAME)] = request_json.get(PARAM_FILENAME) + current_app.logger.info(f"update doc info mapping\n{request_json}\nto\n{update_payload}") + response = doc_service.update_document_record(bus_doc, update_payload) + if response.ok: + return jsonify({"message": f"Document record {document_key} updated successfully."}), HTTPStatus.OK + api_error = doc_service.get_content(response) + current_app.logger.error(f"Error updating document record {document_key}: {api_error}") + return jsonify( + message=f"Error updating document record {document_key}." + ), response.status_code + except Exception as e: + current_app.logger.error(f"Error updating document record {document_key}: {e}") + return jsonify( + message=f"Error updating document record {document_key}." + ), HTTPStatus.INTERNAL_SERVER_ERROR + + +def get_drs_id_from_key(document_key: str) -> str: + """ + Extract the DRS identifier GUID from the business document file key. + + document_key: Existing documents table record containing the DRS identifier as the file_key value. + return: The DRS identifier. + """ + tokens = document_key.split("-") + if tokens and len(tokens) > 1: + return tokens[1] + return document_key + + +def get_drs_class_from_key(document_key: str) -> str: + """ + Extract the DRS document class from the business document file key. + + document_key: Existing documents table record containing the DRS identifier as the file_key value. + return: The DRS document class or DOCUMENT_CLASS_DEFAULT. + """ + tokens = document_key.split("-") + if tokens and len(tokens) > 1: + return tokens[0] + return DOCUMENT_CLASS_DEFAULT + + +def build_create_info(filing_type: str, entity_type: str, document_type: str, request_params: dict) -> dict: + """ + Build the DRS create document information from the request indcluding document class and document type. + + return: The document service request information. + """ + info: dict = { + "documentClass": DRS_MAPPING_DOC_CLASS.get(entity_type, DOCUMENT_CLASS_DEFAULT) + } + doc_type = DOCUMENT_TYPE_DEFAULT + if DRS_MAPPING_DOC_TYPE.get(filing_type) and DRS_MAPPING_DOC_TYPE[filing_type]["documentTypes"].get(document_type): + doc_type = DRS_MAPPING_DOC_TYPE[filing_type]["documentTypes"].get(document_type) + info["documentType"] = doc_type + if request_params.get(PARAM_IDENTIFIER): + info[DRS_PROPERTY_MAPPING.get(PARAM_IDENTIFIER)] = request_params.get(PARAM_IDENTIFIER) + if request_params.get(PARAM_FILENAME): + info[DRS_PROPERTY_MAPPING.get(PARAM_FILENAME)] = request_params.get(PARAM_FILENAME) + if request_params.get(PARAM_FILING_ID): + info[DRS_PROPERTY_MAPPING.get(PARAM_FILING_ID)] = str(request_params.get(PARAM_FILING_ID)) + if request_params.get(PARAM_FILEDATE): + info[DRS_PROPERTY_MAPPING.get(PARAM_FILEDATE)] = request_params.get(PARAM_FILEDATE) + return info diff --git a/legal-api/src/legal_api/services/__init__.py b/legal-api/src/legal_api/services/__init__.py index 51f2ea4f77..1c3a85820a 100644 --- a/legal-api/src/legal_api/services/__init__.py +++ b/legal-api/src/legal_api/services/__init__.py @@ -38,7 +38,6 @@ from .business_details_version import VersionedBusinessDetailsService from .colin import ColinService from .furnishing_documents_service import FurnishingDocumentsService -from .minio import MinioService from .mras_service import MrasService from .naics import NaicsService from .namex import NameXService @@ -71,7 +70,6 @@ "DigitalCredentialsRulesService", "Flags", "FurnishingDocumentsService", - "MinioService", "MrasService", "NaicsService", "NameXService", diff --git a/legal-api/src/legal_api/services/authz.py b/legal-api/src/legal_api/services/authz.py index ddd2f2f215..12cd07dbad 100644 --- a/legal-api/src/legal_api/services/authz.py +++ b/legal-api/src/legal_api/services/authz.py @@ -33,7 +33,7 @@ are_digital_credentials_allowed, get_digital_credentials_preconditions, ) -from legal_api.services.request_context import get_request_context +from legal_api.services.request_context import add_account_linking_key_header, get_request_context from legal_api.services.warnings.business.business_checks import WarningType SYSTEM_ROLE = "system" @@ -86,12 +86,14 @@ def _call_auth_api(path: str, token: str) -> Response: auth_url += path headers = {"Authorization": "Bearer " + token} + add_account_linking_key_header(headers) try: http = Session() retries = Retry(total=5, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504]) http.mount("http://", HTTPAdapter(max_retries=retries)) + http.mount("https://", HTTPAdapter(max_retries=retries)) resp = http.get(url=auth_url, headers=headers) current_app.logger.debug(f"Auth get {path} response status: {resp.status_code!s}") return resp @@ -155,6 +157,14 @@ def has_roles(jwt: JwtManager, roles: list[str]) -> bool: return bool(jwt.validate_roles(request_ctx.current_user, roles)) +def has_any_roles(jwt: JwtManager, roles: list[str]) -> bool: + """Assert the users JWT has any of the required role(s). + + Assumes the JWT is already validated. + """ + return any(bool(jwt.validate_roles(request_ctx.current_user, [role])) for role in roles) + + def get_allowable_filings_dict(is_authorization: bool = False): """Return dictionary containing rules for when filings are allowed.""" # importing here to avoid circular dependencies @@ -439,6 +449,13 @@ def get_allowable_filings_dict(is_authorization: bool = False): } }, Business.State.HISTORICAL: { + "correction": { + "legalTypes": ["BEN", "BC", "ULC", "CC", "C", "CBEN", "CUL", "CCC"], + "blockerChecks": { + "warningTypes": [WarningType.MISSING_REQUIRED_BUSINESS_INFO], + "business": [BusinessBlocker.DEFAULT] + } + }, "courtOrder": { "legalTypes": ["SP", "GP", "CP", "BC", "BEN", "CC", "ULC", "C", "CBEN", "CUL", "CCC"], }, diff --git a/legal-api/src/legal_api/services/colin.py b/legal-api/src/legal_api/services/colin.py index 7578700018..4166f96a3e 100644 --- a/legal-api/src/legal_api/services/colin.py +++ b/legal-api/src/legal_api/services/colin.py @@ -13,25 +13,99 @@ # limitations under the License. """This provides the service for colin-api calls.""" -import requests +from http import HTTPStatus + from flask import current_app +from requests import Response, Session, exceptions +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry from business_account import AccountService +from legal_api.services.cache import cache + + +def get_colin_cache_key(path: str, token: str | None = None, use_cache = True) -> str: + """Return the cache key for a colin call.""" + return f"colin-{path}-{token}" + + +def skip_colin_cache(func, path, token: str | None = None, use_cache = True) -> bool: + """Bypass the cache entirely (no read, no write) when the caller opts out.""" + return not use_cache + + +def is_cacheable_response(result: tuple) -> bool: + """Cache only definitive answers - never a failed call, a COLIN 5xx or an OK with no JSON body.""" + json_data, status_code = result + if status_code is None or status_code >= HTTPStatus.INTERNAL_SERVER_ERROR: + return False + return not (status_code == HTTPStatus.OK and json_data is None) + + +def safe_json(response: Response | None) -> dict | None: + """Return the response body as json, or None when there is no valid JSON body.""" + if response is None: + return None + try: + return response.json() + except ValueError: + return None class ColinService: """Provides services to use the colin-api.""" @staticmethod - def query_business(identifier: str): - """Return a JSON object with business information.""" - # Perform proxy call with identifier - url = f'{current_app.config["COLIN_URL"]}/businesses/{identifier}/public' - token = AccountService.get_bearer_token() - headers = { - "Content-Type": "application/json", - "Authorization": "Bearer " + token - } - response = requests.get(url, headers=headers) - - return response + @cache.cached(make_cache_key=get_colin_cache_key, unless=skip_colin_cache, response_filter=is_cacheable_response) + def call_colin_api(path: str, token: str | None = None, use_cache = True) -> tuple[dict | None, int | None]: + """Return (json body, status code) of the colin api response for the given endpoint path. + + Returns (None, None) when the call could not be made or colin was unreachable. + """ + current_app.logger.debug(f"Colin get {path}...") + timeout = current_app.config.get("COLIN_TIMEOUT", 20) + template_url = current_app.config.get("COLIN_URL") + if not template_url: + current_app.logger.error(f"Colin call to {path} failed: COLIN_URL is not configured") + return None, None + colin_url = template_url + "/" if template_url[-1] != "/" else template_url + colin_url += path + + try: + token = token or AccountService.get_bearer_token() + if not token: + current_app.logger.error(f"Colin call to {path} failed: no service token") + return None, None + + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer " + token + } + http = Session() + retries = Retry(total=5, + backoff_factor=0.1, + status_forcelist=[500, 502, 503, 504]) + http.mount("http://", HTTPAdapter(max_retries=retries)) + http.mount("https://", HTTPAdapter(max_retries=retries)) + resp = http.get(url=colin_url, headers=headers, timeout=timeout) + current_app.logger.debug(f"Colin get {path} response status: {resp.status_code!s}") + if not resp.ok and resp.status_code != HTTPStatus.NOT_FOUND: + current_app.logger.error("%s call failed with status %s", path, resp.status_code) + return safe_json(resp), resp.status_code + + except (exceptions.ConnectionError, + exceptions.Timeout, + Exception) as err: + current_app.logger.debug(err.with_traceback(None)) + current_app.logger.error(f"Colin connection failure, url: {colin_url}") + return None, None + + @staticmethod + def query_business(identifier: str, use_cache: bool = True) -> tuple[dict | None, int | None]: + """Return (json, status code) of the business information.""" + return ColinService.call_colin_api(f"businesses/{identifier}/public", use_cache=use_cache) + + @staticmethod + def get_snapshot(identifier: str, use_cache: bool = True) -> tuple[dict | None, int | None]: + """Return (json, status code) of the LEAR-shaped snapshot of a COLIN business.""" + return ColinService.call_colin_api(f"businesses/{identifier}/snapshot", use_cache=use_cache) diff --git a/legal-api/src/legal_api/services/doc_service.py b/legal-api/src/legal_api/services/doc_service.py new file mode 100644 index 0000000000..4a22ba3cdb --- /dev/null +++ b/legal-api/src/legal_api/services/doc_service.py @@ -0,0 +1,269 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""This manages all of the BC Registries document record service DRS integration for client submitted documents.""" +import json + +import requests +from flask import current_app + +from business_account import AccountService +from business_model.models import Document + +POST_DOCUMENT_PATH: str = "{url}/documents/{document_class}/{document_type}" +GET_DOCUMENT_PATH: str = "{url}/searches/{document_class}?documentServiceId={drs_id}" +DELETE_DOCUMENT_PATH: str = "{url}/documents/{drs_id}" +PATCH_DOCUMENT_PATH: str = "{url}/documents/{drs_id}" +PUT_DOCUMENT_PATH: str = "{url}/documents/{drs_id}" +BUSINESS_API_ACCOUNT_ID: str = "business-api" +APP_JSON = "application/json" +APP_PDF = "application/pdf" +BEARER = "Bearer " +SERVICE_TIMEOUT = 30.0 +DOC_PATH = "/documents" +# Example with all optional document properties relevant to the Business API +DOCUMENT_PROPERTIES = { + "documentClass": "", + "documentType": "", + "consumerDocumentId": "", + "consumerFilename": "", + "consumerIdentifier": "", + "consumerReferenceId": "", + "consumerFilingDate": "", + "description": "", +} + + +def create_document(doc_info: dict, doc_data): + """ + Create a new document record for the doc_data document. Expected doc_info properties: + documentClass: required + documentType: required + consumerFilename: optional + consumerIdentifier: optional + consumerReferenceId: optional + consumerFilingDate: optional + + DRS API reference https://okagqp-test-bcregrestricted.apigee.io/docs/docserviceproxy/1/overview + POST /doc/api/v1/documents/{documentClass}/{documentType} + + doc_info: contains the document record information associated with the document. + doc_data: the document binary data to add to or replace in the DRS for an existing document record. + return: The DRS response as json, including the DRS unique identifier for the document record. + """ + doc_class = doc_info.get("documentClass") + doc_type = doc_info.get("documentType") + if not doc_class or not doc_type: + current_app.logger.info("DRS create_document aborted: no document class or no document type.") + return None + headers = _get_request_headers(APP_PDF) + url = build_create_doc_url(doc_class, doc_type, doc_info) + current_app.logger.info(f"DRS create_document url={url}") + response = requests.post(url=url, headers=headers, timeout=SERVICE_TIMEOUT, data=doc_data) + current_app.logger.info(f"DRS post call {url} status={response.status_code}") + if not response.ok: + current_app.logger.error(f"DRS post call {url} response={response.content}") + return response + + +def get_document(drs_id: str, doc_class: str, doc_binary: bool = True): + """ + Get a previously saved document from the document service by unique DRS identifier and document class. + + DRS API reference https://okagqp-test-bcregrestricted.apigee.io/docs/docserviceproxy/1/overview + GET doc/api/v1/searches/{documentClass}?documentServiceId={docServiceId} + + drs_id: The unique DRS identifier for the requested document. + document_class: The DRS document class for the business to filter on. + doc_binary: True if the response is the pdf binary data. False if the response is JSON with a storage download link. + return: Response containing the document binary data or json depending on the doc_binary value. + """ + accept = APP_JSON if not doc_binary else APP_PDF + headers = _get_request_headers(APP_JSON, accept) + url = GET_DOCUMENT_PATH.format( + url=str(current_app.config.get("DOCUMENT_SVC_URL")).replace(DOC_PATH, ""), + document_class=doc_class, + drs_id=drs_id + ) + current_app.logger.info(f"DRS get_document url={url}") + response = requests.get(url=url, headers=headers, timeout=SERVICE_TIMEOUT) + if not response.ok: + current_app.logger.error(f"DRS call {url} failed status={response.status_code}: {response.content}") + return response + + +def delete_document(bus_document: Document): + """ + Permanently delete a document record from the document service by unique DRS identifier. Record deletion + is only allowed with existing DRS identifiers that exist in the documents table. + + DRS API reference https://okagqp-test-bcregrestricted.apigee.io/docs/docserviceproxy/1/overview + DELETE /doc/api/v1/documents/{docServiceId} + + bus_document: Existing documents table record containing the DRS identifier as the file_key value. + return: The DRS response, or None if no documents record file key exists. + """ + drs_id = get_drs_id(bus_document) + if not drs_id: + current_app.logger.info("DRS delete_document aborted: no document file_key.") + return None + headers = _get_request_headers(APP_JSON) + url = DELETE_DOCUMENT_PATH.format( + url=str(current_app.config.get("DOCUMENT_SVC_URL")).replace(DOC_PATH, ""), + drs_id=drs_id + ) + current_app.logger.info(f"DRS delete_document url={url}") + response = requests.delete(url=url, headers=headers, timeout=SERVICE_TIMEOUT) + current_app.logger.info(f"DRS delete call {url} status={response.status_code}") + if not response.ok: + current_app.logger.error(f"DRS delete call {url} response={response.content}") + return response + + +def update_document_record(bus_document: Document, update_info: dict): + """ + Update document record properties by unique DRS identifier. Record changes + are only allowed with existing DRS identifiers that exist in the documents table. + Updates from the filer: + consumerReferenceId: the filings record ID. + consumerFilingDate: the filings record filing date. + consumerIdentifier: the businesses record identifier. + + DRS API reference https://okagqp-test-bcregrestricted.apigee.io/docs/docserviceproxy/1/overview + PATCH /doc/api/v1/documents/{docServiceId} + + bus_document: Existing documents table record containing the DRS identifier as the file_key value. + return: The DRS response, or None if no documents record file key exists. + """ + drs_id = get_drs_id(bus_document) + if not drs_id: + current_app.logger.info("DRS update_document_record aborted: no document file_key.") + return None + headers = _get_request_headers(APP_JSON) + url = PATCH_DOCUMENT_PATH.format( + url=str(current_app.config.get("DOCUMENT_SVC_URL")).replace(DOC_PATH, ""), + drs_id=drs_id + ) + if update_info.get("consumerReferenceId"): + update_info["consumerReferenceId"] = str(update_info["consumerReferenceId"]) + current_app.logger.info(f"DRS update_document_record url={url}") + response = requests.patch(url=url, headers=headers, timeout=SERVICE_TIMEOUT, json=update_info) + current_app.logger.info(f"DRS patch call {url} status={response.status_code}") + if not response.ok: + current_app.logger.error(f"DRS patch call {url} response={response.content}") + return response + + +def add_replace_document(bus_document: Document, doc_data): + """ + Add or replace a document associated with an existing document record identified by the unique DRS identifier. + + DRS API reference https://okagqp-test-bcregrestricted.apigee.io/docs/docserviceproxy/1/overview + PUT /doc/api/v1/documents/{docServiceId} + + bus_document: Existing documents table record containing the DRS identifier as the file_key value. + doc_data: the document binary data to add to or replace in the DRS for an existing document record. + return: The DRS response, or None if no documents record file key exists. + """ + drs_id = get_drs_id(bus_document) + if not drs_id: + current_app.logger.info("DRS add_replace_document aborted: no document file_key.") + return None + headers = _get_request_headers(APP_PDF) + url = PUT_DOCUMENT_PATH.format( + url=str(current_app.config.get("DOCUMENT_SVC_URL")).replace(DOC_PATH, ""), + drs_id=drs_id + ) + current_app.logger.info(f"DRS add_replace_document url={url}") + response = requests.put(url=url, headers=headers, timeout=SERVICE_TIMEOUT, data=doc_data) + current_app.logger.info(f"DRS put call {url} status={response.status_code}") + if not response.ok: + current_app.logger.error(f"DRS put call {url} response={response.content}") + return response + + +def build_create_doc_url(doc_class: str, doc_type: str, doc_info: dict) -> str: + """ + Build the url for the DRS API create document record request. + + doc_class: New record document class. + doc_type: New record document type. + doc_info: All other document record extra information. + return: The create record request request url. + """ + url = POST_DOCUMENT_PATH.format( + url=str(current_app.config.get("DOCUMENT_SVC_URL")).replace(DOC_PATH, ""), + document_class=doc_class, + document_type=doc_type + ) + request_params: str = "" + if doc_info.get("consumerFilename"): + request_params += f"&consumerFilename={doc_info['consumerFilename']}" + if doc_info.get("consumerIdentifier"): + request_params += f"&consumerIdentifier={doc_info['consumerIdentifier']}" + if doc_info.get("consumerReferenceId"): + request_params += f"&consumerReferenceId={doc_info['consumerReferenceId']}" + if doc_info.get("consumerFilingDate"): + request_params += f"&consumerFilingDate={doc_info['consumerFilingDate']}" + if request_params != "": + url += "?" + request_params[1:] + return url + + +def get_drs_id(bus_document: Document): + """ + Extract the DRS identifier GUID from the business document file key. + + bus_document: Existing documents table record containing the DRS identifier as the file_key value. + return: The DRS identifier if available. + """ + filekey: str = bus_document.file_key if bus_document else None + if not filekey or filekey.startswith("DS"): + return filekey + tokens = filekey.split("-") + if tokens and len(tokens) > 1: + filekey = tokens[1] + return filekey + + +def _get_request_headers(content_type, accept_mime_type = None) -> dict: + """ + Get request headers for the DRS api call. + + content_type: The request header Content-Type value to use. + accept_mime_type: The request header Accept value to use. + return: The request headers. + """ + headers = { + "Account-Id": BUSINESS_API_ACCOUNT_ID, + "Content-Type": content_type + } + if current_app.config.get("ACCOUNT_SVC_CLIENT_SECRET"): + token = AccountService.get_bearer_token() + headers["Authorization"] = BEARER + token + if apikey := current_app.config.get("DOCUMENT_API_KEY"): + headers["x-apikey"] = apikey + if accept_mime_type: + headers["Accept"] = accept_mime_type + return headers + + +def get_content(response): + """Get the content of the response useful for test methods.""" + c = response.content + try: + c = c.decode() + c = json.loads(c) + except Exception: + pass + return c diff --git a/legal-api/src/legal_api/services/filings/validations/agm_extension.py b/legal-api/src/legal_api/services/filings/validations/agm_extension.py index ccf05fd981..2e8c39a611 100644 --- a/legal-api/src/legal_api/services/filings/validations/agm_extension.py +++ b/legal-api/src/legal_api/services/filings/validations/agm_extension.py @@ -40,6 +40,22 @@ def validate(business: Business, filing: dict) -> Error | None: [{"error": babel(f"{business.legal_type} does not support agm extension filing.")}]) msg = [] + # A four-digit year is enforced by the schema; only the business-rule year range is checked here. + agm_year_path: Final = "/filing/agmExtension/year" + agm_year = get_int(filing, agm_year_path) + + if agm_year: + founding_year = ( + LegislationDatetime.as_legislation_timezone(business.founding_date).year + if business.founding_date else None + ) + + if founding_year and agm_year < founding_year: + msg.append({ + "error": "AGM year cannot be earlier than the incorporation year.", + "path": agm_year_path + }) + is_first_agm = get_bool(filing, f"{AGM_EXTENSION_PATH}/isFirstAgm") if is_first_agm: diff --git a/legal-api/src/legal_api/services/filings/validations/agm_location_change.py b/legal-api/src/legal_api/services/filings/validations/agm_location_change.py index bb99f2c87a..bf3ea374a2 100644 --- a/legal-api/src/legal_api/services/filings/validations/agm_location_change.py +++ b/legal-api/src/legal_api/services/filings/validations/agm_location_change.py @@ -41,10 +41,16 @@ def validate(business: Business, filing: dict) -> Error | None: agm_year_path: Final = "/filing/agmLocationChange/year" year = get_int(filing, agm_year_path) if year: - expected_min = LegislationDatetime.now().year - 2 - expected_max = LegislationDatetime.now().year + 1 + current_year = LegislationDatetime.now().year + # AGM year can never be before the year the business was founded, even if that is + # more recent than the default 2-year lookback window. + founding_year = LegislationDatetime.as_legislation_timezone(business.founding_date).year \ + if business.founding_date else current_year - 2 + expected_min = max(current_year - 2, founding_year) + expected_max = current_year + 1 if expected_min > year or year > expected_max: - msg.append({"error": "AGM year must be between -2 or +1 year from current year.", "path": agm_year_path}) + msg.append({"error": f"AGM year must be between {expected_min} and {expected_max}.", + "path": agm_year_path}) # A non-empty reason (at least one non-whitespace character) is enforced by the schema # (business-schemas agm_location_change reason pattern). diff --git a/legal-api/src/legal_api/services/filings/validations/alteration.py b/legal-api/src/legal_api/services/filings/validations/alteration.py index 2061c10553..8ce3403f83 100644 --- a/legal-api/src/legal_api/services/filings/validations/alteration.py +++ b/legal-api/src/legal_api/services/filings/validations/alteration.py @@ -46,9 +46,11 @@ def validate(business: Business, filing: dict) -> Error: # pylint: disable=too- error = PermissionService.check_user_permission(required_permission, message=message) if error: return error - msg.extend(type_change_validation(filing, business)) - msg.extend(company_name_validation(filing, business)) - msg.extend(share_structure_validation(filing, business)) + new_legal_type_path: Final = "/filing/alteration/business/legalType" + new_legal_type = get_str(filing, new_legal_type_path) + msg.extend(validate_type_change(filing, business, new_legal_type_path)) + msg.extend(company_name_validation(filing, business, new_legal_type)) + msg.extend(share_structure_validation(filing, business, new_legal_type)) msg.extend(court_order_validation(filing)) msg.extend(rules_change_validation(filing)) msg.extend(memorandum_change_validation(filing)) @@ -74,16 +76,13 @@ def court_order_validation(filing): """Validate court order.""" court_order_path: Final = "/filing/alteration/courtOrder" if get_str(filing, court_order_path): - err = validate_court_order(court_order_path, filing["filing"]["alteration"]["courtOrder"]) - if err: - return err + return validate_court_order(court_order_path, filing["filing"]["alteration"]["courtOrder"]) return [] -def share_structure_validation(filing, business: Business): +def share_structure_validation(filing, business: Business, new_legal_type: str): """Validate share structure.""" share_structure_path: Final = "/filing/alteration/shareStructure" - new_legal_type = get_str(filing, "/filing/alteration/business/legalType") if get_str(filing, share_structure_path): err = validate_share_structure(filing, "alteration", new_legal_type or business.legal_type) @@ -95,13 +94,12 @@ def share_structure_validation(filing, business: Business): return [] -def company_name_validation(filing, business: Business): +def company_name_validation(filing, business: Business, new_legal_type: str): """Validate company name.""" msg = [] if filing["filing"]["header"].get("source") == "COLIN": return msg - new_legal_type = get_str(filing, "/filing/alteration/business/legalType") if get_str(filing, "/filing/alteration/nameRequest/nrNumber"): accepted_request_types = [ "BEC", "CCC", "CCP", "CCR", "CUL", # name change types @@ -126,10 +124,9 @@ def company_name_validation(filing, business: Business): return msg -def type_change_validation(filing, business: Business): +def validate_type_change(filing, business: Business, legal_type_path: str): """Validate type change.""" msg = [] - legal_type_path: Final = "/filing/alteration/business/legalType" new_legal_type = get_str(filing, legal_type_path) # Valid type changes @@ -208,15 +205,10 @@ def rules_change_validation(filing): error_msg = "Cannot provide both file upload and rules change in SR" msg.append({"error": babel(error_msg), "path": rules_file_key_path + " and " + rules_change_in_sr_path}) - return msg - - if rules_file_key: - rules_err = validate_pdf(rules_file_key, rules_file_key_path) - if rules_err: - msg.extend(rules_err) - return msg + elif rules_file_key: + msg.extend(validate_pdf(rules_file_key, rules_file_key_path)) - return [] + return msg def memorandum_change_validation(filing): @@ -232,11 +224,7 @@ def memorandum_change_validation(filing): error_msg = "Cannot provide both file upload and memorandum change in SR" msg.append({"error": babel(error_msg), "path": memorandum_file_key + " and " + memorandum_change_in_sr_path}) - return msg - - if memorandum_file_key: - memorandum_err = validate_pdf(memorandum_file_key, memorandum_file_key_path) - if memorandum_err: - msg.extend(memorandum_err) + elif memorandum_file_key: + msg.extend(validate_pdf(memorandum_file_key, memorandum_file_key_path)) return msg diff --git a/legal-api/src/legal_api/services/filings/validations/amalgamation_application.py b/legal-api/src/legal_api/services/filings/validations/amalgamation_application.py index 50915e64e2..902b4ae7bb 100644 --- a/legal-api/src/legal_api/services/filings/validations/amalgamation_application.py +++ b/legal-api/src/legal_api/services/filings/validations/amalgamation_application.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Validation for the Amalgamation Application filing.""" +from datetime import UTC, datetime from http import HTTPStatus from typing import Final @@ -19,9 +20,9 @@ from flask_babel import _ as babel from business_account import AccountService -from business_model.models import AmalgamatingBusiness, Amalgamation, Business, Filing, PartyRole +from business_model.models import AmalgamatingBusiness, Amalgamation, Business, Filing, OfficeType, PartyRole from legal_api.errors import Error -from legal_api.services import STAFF_ROLE, flags +from legal_api.services import STAFF_ROLE, colin, flags from legal_api.services.filings.validations.common_validations import ( validate_court_order, validate_effective_date, @@ -43,6 +44,13 @@ from legal_api.services.utils import get_str from legal_api.utils.auth import jwt +# COLIN corps not loaded in LEAR that can amalgamate +COLIN_AMALGAMATION_LEGAL_TYPES: Final = [ + Business.LegalTypes.COMP.value, + Business.LegalTypes.BC_ULC_COMPANY.value, + Business.LegalTypes.BC_CCC.value, +] + def validate(amalgamation_json: dict, account_id) -> Error | None: """Validate the Amalgamation Application filing.""" @@ -78,19 +86,13 @@ def validate(amalgamation_json: dict, account_id) -> Error | None: msg.extend(validate_parties_names(amalgamation_json, filing_type, legal_type)) msg.extend(validate_parties_addresses(amalgamation_json, filing_type)) + structural_errors = _structural_validation_errors(amalgamation_json, legal_type, filing_type) if amalgamation_type == Amalgamation.AmalgamationTypes.regular.name: - msg.extend(validate_offices(amalgamation_json, legal_type, filing_type)) - msg.extend(validate_offices_addresses(amalgamation_json, filing_type)) - err = validate_share_structure(amalgamation_json, filing_type, legal_type) - if err: - msg.extend(err) - err = validate_share_currency(amalgamation_json, filing_type) - if err: - msg.extend(err) - - err = validate_parties_delivery_address(amalgamation_json, legal_type, filing_type) - if err: - msg.extend(err) + msg.extend(structural_errors) + else: + # short-form: the filing must mirror the primary/holding business's current data + msg.extend(validate_primary_or_holding_match(amalgamation_json, filing_type, amalgamation_type)) + msg.extend(_augment_with_source_update_hint(structural_errors, amalgamation_type)) msg.extend(validate_amalgamation_court_order(amalgamation_json, filing_type)) msg.extend(validate_amalgamating_businesses(amalgamation_json, @@ -126,6 +128,43 @@ def _validate_name_request_section(amalgamation_json, amalgamation_type, legal_t return validate_name_request(amalgamation_json, legal_type, filing_type) +def _structural_validation_errors(amalgamation_json, legal_type, filing_type) -> list: + """Validate the office, share structure and party address data. + + Sections are gated on presence: short-form filings are not schema-required to carry + them (a missing section gets its own required-section error from the match validation). + """ + filing_section = amalgamation_json.get("filing", {}).get(filing_type, {}) + msg = [] + if "offices" in filing_section: + msg.extend(validate_offices(amalgamation_json, legal_type, filing_type)) + msg.extend(validate_offices_addresses(amalgamation_json, filing_type)) + if "shareStructure" in filing_section: + if err := validate_share_structure(amalgamation_json, filing_type, legal_type): + msg.extend(err) + if err := validate_share_currency(amalgamation_json, filing_type): + msg.extend(err) + if err := validate_parties_delivery_address(amalgamation_json, legal_type, filing_type): + msg.extend(err) + return msg + + +def _primary_or_holding_role(amalgamation_type: str) -> str: + """Return the TED-defining role for a short-form amalgamation type.""" + return (AmalgamatingBusiness.Role.primary.name + if amalgamation_type == Amalgamation.AmalgamationTypes.horizontal.name + else AmalgamatingBusiness.Role.holding.name) + + +def _augment_with_source_update_hint(errors: list, amalgamation_type: str) -> list: + """Point short-form structural problems at the source business - they cannot be fixed in this filing.""" + role = _primary_or_holding_role(amalgamation_type) + for error in errors: + error["error"] += (f" This data comes from the {role} business - its information must be " + "corrected outside of this filing before this amalgamation can be filed.") + return errors + + def validate_amalgamating_businesses( # noqa: PLR0912, PLR0915 amalgamation_json, filing_type, @@ -154,13 +193,15 @@ def validate_amalgamating_businesses( # noqa: PLR0912, PLR0915 business_identifiers = [] duplicate_businesses = [] adoptable_names = [] - primary_or_holding_business = None + primary_or_holding_legal_type = None amalgamating_business_roles = { AmalgamatingBusiness.Role.amalgamating.name: 0, AmalgamatingBusiness.Role.holding.name: 0, AmalgamatingBusiness.Role.primary.name: 0 } amalgamating_businesses = {} + colin_businesses = {} + colin_fetch_failures = [] # collect data for validation for amalgamating_business_json in amalgamating_businesses_json: @@ -178,14 +219,28 @@ def validate_amalgamating_businesses( # noqa: PLR0912, PLR0915 if (identifier.startswith("A") and foreign_jurisdiction.get("country") == "CA" and foreign_jurisdiction.get("region") == "BC"): is_any_expro_a = True - elif business := Business.find_by_identifier(identifier): + continue + + if business := Business.find_by_identifier(identifier): amalgamating_businesses[identifier] = business - is_any_business[business.legal_type] = True - if legal_type == business.legal_type: - adoptable_names.append(business.legal_name) - if amalgamating_business_json["role"] in [AmalgamatingBusiness.Role.primary.name, - AmalgamatingBusiness.Role.holding.name]: - primary_or_holding_business = business + ting_legal_type, ting_legal_name = business.legal_type, business.legal_name + else: + # not foreign and not in LEAR - maybe a COLIN corp that can amalgamate + colin_business, colin_fetch_failed = _find_colin_business(identifier) + if not colin_business: + if colin_fetch_failed: + colin_fetch_failures.append(identifier) + continue + colin_businesses[identifier] = colin_business + ting_legal_type, ting_legal_name = colin_business["legalType"], colin_business["legalName"] + + # aggregate bookkeeping (CC/ULC mixes, name adoption, type match) shared by LEAR and COLIN TINGs + is_any_business[ting_legal_type] = True + if legal_type == ting_legal_type: + adoptable_names.append(ting_legal_name) + if amalgamating_business_json["role"] in [AmalgamatingBusiness.Role.primary.name, + AmalgamatingBusiness.Role.holding.name]: + primary_or_holding_legal_type = ting_legal_type is_any_bc_company = (is_any_business[Business.LegalTypes.BCOMP.value] or is_any_business[Business.LegalTypes.COMP.value] or @@ -205,12 +260,30 @@ def validate_amalgamating_businesses( # noqa: PLR0912, PLR0915 f"{amalgamating_businesses_path}/{index}")) else: identifier = amalgamating_business_json.get("identifier") - amalgamating_business = amalgamating_businesses.get(identifier) - msg.extend(_validate_lear_businesses(identifier, - amalgamating_business, - account_id, - is_staff, - f"{amalgamating_businesses_path}/{index}")) + amalgamating_business_path = f"{amalgamating_businesses_path}/{index}" + if amalgamating_business := amalgamating_businesses.get(identifier): + business_info = _business_info_from_lear(amalgamating_business, is_staff) + elif colin_business := colin_businesses.get(identifier): + business_info = colin_business + elif identifier in colin_fetch_failures: + # COLIN could not be reached - a clear retryable error, not "not found" + msg.append({ + "error": f"Unable to verify {identifier} - COLIN is unavailable, try again.", + "path": amalgamating_business_path + }) + continue + else: + msg.append({ + "error": f"A business with identifier:{identifier} not found.", + "path": amalgamating_business_path + }) + continue + + msg.extend(_validate_amalgamating_business(identifier, + business_info, + account_id, + is_staff, + amalgamating_business_path)) if duplicate_businesses: msg.append({ @@ -232,15 +305,15 @@ def validate_amalgamating_businesses( # noqa: PLR0912, PLR0915 "path": f"/filing/{filing_type}/nameRequest/legalName" }) - if primary_or_holding_business: + if primary_or_holding_legal_type: continued_types_map = { Business.LegalTypes.CONTINUE_IN.value: Business.LegalTypes.COMP.value, Business.LegalTypes.BCOMP_CONTINUE_IN.value: Business.LegalTypes.BCOMP.value, Business.LegalTypes.ULC_CONTINUE_IN.value: Business.LegalTypes.BC_ULC_COMPANY.value, Business.LegalTypes.CCC_CONTINUE_IN.value: Business.LegalTypes.BC_CCC.value } - legal_type_to_compare = continued_types_map.get(primary_or_holding_business.legal_type, - primary_or_holding_business.legal_type) + legal_type_to_compare = continued_types_map.get(primary_or_holding_legal_type, + primary_or_holding_legal_type) if legal_type_to_compare != legal_type: msg.append({ "error": "Legal type should be same as the legal type in primary or holding business.", @@ -344,65 +417,288 @@ def _check_aml_permission_or_default_error(msg: list, message: str, default_erro return True return False -def _validate_lear_businesses( # pylint: disable=too-many-arguments +def _business_info_from_lear(business: Business, is_staff: bool) -> dict: + """Normalize a LEAR business row to the snapshot shape shared by TING validation.""" + return { + "state": business.state.name if business.state else None, + "hasFutureEffectiveFiling": _has_pending_filing(business), + "adminFreeze": business.admin_freeze, + # good_standing costs a filing-history query and only the non-staff checks read it + "goodStanding": True if is_staff else business.good_standing, + } + + +def _validate_amalgamating_business( # pylint: disable=too-many-arguments identifier, - amalgamating_business, + business_info, account_id, is_staff, amalgamating_business_path) -> list: + """Validate a TING business from its normalized info (LEAR row or COLIN snapshot).""" msg = [] - if amalgamating_business: - if amalgamating_business.state == Business.State.HISTORICAL: - msg.append({ - "error": f"Cannot amalgamate with {identifier} which is in historical state.", - "path": amalgamating_business_path - }) - elif _has_pending_filing(amalgamating_business): - msg.append({ - "error": f"{identifier} has a draft, pending or future effective filing.", - "path": amalgamating_business_path - }) - elif Business.is_pending_amalgamating_business(identifier): - msg.append({ - "error": f"{identifier} is part of a future effective amalgamation filing.", - "path": amalgamating_business_path - }) + if business_info.get("state") == Business.State.HISTORICAL.name: + msg.append({ + "error": f"Cannot amalgamate with {identifier} which is in historical state.", + "path": amalgamating_business_path + }) + elif business_info.get("hasFutureEffectiveFiling"): + msg.append({ + "error": f"{identifier} has a draft, pending or future effective filing.", + "path": amalgamating_business_path + }) + elif Business.is_pending_amalgamating_business(identifier): + msg.append({ + "error": f"{identifier} is part of a future effective amalgamation filing.", + "path": amalgamating_business_path + }) + + if business_info.get("adminFreeze"): + msg.append({ + "error": f"{identifier} is frozen.", + "path": amalgamating_business_path + }) - if not is_staff: - if not _is_business_affliated(identifier, account_id): - error = _check_aml_permission_or_default_error( - msg, - "Permission Denied - You do not have permissions to amalgamate an unaffiliated business.", - { - "error": (f"{identifier} is not affiliated with the currently " - "selected BC Registries account."), - "path": amalgamating_business_path - } - ) - if error: - return msg - - if not amalgamating_business.good_standing: - error = _check_aml_permission_or_default_error( + if not is_staff: + if not _is_business_affliated(identifier, account_id): + error = _check_aml_permission_or_default_error( + msg, + "Permission Denied - You do not have permissions to amalgamate an unaffiliated business.", + { + "error": (f"{identifier} is not affiliated with the currently " + "selected BC Registries account."), + "path": amalgamating_business_path + } + ) + if error: + return msg + + if not business_info.get("goodStanding"): + error = _check_aml_permission_or_default_error( msg, "Permission Denied - You do not have permissions to amalgamate a business not in good standing.", { "error": f"{identifier} is not in good standing.", "path": amalgamating_business_path } - ) - if error: - return msg - + ) + if error: + return msg + + return msg + + +def _find_colin_business(identifier: str) -> tuple: + """Return (colin business dict | None, fetch_failed) for a business not found in LEAR.""" + snapshot_json, status_code = colin.get_snapshot(identifier) + if status_code is None or status_code >= HTTPStatus.INTERNAL_SERVER_ERROR: + return None, True + if status_code != HTTPStatus.OK: + return None, False + if snapshot_json is None: + # a 200 with a non-JSON body is an infrastructure failure - retryable, not "not found" + return None, True + + business_json = snapshot_json.get("business") or {} + if business_json.get("legalType") not in COLIN_AMALGAMATION_LEGAL_TYPES: + return None, False + return business_json, False + + +def validate_primary_or_holding_match(amalgamation_json, filing_type, amalgamation_type) -> list: + """Ensure the filing has the primary/holding business's current data. + + A short-form amalgamation adopts the primary/holding business's legal name, directors, + offices and share structure. The filing must record exactly what will be applied, so the + submitted sections are compared against that business's current data (LEAR row or COLIN + snapshot). A mismatch means the draft is stale and must be refreshed. + """ + filing_section = amalgamation_json.get("filing", {}).get(filing_type, {}) + role = _primary_or_holding_role(amalgamation_type) + source = _find_primary_or_holding_data(filing_section, role) + if source is None: + # a missing or unresolvable primary/holding business is reported by the TING validations + return [] + + msg = [] + section_path = f"/filing/{filing_type}" + refresh_hint = "Refresh the draft with the business's current data." + + if legal_name := filing_section.get("nameRequest", {}).get("legalName"): + if legal_name.strip() != ((source.get("business") or {}).get("legalName") or "").strip(): + msg.append({ + "error": f"Legal name does not match the {role} business. {refresh_hint}", + "path": f"{section_path}/nameRequest/legalName" + }) else: msg.append({ - "error": f"A business with identifier:{identifier} not found.", - "path": amalgamating_business_path + "error": f"Legal name of the {role} business is required for a short-form amalgamation.", + "path": f"{section_path}/nameRequest/legalName" + }) + + if offices := filing_section.get("offices"): + if _norm_offices(offices) != _norm_offices(source.get("offices")): + msg.append({ + "error": f"Offices do not match the {role} business's current offices. {refresh_hint}", + "path": f"{section_path}/offices" + }) + else: + msg.append({ + "error": f"Offices of the {role} business are required for a short-form amalgamation.", + "path": f"{section_path}/offices" + }) + + if share_structure := filing_section.get("shareStructure"): + if _norm_share_classes(share_structure.get("shareClasses")) != _norm_share_classes(source.get("shareClasses")): + msg.append({ + "error": f"Share structure does not match the {role} business's current share structure. " + f"{refresh_hint}", + "path": f"{section_path}/shareStructure" + }) + source_dates = [resolution.get("date") for resolution in source.get("resolutions") or []] + if _norm_resolution_dates(share_structure.get("resolutionDates")) != _norm_resolution_dates(source_dates): + msg.append({ + "error": f"Resolution dates do not match the {role} business's current resolution dates. " + f"{refresh_hint}", + "path": f"{section_path}/shareStructure/resolutionDates" + }) + else: + msg.append({ + "error": f"Share structure of the {role} business is required for a short-form amalgamation.", + "path": f"{section_path}/shareStructure" + }) + + if _norm_directors(filing_section.get("parties")) != _norm_directors(source.get("parties")): + msg.append({ + "error": f"Directors do not match the {role} business's current directors. {refresh_hint}", + "path": f"{section_path}/parties" }) return msg +def _find_primary_or_holding_data(filing_section: dict, role: str) -> dict | None: + """Return the primary/holding business's current data in the same shape as COLIN snapshot.""" + entry = next((business_json for business_json in filing_section.get("amalgamatingBusinesses") or [] + if business_json.get("role") == role and not business_json.get("foreignJurisdiction")), None) + if not (entry and (identifier := entry.get("identifier"))): + return None + + if business := Business.find_by_identifier(identifier): + return _full_business_data_from_lear(business) + + snapshot_json, status_code = colin.get_snapshot(identifier) + if status_code == HTTPStatus.OK: + return snapshot_json or None + return None + + +def _full_business_data_from_lear(business: Business) -> dict: + """Return the business's current snapshot data (what the filer applies for amalgamation).""" + offices = {} + for office in business.offices.all(): + if office.office_type in [OfficeType.REGISTERED, OfficeType.RECORDS]: + offices[office.office_type] = { + f"{address.address_type}Address": address.json for address in office.addresses + } + directors = [] + for party_role in PartyRole.get_active_directors(business.id, datetime.now(UTC).date()): + # party_role.json carries a singular 'role' - reshape to the snapshot's roles list + director = party_role.json + director["roles"] = [{"roleType": "Director"}] + directors.append(director) + return { + "business": {"legalName": business.legal_name}, + "parties": directors, + "offices": offices, + "shareClasses": [share_class.json for share_class in business.share_classes.all()], + "resolutions": [{"date": resolution.resolution_date.isoformat()} for resolution in business.resolutions], + } + + +_ADDRESS_MATCH_FIELDS: Final = ("streetAddress", "streetAddressAdditional", "addressCity", "addressRegion", + "addressCountry", "postalCode", "deliveryInstructions") + + +def _norm_value(value): + """Strip strings and collapse empties so '' and a missing key compare equal.""" + if isinstance(value, str): + value = value.strip() + return value if value not in ("", None) else None + + +def _norm_number(value): + """Coerce numbers so 1, 1.0 and '1' compare equal.""" + try: + return float(value) if value not in ("", None) else None + except (TypeError, ValueError): + return _norm_value(value) + + +def _norm_address(address: dict | None) -> tuple | None: + if not address: + return None + return tuple(_norm_value(address.get(field)) for field in _ADDRESS_MATCH_FIELDS) + + +def _norm_offices(offices: dict | None) -> dict: + return { + office_type: (_norm_address(office.get("deliveryAddress")), _norm_address(office.get("mailingAddress"))) + for office_type, office in (offices or {}).items() + if office_type in [OfficeType.REGISTERED, OfficeType.RECORDS] + } + + +def _norm_directors(parties: list | None) -> set: + directors = set() + for party in parties or []: + if not any((party_role.get("roleType") or "").lower() == PartyRole.RoleTypes.DIRECTOR.value + for party_role in party.get("roles") or []): + continue + officer = party.get("officer") or {} + directors.add(( + _norm_value(officer.get("firstName")), + _norm_value(officer.get("middleInitial")), + _norm_value(officer.get("lastName")), + _norm_value(officer.get("organizationName")), + _norm_address(party.get("deliveryAddress")), + )) + return directors + + +def _norm_share_classes(share_classes: list | None) -> set: + # Can skip: + # - ids (DB artifact) + # - priority (is synthesized from the COLIN class id) + # - currencyAdditional (is only set with currency OTHER) + return { + ( + _norm_value(share_class.get("name")), + bool(share_class.get("hasMaximumShares")), + _norm_number(share_class.get("maxNumberOfShares")), + bool(share_class.get("hasParValue")), + _norm_number(share_class.get("parValue")), + _norm_value(share_class.get("currency")), + bool(share_class.get("hasRightsOrRestrictions")), + tuple(sorted((_norm_share_series(series) for series in share_class.get("series") or []), key=repr)), + ) + for share_class in share_classes or [] + } + + +def _norm_share_series(series: dict) -> tuple: + return ( + _norm_value(series.get("name")), + bool(series.get("hasMaximumShares")), + _norm_number(series.get("maxNumberOfShares")), + bool(series.get("hasRightsOrRestrictions")), + ) + + +def _norm_resolution_dates(dates: list | None) -> set: + """Return the dates as YYYY-MM-DD strings (tolerates datetimes and date objects).""" + return {str(date)[:10] for date in dates or [] if date} + + def _validate_amalgamation_type( # pylint: disable=too-many-arguments amalgamation_type, amalgamating_business_roles, @@ -501,7 +797,5 @@ def validate_amalgamation_court_order(filing: dict, filing_type) -> list: """Validate court order.""" if court_order := filing.get("filing", {}).get(filing_type, {}).get("courtOrder", None): court_order_path: Final = f"/filing/{filing_type}/courtOrder" - err = validate_court_order(court_order_path, court_order) - if err: - return err + return validate_court_order(court_order_path, court_order) return [] diff --git a/legal-api/src/legal_api/services/filings/validations/amalgamation_out.py b/legal-api/src/legal_api/services/filings/validations/amalgamation_out.py index ef7f2e1f1f..e899478c15 100644 --- a/legal-api/src/legal_api/services/filings/validations/amalgamation_out.py +++ b/legal-api/src/legal_api/services/filings/validations/amalgamation_out.py @@ -44,7 +44,7 @@ def validate(business: Business, filing: dict) -> Error | None: is_valid_co_date = True is_valid_foreign_jurisdiction = True - if err := validate_amalgamation_out_date(filing, filing_type): + if err := validate_amalgamation_out_date(filing, f"/filing/{filing_type}/amalgamationOutDate"): msg.extend(err) is_valid_co_date = False @@ -58,9 +58,7 @@ def validate(business: Business, filing: dict) -> Error | None: if court_order := filing.get("filing", {}).get(filing_type, {}).get("courtOrder", None): court_order_path: Final = f"/filing/{filing_type}/courtOrder" - err = validate_court_order(court_order_path, court_order) - if err: - msg.extend(err) + msg.extend(validate_court_order(court_order_path, court_order)) if msg: return Error(HTTPStatus.BAD_REQUEST, msg) @@ -96,10 +94,9 @@ def validate_active_cao(business: Business, filing: dict, filing_type: str) -> l return msg -def validate_amalgamation_out_date(filing: dict, filing_type: str) -> list: +def validate_amalgamation_out_date(filing: dict, amalgamation_out_date_path: str) -> list: """Validate amalgamation out date.""" msg = [] - amalgamation_out_date_path = f"/filing/{filing_type}/amalgamationOutDate" amalgamation_out_date = get_date(filing, amalgamation_out_date_path) now = LegislationDatetime.now().date() diff --git a/legal-api/src/legal_api/services/filings/validations/common_validations.py b/legal-api/src/legal_api/services/filings/validations/common_validations.py index 7a3da66d19..86d979b824 100644 --- a/legal-api/src/legal_api/services/filings/validations/common_validations.py +++ b/legal-api/src/legal_api/services/filings/validations/common_validations.py @@ -21,7 +21,8 @@ from typing import Final import pycountry -from flask import current_app, g, request +from flask import current_app, g, has_request_context, request +from flask.globals import request_ctx from flask_babel import _ from pypdf import PdfReader @@ -29,10 +30,10 @@ from business_common.utils.datetime import date from business_common.utils.datetime import datetime as dt from business_common.utils.legislation_datetime import LegislationDatetime -from business_model.models import Address, Business, PartyRole +from business_model.models import Address, Business, Filing, PartyRole from legal_api.core.filing import Filing as CoreFiling from legal_api.errors import Error -from legal_api.services import MinioService, colin, flags, namex +from legal_api.services import STAFF_ROLE, colin, doc_service, flags, namex from legal_api.services.permissions import ListActionsPermissionsAllowed, PermissionService from legal_api.services.request_context import get_request_context from legal_api.services.utils import get_str @@ -103,13 +104,47 @@ CoreFiling.FilingTypes.NOTICEOFWITHDRAWAL, CoreFiling.FilingTypes.RESTORATION, } +DRS_KEY_PATTERN = re.compile(r"^([A-Z]+)-(DS\d+)$") + +FILING_TYPES_WITH_NR = [ + CoreFiling.FilingTypes.INCORPORATIONAPPLICATION, + CoreFiling.FilingTypes.REGISTRATION, + CoreFiling.FilingTypes.AMALGAMATIONAPPLICATION, + CoreFiling.FilingTypes.CONTINUATIONIN, + CoreFiling.FilingTypes.ALTERATION, + CoreFiling.FilingTypes.CHANGEOFNAME, + CoreFiling.FilingTypes.CHANGEOFREGISTRATION, + CoreFiling.FilingTypes.CONVERSION, +] + +NR_BLOCKING_STATUSES = [ + Filing.Status.PENDING, + Filing.Status.PAID, + Filing.Status.AWAITING_REVIEW, + Filing.Status.CHANGE_REQUESTED, + Filing.Status.APPROVED, +] + + +def _nr_in_pending_filing(nr_number: str) -> bool: + """Return True if the NR is already referenced in a non-draft/non-completed filing.""" + for filing_type in FILING_TYPES_WITH_NR: + if Filing.query.filter( + Filing._status.in_([s.value for s in NR_BLOCKING_STATUSES]), + Filing.filing_json["filing"][filing_type.value]["nameRequest"]["nrNumber"].astext == nr_number + ).first(): + return True + return False + + + def validate_resolution_date_in_share_structure(filing_json, filing_type, business) -> list[dict]: """Validate the resolution date of a share structure. Rules: - If hasRightsOrRestrictions is true in any share class or series, resolution date is required. - - Only one resolution date is permitted. + - Only one resolution date is permitted (alteration and old correction). - Resolution date cannot be in the future. - Resolution date cannot be before the business founding date. """ @@ -118,7 +153,7 @@ def validate_resolution_date_in_share_structure(filing_json, filing_type, busine resolution_dates = share_structure.get("resolutionDates", []) err_path = f"/filing/{filing_type}/shareStructure/resolutionDates" - + msg = [] if ( ( @@ -132,6 +167,29 @@ def validate_resolution_date_in_share_structure(filing_json, filing_type, busine "path": err_path }) + if not resolution_dates: + return msg + + if isinstance(resolution_dates[0], str): + # Kept for backward compatibility (existing alteration and correction filings) + msg.extend(_validate_resolution_dates_old_format(resolution_dates, business, err_path)) + else: + # Did not include "Only one resolution date is permitted.", not required for correction + # If its required for alteration add while updating alteration filing + existing_ids = {resolution.id for resolution in business.resolutions.all()} + for idx, resolution_date in enumerate(resolution_dates): + if (resolution_id := resolution_date.get("id")) and (resolution_id not in existing_ids): + msg.append({ + "error": "Not a valid Resolution Id for this business.", + "path": f"{err_path}/{idx}" + }) + else: + msg.extend(_validate_resolution_date(resolution_date["date"], business, f"{err_path}/{idx}")) + + return msg + +def _validate_resolution_dates_old_format(resolution_dates, business, err_path): + msg = [] if len(resolution_dates) > 1: msg.append({ "error": "Only one resolution date is permitted.", @@ -139,22 +197,27 @@ def validate_resolution_date_in_share_structure(filing_json, filing_type, busine }) elif len(resolution_dates) == 1: - resolution_date_leg = date.fromisoformat(resolution_dates[0]) - founding_date_leg = LegislationDatetime.as_legislation_timezone(business.founding_date).date() - today_leg = LegislationDatetime.datenow() + msg.extend(_validate_resolution_date(resolution_dates[0], business, err_path)) - if resolution_date_leg > today_leg: - msg.append({ - "error": "Resolution date cannot be in the future.", - "path": err_path - }) + return msg - if resolution_date_leg < founding_date_leg: - msg.append({ - "error": "Resolution date cannot be before the business founding date.", - "path": err_path - }) +def _validate_resolution_date(resolution_date_str: str, business, err_path: str) -> list[dict]: + resolution_date = date.fromisoformat(resolution_date_str) + founding_date = LegislationDatetime.as_legislation_timezone(business.founding_date).date() + today = LegislationDatetime.datenow() + + msg = [] + if resolution_date > today: + msg.append({ + "error": "Resolution date cannot be in the future.", + "path": err_path + }) + if resolution_date < founding_date: + msg.append({ + "error": "Resolution date cannot be before the business founding date.", + "path": err_path + }) return msg @@ -470,48 +533,41 @@ def validate_share_currency(filing_json, filing_type, business=None): return msg -def validate_court_order(court_order_path, court_order): +def validate_court_order( + court_order_path: str, + court_order: dict, + is_file_or_details_required: bool = False +) -> list[dict]: """Validate the courtOrder data of the filing.""" msg = [] - # TODO remove it when the issue with schema validation is fixed - min_file_number_length: Final = 5 - max_file_number_length: Final = 20 - if "fileNumber" not in court_order: - err_path = court_order_path + "/fileNumber" - msg.append({"error": "Court order file number is required.", "path": err_path}) - elif ( - len(court_order["fileNumber"]) < min_file_number_length or - len(court_order["fileNumber"]) > max_file_number_length - ): - err_path = court_order_path + "/fileNumber" - msg.append({"error": "Length of court order file number must be from 5 to 20 characters.", - "path": err_path}) - - if (effect_of_order := court_order.get("effectOfOrder", None)) and effect_of_order != "planOfArrangement": + if (effect_of_order := court_order.get("effectOfOrder")) and effect_of_order != "planOfArrangement": msg.append({"error": "Invalid effectOfOrder.", "path": f"{court_order_path}/effectOfOrder"}) - court_order_date_path = court_order_path + "/orderDate" - if "orderDate" in court_order: - try: - court_order_date = dt.fromisoformat(court_order["orderDate"]) - if court_order_date.timestamp() > datetime.now(UTC).timestamp(): - err_path = court_order_date_path - msg.append({"error": "Court order date cannot be in the future.", "path": err_path}) - except ValueError: - err_path = court_order_date_path - msg.append({"error": "Invalid court order date format.", "path": err_path}) - + if order_date := court_order.get("orderDate"): + court_order_date = dt.fromisoformat(order_date) + if court_order_date.timestamp() > datetime.now(UTC).timestamp(): + msg.append({"error": "Court order date cannot be in the future.", "path": f"{court_order_path}/orderDate"}) + + if is_file_or_details_required: + file_key_path = f"{court_order_path}/fileKey" + file_key = court_order.get("fileKey") + + if not court_order.get("orderDetails") and not file_key: + msg.append({"error": _("Court Order is required (in orderDetails/fileKey)."), "path": court_order_path}) + + if file_key: + msg.extend(validate_pdf(file_key, file_key_path)) + if flags.is_on("enabled-deeper-permission-action"): required_permission = ListActionsPermissionsAllowed.COURT_ORDER_POA.value message = "Permission Denied - You do not have permissions add court order details in this filing." permission_error = PermissionService.check_user_permission(required_permission, message=message) if permission_error: msg.append({"error": permission_error.msg[0].get("message", message), "path": court_order_path}) - if msg: - return msg - return None + return msg + def check_good_standing_permission(business: Business) -> Error | None: """Check if user has permission to file for a business not in good standing.""" @@ -525,12 +581,13 @@ def check_good_standing_permission(business: Business) -> Error | None: message = "Permission Denied - You do not have permissions send not in good standing business in this filing." return PermissionService.check_user_permission(required_permission, message=message) + def validate_pdf(file_key: str, file_key_path: str, verify_paper_size: bool = True) -> list | None: """Validate the PDF file.""" msg = [] try: - file = MinioService.get_file(file_key) - open_pdf_file = io.BytesIO(file.data) + file_data, file_size = _get_file_data(file_key) + open_pdf_file = io.BytesIO(file_data) pdf_reader = PdfReader(open_pdf_file) # Check that all pages in the pdf are letter size and able to be processed. @@ -543,9 +600,8 @@ def validate_pdf(file_key: str, file_key_path: str, verify_paper_size: bool = Tr msg.append({"error": _("Document must be set to fit onto 8.5” x 11” letter-size paper."), "path": file_key_path}) - file_info = MinioService.get_file_info(file_key) max_file_size: Final = 30000000 - if file_info.size > max_file_size: + if file_size > max_file_size: msg.append({"error": _("File exceeds maximum size."), "path": file_key_path}) if pdf_reader.is_encrypted: @@ -555,10 +611,17 @@ def validate_pdf(file_key: str, file_key_path: str, verify_paper_size: bool = Tr current_app.logger.debug(f"Error validating PDF: {ex}") msg.append({"error": _("Invalid file."), "path": file_key_path}) - if msg: - return msg + return msg - return None + +def _get_file_data(file_key: str) -> tuple[bytes, int]: + """Return (file_bytes, file_size) for a DRS-backed file_key.""" + if match := DRS_KEY_PATTERN.match(file_key): + doc_class, drs_id = match.group(1), match.group(2) + response = doc_service.get_document(drs_id, doc_class, doc_binary=True) + if not response.ok: + raise ValueError(f"DRS get_document failed: status={response.status_code}") + return response.content, len(response.content) def validate_parties_names(filing_json: dict, filing_type: str, legal_type: str) -> list: @@ -687,8 +750,7 @@ def validate_relationships( # noqa: PLR0913 msg.append({"error": "New Relationships are not allowed in this filing.", "path": f"{path}/entity"}) msg.extend(validate_relationship_entity_name(relationship, path)) - msg.extend(validate_relationship_roles(relationship["roles"], role_types, path)) - + msg.extend(validate_relationship_roles(relationship, role_types, path, business)) # Below is for colin sync checking only (i.e. any relationship with Director roles) converted_sync_roles = [role.value.lower().replace(" ", "_") for role in role_types_for_colin_sync or []] if any(role for role in relationship["roles"] if role["roleType"].lower() in converted_sync_roles): @@ -826,18 +888,90 @@ def validate_relationship_entity_colin_sync(relationship: dict, legal_type: str, return msg -def validate_relationship_roles(roles: list[dict[str, str]], +def validate_relationship_roles(relationship: dict, allowed_roles: list[PartyRole.RoleTypes], - path: str) -> list: + path: str, + business: Business) -> list: """Validate relationship roles.""" msg = [] converted_allowed_roles = [role.value for role in allowed_roles] + earliest_allowed_date = LegislationDatetime.as_legislation_timezone(business.founding_date).date() + + roles = relationship["roles"] for index, role in enumerate(roles): if role.get("roleType").lower().replace(" ", "_") not in converted_allowed_roles: err_msg = "Invalid role type for this filing." msg.append({"error": err_msg, "path": f"{path}/{index}/roleType"}) # FUTURE: appointment/cessation date checks (currently set to filing effective date by filer) + appointment_date = date.fromisoformat(role.get("appointmentDate")) if role.get("appointmentDate") else None + cessation_date = date.fromisoformat(role.get("cessationDate")) if role.get("cessationDate") else None + + msg.extend(_validate_relationship_date(appointment_date, + f"{path}/{index}/appointmentDate", + "Appointment", + earliest_allowed_date)) + msg.extend(_validate_relationship_date(cessation_date, + f"{path}/{index}/cessationDate", + "Cessation", + earliest_allowed_date)) + + msg.extend(_validate_director_dates(appointment_date, + cessation_date, + f"{path}/{index}", + role)) + return msg + +def _validate_director_dates(appointment_date: date, cessation_date: date, path: str, role: dict) -> list: + """Validate director dates.""" + msg = [] + if role.get("roleType").lower() != PartyRole.RoleTypes.DIRECTOR.value: + return msg + + date_path = f"{path}/appointmentDate" + if not appointment_date and cessation_date: + msg.append({ + "error": _("Appointment date is required when cessation date is present."), + "path": date_path + }) + else: + msg.extend(_compare_director_dates(appointment_date, cessation_date, date_path)) + + return msg + + +def _compare_director_dates(appointment_date: date, cessation_date: date, path: str) -> list: + """Validate director dates.""" + msg = [] + if appointment_date and cessation_date and appointment_date > cessation_date: + msg.append({ + "error": _("Appointment date cannot be after cessation date."), + "path": path + }) + return msg + + +def _validate_relationship_date(date_value: date, + path: str, + error_name: str, + earliest_allowed_date: date) -> list: + msg = [] + if date_value is None: + return msg + + today = LegislationDatetime.datenow() + if date_value > today: + msg.append({ + "error": _(f"{error_name} date cannot be in the future."), + "path": path + }) + + if date_value < earliest_allowed_date: + msg.append({ + "error": _(f"{error_name} date cannot be before the business founding date."), + "path": path + }) + return msg @@ -877,6 +1011,11 @@ def validate_name_request(filing_json: dict, # pylint: disable=too-many-locals if not validation_result["is_consumable"]: msg.append({"error": _("Name Request is not approved."), "path": nr_number_path}) + # ensure NR is not already referenced in another pending/paid filing + if _nr_in_pending_filing(nr_number): + msg.append({"error": _("Name Request is already part of a pending filing."), + "path": nr_number_path}) + # ensure NR request type code if accepted_request_types and nr_response_json["requestTypeCd"] not in accepted_request_types: msg.append({"error": _("The name type associated with the name request number entered cannot be used."), @@ -1298,7 +1437,25 @@ def validate_certified_by(filing_json: dict, filing_type: str, legal_type: str) certified_by = filing_json["filing"]["header"].get("certifiedBy") if legal_type in Business.CORPS: - return msg # certifiedBy is not required for corporations + # the completing party statement takes its name from certifiedBy for staff and API users, + # so certifiedBy is required for them on a corp incorporation application + api_login_source = "API_GW" # jwt loginSource of an API gateway user + current_user = getattr(request_ctx, "current_user", None) if has_request_context() else None + if (filing_type == CoreFiling.FilingTypes.INCORPORATIONAPPLICATION + and current_user + and (jwt.validate_roles(current_user, [STAFF_ROLE]) + or current_user.get("loginSource") == api_login_source)): + if not certified_by: + msg.append({ + "error": "Certified by field is required.", + "path": "/filing/header/certifiedBy" + }) + elif certified_by != certified_by.strip(): + msg.append({ + "error": "Certified by field cannot start or end with whitespace.", + "path": "/filing/header/certifiedBy" + }) + return msg # certifiedBy is not otherwise required for corporations is_cert_filing = filing_type in FILINGS_REQUIRING_CERTIFICATION is_client_correction = ( @@ -1391,8 +1548,8 @@ def validate_party_role_firms(parties: list, filing_type: str) -> list: if business_identifier: business_found = Business.find_by_identifier(business_identifier) is not None if not business_found: - colin_business = colin.query_business(business_identifier) - business_found = colin_business.status_code == HTTPStatus.OK + _, colin_status = colin.query_business(business_identifier) + business_found = colin_status == HTTPStatus.OK if business_found: continue diff --git a/legal-api/src/legal_api/services/filings/validations/consent_amalgamation_out.py b/legal-api/src/legal_api/services/filings/validations/consent_amalgamation_out.py index 6c772cac67..d7c6a164df 100644 --- a/legal-api/src/legal_api/services/filings/validations/consent_amalgamation_out.py +++ b/legal-api/src/legal_api/services/filings/validations/consent_amalgamation_out.py @@ -61,9 +61,7 @@ def validate(business: Business, filing: dict) -> Error | None: if court_order := filing.get("filing", {}).get(filing_type, {}).get("courtOrder", None): court_order_path: Final = f"/filing/{filing_type}/courtOrder" - err = validate_court_order(court_order_path, court_order) - if err: - msg.extend(err) + msg.extend(validate_court_order(court_order_path, court_order)) if msg: return Error(HTTPStatus.BAD_REQUEST, msg) diff --git a/legal-api/src/legal_api/services/filings/validations/consent_continuation_out.py b/legal-api/src/legal_api/services/filings/validations/consent_continuation_out.py index 9b5cfd37f3..908410589b 100644 --- a/legal-api/src/legal_api/services/filings/validations/consent_continuation_out.py +++ b/legal-api/src/legal_api/services/filings/validations/consent_continuation_out.py @@ -60,9 +60,7 @@ def validate(business: Business, filing: dict) -> Error | None: if court_order := filing.get("filing", {}).get(filing_type, {}).get("courtOrder", None): court_order_path: Final = f"/filing/{filing_type}/courtOrder" - err = validate_court_order(court_order_path, court_order) - if err: - msg.extend(err) + msg.extend(validate_court_order(court_order_path, court_order)) if msg: return Error(HTTPStatus.BAD_REQUEST, msg) diff --git a/legal-api/src/legal_api/services/filings/validations/continuation_in.py b/legal-api/src/legal_api/services/filings/validations/continuation_in.py index d36db50073..d578e86bba 100644 --- a/legal-api/src/legal_api/services/filings/validations/continuation_in.py +++ b/legal-api/src/legal_api/services/filings/validations/continuation_in.py @@ -61,9 +61,18 @@ def validate(filing_json: dict) -> Error | None: # pylint: disable=too-many-bra return Error(HTTPStatus.FORBIDDEN, [{"error": babel(f"{legal_type} does not support continuation in filing.")}]) - msg.extend(validate_business_in_colin(filing_json, filing_type)) + msg.extend(validate_continuation_in_expro_business_in_colin( + filing_json["filing"][filing_type].get("business"), + f"/filing/{filing_type}/business" + )) msg.extend(validate_continuation_in_authorization(filing_json, filing_type, legal_type)) - msg.extend(_validate_foreign_jurisdiction(filing_json, filing_type, legal_type)) + + foreign_jurisdiction = filing_json["filing"][filing_type]["foreignJurisdiction"] + msg.extend(validate_continuation_in_foreign_jurisdiction( + legal_type, + foreign_jurisdiction, + f"/filing/{filing_type}/foreignJurisdiction" + )) msg.extend(validate_name_request(filing_json, legal_type, filing_type)) if get_bool(filing_json, "/filing/continuationIn/isApproved"): @@ -161,13 +170,14 @@ def _validate_incorporation_date(incorporation_date: str, incorporation_date_pat return msg -def _validate_foreign_jurisdiction(filing_json: dict, filing_type: str, legal_type: str) -> list: +def validate_continuation_in_foreign_jurisdiction( + legal_type: str, + foreign_jurisdiction: dict, + foreign_jurisdiction_path: str, + skip_affidavit: bool = False +) -> list: """Validate continuation in foreign jurisdiction.""" msg = [] - foreign_jurisdiction = filing_json["filing"][filing_type]["foreignJurisdiction"] - incorporation_date = filing_json["filing"][filing_type]["foreignJurisdiction"]["incorporationDate"] - foreign_jurisdiction_path = f"/filing/{filing_type}/foreignJurisdiction" - incorporation_date_path = f"/filing/{filing_type}/foreignJurisdiction/incorporationDate" # identifier required (non-empty / non-whitespace) is enforced by the schema # (business-schemas continuation_in foreignJurisdiction.identifier pattern). @@ -189,16 +199,22 @@ def _validate_foreign_jurisdiction(filing_json: dict, filing_type: str, legal_ty if err := validate_foreign_jurisdiction(foreign_jurisdiction, foreign_jurisdiction_path): msg.extend(err) - elif (legal_type == Business.LegalTypes.ULC_CONTINUE_IN.value and - foreign_jurisdiction["country"] == "CA" and - ((region := foreign_jurisdiction.get("region")) and region == "AB")): + + # Skip affidavit if skip_affidavit is True (correction) + if ( + not skip_affidavit and + legal_type == Business.LegalTypes.ULC_CONTINUE_IN.value and + foreign_jurisdiction.get("country") == "CA" and + foreign_jurisdiction.get("region") == "AB" + ): affidavit_file_key_path = f"{foreign_jurisdiction_path}/affidavitFileKey" if file_key := foreign_jurisdiction.get("affidavitFileKey"): - if err := validate_pdf(file_key, affidavit_file_key_path, False): - msg.extend(err) + msg.extend(validate_pdf(file_key, affidavit_file_key_path, False)) else: msg.append({"error": "Affidavit from the directors is required.", "path": affidavit_file_key_path}) + incorporation_date = foreign_jurisdiction["incorporationDate"] + incorporation_date_path = f"{foreign_jurisdiction_path}/incorporationDate" msg.extend(_validate_incorporation_date(incorporation_date, incorporation_date_path)) return msg @@ -217,8 +233,7 @@ def validate_continuation_in_authorization(filing_json: dict, filing_type: str, for index, file in enumerate(file_list): file_key = file["fileKey"] file_key_path = f"{authorization_path}/files/{index}/fileKey" - if err := validate_pdf(file_key, file_key_path, False): - msg.extend(err) + msg.extend(validate_pdf(file_key, file_key_path, False)) return msg @@ -227,32 +242,29 @@ def validate_continuation_in_court_order(filing: dict, filing_type) -> list: """Validate court order.""" if court_order := filing.get("filing", {}).get(filing_type, {}).get("courtOrder", None): court_order_path: Final = f"/filing/{filing_type}/courtOrder" - err = validate_court_order(court_order_path, court_order) - if err: - return err + return validate_court_order(court_order_path, court_order) return [] -def validate_business_in_colin(filing_json: dict, filing_type: str) -> list: +def validate_continuation_in_expro_business_in_colin(expro: dict, path: str, skip_founding_date: bool = False) -> list: """Validate continuation EXPRO business by making a call to Colin API.""" msg = [] - business_identifier_path = f"/filing/{filing_type}/business/identifier" - business_legal_name_path = f"/filing/{filing_type}/business/legalName" - business_founding_date_path = f"/filing/{filing_type}/business/foundingDate" - - if filing_json["filing"][filing_type].get("business"): - identifier = filing_json["filing"][filing_type]["business"]["identifier"] - legal_name = filing_json["filing"][filing_type]["business"].get("legalName") - founding_date = filing_json["filing"][filing_type]["business"].get("foundingDate") - response = colin.query_business(identifier) - response_json = response.json() - if response.status_code != HTTPStatus.OK: + business_identifier_path = f"{path}/identifier" + business_legal_name_path = f"{path}/legalName" + business_founding_date_path = f"{path}/foundingDate" + + if expro: + identifier = expro["identifier"] + legal_name = expro.get("legalName") + founding_date = expro.get("foundingDate") + response_json, response_status = colin.query_business(identifier) + if response_json is None or response_status != HTTPStatus.OK: msg.append({"error": "Could not fetch business data for company from Colin.", "path": business_identifier_path}) elif legal_name != response_json["business"]["legalName"]: msg.append({"error": "Legal name does not match with company legal name from Colin.", "path": business_legal_name_path}) - elif founding_date != response_json["business"]["foundingDate"]: + elif not skip_founding_date and founding_date != response_json["business"]["foundingDate"]: msg.append({"error": "Founding date does not match with founding date from Colin.", "path": business_founding_date_path}) diff --git a/legal-api/src/legal_api/services/filings/validations/continuation_out.py b/legal-api/src/legal_api/services/filings/validations/continuation_out.py index 928c1d9d16..d44219571d 100644 --- a/legal-api/src/legal_api/services/filings/validations/continuation_out.py +++ b/legal-api/src/legal_api/services/filings/validations/continuation_out.py @@ -18,7 +18,7 @@ from flask_babel import _ as babel from business_common.utils.legislation_datetime import LegislationDatetime -from business_model.models import Business, ConsentContinuationOut +from business_model.models import Business from legal_api.errors import Error from legal_api.services import flags from legal_api.services.filings.validations.common_validations import ( @@ -41,64 +41,24 @@ def validate(business: Business, filing: dict) -> Error | None: msg = [] filing_type = "continuationOut" - is_valid_co_date = True - is_valid_foreign_jurisdiction = True - - if err := validate_continuation_out_date(filing, filing_type): - msg.extend(err) - is_valid_co_date = False - - if err := validate_foreign_jurisdiction(filing["filing"][filing_type]["foreignJurisdiction"], - f"/filing/{filing_type}/foreignJurisdiction"): - msg.extend(err) - is_valid_foreign_jurisdiction = False - - if is_valid_co_date and is_valid_foreign_jurisdiction: - msg.extend(validate_active_cco(business, filing, filing_type)) + msg.extend(validate_continuation_out_date(filing, f"/filing/{filing_type}/continuationOutDate")) + msg.extend(validate_foreign_jurisdiction( + filing["filing"][filing_type]["foreignJurisdiction"], + f"/filing/{filing_type}/foreignJurisdiction" + )) if court_order := filing.get("filing", {}).get(filing_type, {}).get("courtOrder", None): court_order_path: Final = f"/filing/{filing_type}/courtOrder" - err = validate_court_order(court_order_path, court_order) - if err: - msg.extend(err) + msg.extend(validate_court_order(court_order_path, court_order)) if msg: return Error(HTTPStatus.BAD_REQUEST, msg) return None -def validate_active_cco(business: Business, filing: dict, filing_type: str) -> list: - """Validate active consent continuation out.""" - msg = [] - continuation_out_date_str = filing["filing"][filing_type]["continuationOutDate"] - continuation_out_date = LegislationDatetime.as_legislation_timezone_from_date_str(continuation_out_date_str) - - foreign_jurisdiction = filing["filing"][filing_type]["foreignJurisdiction"] - country_code = foreign_jurisdiction.get("country") - region = foreign_jurisdiction.get("region") - - continuation_out_date_utc = LegislationDatetime.as_utc_timezone(continuation_out_date) - ccos = ConsentContinuationOut.get_active_cco(business.id, continuation_out_date_utc, country_code, region) - - active_consent = False - # Make sure continuation_out_date is on or after consent filing effective date - for consent in ccos: - if continuation_out_date.date() >= \ - LegislationDatetime.as_legislation_timezone(consent.filing.effective_date).date(): - active_consent = True - break - - if not active_consent: - msg.extend([{"error": "No active consent continuation out for this date and/or jurisdiction.", - "path": f"/filing/{filing_type}/continuationOutDate"}]) - - return msg - - -def validate_continuation_out_date(filing: dict, filing_type: str) -> list: +def validate_continuation_out_date(filing: dict, continuation_out_date_path: str) -> list: """Validate continuation out date.""" msg = [] - continuation_out_date_path = f"/filing/{filing_type}/continuationOutDate" continuation_out_date = get_date(filing, continuation_out_date_path) now = LegislationDatetime.now().date() diff --git a/legal-api/src/legal_api/services/filings/validations/correction.py b/legal-api/src/legal_api/services/filings/validations/correction.py index 7e3f587496..03459b520f 100644 --- a/legal-api/src/legal_api/services/filings/validations/correction.py +++ b/legal-api/src/legal_api/services/filings/validations/correction.py @@ -20,12 +20,15 @@ from flask.globals import request_ctx from flask_babel import _ -from business_model.models import Business, Filing, PartyRole +from business_model.models import Business, CourtOrder, Filing, PartyRole from legal_api.core.filing_helper import is_special_resolution_correction_by_filing_json from legal_api.errors import Error from legal_api.services import STAFF_ROLE, SYSTEM_ROLE, NaicsService +from legal_api.services.filings.validations.alteration import validate_type_change +from legal_api.services.filings.validations.amalgamation_out import validate_amalgamation_out_date from legal_api.services.filings.validations.common_validations import ( validate_court_order, + validate_foreign_jurisdiction, validate_name_request, validate_offices_addresses, validate_parties_addresses, @@ -36,6 +39,11 @@ validate_share_currency, validate_share_structure, ) +from legal_api.services.filings.validations.continuation_in import ( + validate_continuation_in_expro_business_in_colin, + validate_continuation_in_foreign_jurisdiction, +) +from legal_api.services.filings.validations.continuation_out import validate_continuation_out_date from legal_api.services.filings.validations.incorporation_application import ( validate_coop_parties_mailing_address, validate_roles, @@ -79,6 +87,11 @@ def validate(business: Business, filing: dict) -> Error: path = "/filing/correction/correctedFilingId" msg.append({"error": _("Corrected filing is not a valid filing for this business."), "path": path}) + elif corrected_filing.filing_type != filing["filing"]["correction"]["correctedFilingType"]: + path = "/filing/correction/correctedFilingType" + msg.append({"error": _("The corrected filing type does not match filing type of corrected filing."), + "path": path}) + # skip all the other validation checks if comment only correction if not is_comment_only_correction: if filing.get("filing", {}).get("correction", {}).get("parties", None): @@ -100,19 +113,22 @@ def validate(business: Business, filing: dict) -> Error: )) if filing.get("filing", {}).get("correction", {}).get("offices", None): msg.extend(validate_offices_addresses(filing, filing_type)) - # validations for firms - if business.legal_type in [Business.LegalTypes.SOLE_PROP.value, Business.LegalTypes.PARTNERSHIP.value]: - _validate_firms_correction(business, filing, business.legal_type, msg) - elif business.legal_type in Business.CORPS: - _validate_corps_correction(business, filing, business.legal_type, msg) - elif business.legal_type == Business.LegalTypes.COOP.value: - _validate_special_resolution_correction(filing, business.legal_type, msg) + + _validate_type_specific_props(business, filing, msg) if msg: return Error(HTTPStatus.BAD_REQUEST, msg) return None +def _validate_type_specific_props(business: Business, filing: dict, msg: list): + if business.legal_type in [Business.LegalTypes.SOLE_PROP.value, Business.LegalTypes.PARTNERSHIP.value]: + _validate_firms_correction(business, filing, business.legal_type, msg) + elif business.legal_type in Business.CORPS: + _validate_corps_correction(business, filing, business.legal_type, msg) + elif business.legal_type == Business.LegalTypes.COOP.value: + _validate_special_resolution_correction(filing, business.legal_type, msg) + def _validate_firms_correction(business: Business, filing, legal_type, msg): filing_type = "correction" @@ -129,6 +145,13 @@ def _validate_firms_correction(business: Business, filing, legal_type, msg): def _validate_corps_correction(business: Business, filing_dict, legal_type, msg): filing_type = "correction" + if new_legal_type := filing_dict.get("filing", {}).get("correction", {}).get("newLegalType"): + if business.legal_type == new_legal_type: + # This is not a valid type change + path = "/filing/correction/newLegalType" + msg.append({"error": _("New legal type must be different from current legal type."), "path": path}) + else: + msg.extend(validate_type_change(filing_dict, business, "/filing/correction/newLegalType")) if filing_dict.get("filing", {}).get("correction", {}).get("nameRequest", {}).get("nrNumber", None): msg.extend(validate_name_request(filing_dict, legal_type, filing_type)) if filing_dict.get("filing", {}).get("correction", {}).get("offices", None): @@ -144,30 +167,137 @@ def _validate_corps_correction(business: Business, filing_dict, legal_type, msg) if err: msg.extend(err) - err = validate_share_currency(filing_dict, filing_type, business) - if err: - msg.extend(err) + msg.extend(validate_share_currency(filing_dict, filing_type, business)) + msg.extend(validate_resolution_date_in_share_structure(filing_dict, filing_type, business)) - err = validate_resolution_date_in_share_structure(filing_dict, filing_type, business) - if err: - msg.extend(err) + if filing_dict.get("filing", {}).get("correction", {}).get("courtOrder", None): + msg.extend(court_order_validation(filing_dict)) + + msg.extend(_validate_continuation_in_correction(filing_dict, filing_type, legal_type)) + msg.extend(_validate_out_correction(filing_dict, filing_type)) + msg.extend(_validate_amalgamation_correction(filing_dict, filing_type, business)) + msg.extend(_validate_court_orders_correction(filing_dict, business)) + + +def _validate_court_orders_correction(filing_dict, business: Business): + """Validate court orders for a correction filing. + + - If the court order has an Id, verifies it exists in the database for the business. + - If the court order does not have an Id, verifies its filingId matches the corrected filing Id. + - Verifies that only one court order is associated with the corrected filing in the database. + - Verifies that no more than one new court order is being added in the correction. + - Performs common validation on court order details. + """ + msg = [] + if not (orders := filing_dict["filing"]["correction"].get("courtOrders")): + return msg + + corrected_filing_id = filing_dict["filing"]["correction"]["correctedFilingId"] + new_court_orders = [] + court_orders_db = CourtOrder.get_json_with_filing_type(business.id) + for idx, order in enumerate(orders): + path = f"/filing/correction/courtOrders/{idx}" + is_file_or_details_required = False + if order_id := order.get("id"): + court_order_db = next((o for o in court_orders_db if o["id"] == order_id), None) + if not court_order_db: + msg.append({"error": _("Court order not found."), "path": path}) + else: + is_file_or_details_required = (court_order_db["filingType"] == "courtOrder") + elif order.get("filingId") == corrected_filing_id: + if next((o for o in court_orders_db if o["filingId"] == corrected_filing_id), None): + msg.append({"error": _("Only one court order can be added per filing."), "path": path}) + + is_file_or_details_required = (filing_dict["filing"]["correction"]["correctedFilingType"] == "courtOrder") + new_court_orders.append(order) + else: + msg.append({"error": _("Filing Id does not match corrected filing Id."), "path": path}) + + + msg.extend(validate_court_order(path, order, is_file_or_details_required)) + + if len(new_court_orders) > 1: + msg.append({"error": _("Only one court order can be added."), "path": "/filing/correction/courtOrders"}) + + return msg + + +def _validate_amalgamation_correction(filing_dict, filing_type, business: Business): + msg = [] + if not ( + amalgamating_businesses_json := filing_dict["filing"][filing_type].get("amalgamation", {}) + .get("amalgamatingBusinesses", []) + ): + return msg + + amalgamation = business.amalgamation.first() + amalgamating_businesses = amalgamation.amalgamating_businesses.all() + for idx, ting_json in enumerate(amalgamating_businesses_json): + path = f"/filing/{filing_type}/amalgamation/amalgamatingBusinesses/{idx}" + ting_id = ting_json["id"] + ting = next((ting for ting in amalgamating_businesses if ting.id == ting_id), None) + if not ting: + msg.append({"error": _("Amalgamating business not found."), "path": path}) + continue + + if ting.business_id: + msg.append({"error": _("Can only correct foreign businesses."), "path": path}) + continue + + msg.extend( + validate_foreign_jurisdiction( + ting_json["foreignJurisdiction"], + f"{path}/foreignJurisdiction", + is_region_bc_valid=True, + is_region_for_us_required=False + ) + ) + return msg + + +def _validate_continuation_in_correction(filing_dict, filing_type, legal_type): + msg = [] + if continuation_in := filing_dict["filing"][filing_type].get("continuationIn"): + msg.extend(validate_continuation_in_foreign_jurisdiction( + legal_type, + continuation_in, + f"/filing/{filing_type}/continuationIn", + skip_affidavit=True + )) + msg.extend(validate_continuation_in_expro_business_in_colin( + continuation_in.get("expro"), + f"/filing/{filing_type}/continuationIn/expro", + skip_founding_date=True + )) + return msg + + +def _validate_out_correction(filing_dict, filing_type): + msg = [] + if continuation_out := filing_dict["filing"][filing_type].get("continuationOut"): + msg.extend(validate_continuation_out_date(filing_dict, f"/filing/{filing_type}/continuationOut/date")) + msg.extend(validate_foreign_jurisdiction(continuation_out, f"/filing/{filing_type}/continuationOut")) + elif amalgamation_out := filing_dict["filing"][filing_type].get("amalgamationOut"): + msg.extend(validate_amalgamation_out_date(filing_dict, f"/filing/{filing_type}/amalgamationOut/date")) + msg.extend(validate_foreign_jurisdiction(amalgamation_out, f"/filing/{filing_type}/amalgamationOut")) + return msg def _validate_special_resolution_correction(filing_dict, legal_type, msg): filing_type = "correction" if filing_dict.get("filing", {}).get(filing_type, {}).get("nameRequest", {}).get("nrNumber", None): msg.extend(validate_name_request(filing_dict, legal_type, filing_type)) - if filing_dict.get("filing", {}).get(filing_type, {}).get("correction", {}).get("resolution", None): + if filing_dict.get("filing", {}).get(filing_type, {}).get("resolution", None): msg.extend(validate_resolution_content(filing_dict, filing_type)) - if filing_dict.get("filing", {}).get(filing_type, {}).get("correction", {}).get("signingDate", None): + if filing_dict.get("filing", {}).get(filing_type, {}).get("signingDate", None): msg.extend(validate_signing_date(filing_dict, filing_type)) - if filing_dict.get("filing", {}).get(filing_type, {}).get("correction", {}).get("signatory", None): + if filing_dict.get("filing", {}).get(filing_type, {}).get("signatory", None): msg.extend(validate_signatory_name(filing_dict, filing_type)) - if filing_dict.get("filing", {}).get(filing_type, {}).get("correction", {}).get("courtOrder", None): + if filing_dict.get("filing", {}).get(filing_type, {}).get("courtOrder", None): msg.extend(court_order_validation(filing_dict)) - if filing_dict.get("filing", {}).get(filing_type, {}).get("correction", {}).get("rulesFileKey", None): + if filing_dict.get("filing", {}).get(filing_type, {}).get("rulesFileKey", None): msg.extend(rules_change_validation(filing_dict)) - if filing_dict.get("filing", {}).get(filing_type, {}).get("correction", {}).get("memorandumFileKey", None): + if filing_dict.get("filing", {}).get(filing_type, {}).get("memorandumFileKey", None): msg.extend(memorandum_change_validation(filing_dict)) if is_special_resolution_correction_by_filing_json(filing_dict.get("filing", {})): _validate_roles_parties_correction(filing_dict, legal_type, filing_type, msg) @@ -284,9 +414,7 @@ def court_order_validation(filing): """Validate court order.""" court_order_path: Final = "/filing/correction/courtOrder" if get_str(filing, court_order_path): - err = validate_court_order(court_order_path, filing["filing"]["correction"]["courtOrder"]) - if err: - return err + return validate_court_order(court_order_path, filing["filing"]["correction"]["courtOrder"]) return [] @@ -297,11 +425,9 @@ def rules_change_validation(filing): rules_file_key: Final = get_str(filing, rules_file_key_path) if rules_file_key: - rules_err = validate_pdf(rules_file_key, rules_file_key_path) - if rules_err: - msg.extend(rules_err) - return msg - return [] + msg.extend(validate_pdf(rules_file_key, rules_file_key_path)) + + return msg def memorandum_change_validation(filing): @@ -311,8 +437,6 @@ def memorandum_change_validation(filing): memorandum_file_key: Final = get_str(filing, memorandum_file_key_path) if memorandum_file_key: - rules_err = validate_pdf(memorandum_file_key, memorandum_file_key_path) - if rules_err: - msg.extend(rules_err) - return msg - return [] + msg.extend(validate_pdf(memorandum_file_key, memorandum_file_key_path)) + + return msg diff --git a/legal-api/src/legal_api/services/filings/validations/court_order.py b/legal-api/src/legal_api/services/filings/validations/court_order.py index f97eaacced..3345027e65 100644 --- a/legal-api/src/legal_api/services/filings/validations/court_order.py +++ b/legal-api/src/legal_api/services/filings/validations/court_order.py @@ -18,33 +18,19 @@ from business_model.models import Business from legal_api.errors import Error -from legal_api.services.filings.validations.common_validations import validate_pdf -from legal_api.services.utils import get_str +from legal_api.services.filings.validations.common_validations import validate_court_order def validate(business: Business, court_order: dict) -> Error | None: """Validate the Court Order filing.""" if not business or not court_order: return Error(HTTPStatus.BAD_REQUEST, [{"error": babel("A valid business and filing are required.")}]) - msg = [] - effect_of_order = get_str(court_order, "/filing/courtOrder/effectOfOrder") - if effect_of_order and effect_of_order != "planOfArrangement": - msg.append({"error": babel("Invalid effectOfOrder."), "path": "/filing/courtOrder/effectOfOrder"}) - - file_key_path = "/filing/courtOrder/fileKey" - file_key = get_str(court_order, file_key_path) - - order_details_path = "/filing/courtOrder/orderDetails" - order_details = get_str(court_order, order_details_path) - - if not order_details and not file_key: - msg.append({"error": babel("Court Order is required (in orderDetails/fileKey)."), "path": "/filing/courtOrder"}) - - if file_key: - file_err = validate_pdf(file_key, file_key_path) - if file_err: - msg.extend(file_err) + msg = validate_court_order( + "/filing/courtOrder", + court_order["filing"]["courtOrder"], + is_file_or_details_required=True + ) if msg: return Error(HTTPStatus.BAD_REQUEST, msg) diff --git a/legal-api/src/legal_api/services/filings/validations/dissolution.py b/legal-api/src/legal_api/services/filings/validations/dissolution.py index 718bfce116..505bcc22a6 100644 --- a/legal-api/src/legal_api/services/filings/validations/dissolution.py +++ b/legal-api/src/legal_api/services/filings/validations/dissolution.py @@ -106,10 +106,7 @@ def validate(business: Business, dissolution: dict) -> Error | None: # Common validation for addresses msg.extend(validate_parties_addresses(dissolution, filing_type)) - err = validate_affidavit(dissolution, business.legal_type, dissolution_type) - if err: - msg.extend(err) - + msg.extend(validate_affidavit(dissolution, business.legal_type, dissolution_type)) msg.extend(_validate_court_order(dissolution)) @@ -328,13 +325,14 @@ def _validate_party_address(party, idx, address_type, require_bc): return msg -def validate_affidavit(filing_json, legal_type, dissolution_type) -> list | None: +def validate_affidavit(filing_json, legal_type, dissolution_type) -> list: """Validate affidavit document of the filing. This needs not to be validated for administrative dissolution """ + msg = [] if dissolution_type in [DissolutionTypes.ADMINISTRATIVE, DissolutionTypes.DELAY]: - return None + return msg if legal_type == Business.LegalTypes.COOP.value: affidavit_file_key_path = "/filing/dissolution/affidavitFileKey" @@ -342,21 +340,19 @@ def validate_affidavit(filing_json, legal_type, dissolution_type) -> list | None # Validate key values exist if not affidavit_file_key: - return [{"error": _("A valid affidavit key is required."), - "path": affidavit_file_key_path}] + msg.append({"error": _("A valid affidavit key is required."), + "path": affidavit_file_key_path}) + else: + msg.extend(validate_pdf(affidavit_file_key, affidavit_file_key_path)) - return validate_pdf(affidavit_file_key, affidavit_file_key_path) - - return None + return msg def _validate_court_order(filing): """Validate court order.""" if court_order := filing.get("filing", {}).get("dissolution", {}).get("courtOrder", None): court_order_path: Final = "/filing/dissolution/courtOrder" - err = validate_court_order(court_order_path, court_order) - if err: - return err + return validate_court_order(court_order_path, court_order) return [] diff --git a/legal-api/src/legal_api/services/filings/validations/incorporation_application.py b/legal-api/src/legal_api/services/filings/validations/incorporation_application.py index 08e003234b..4c725c97ad 100644 --- a/legal-api/src/legal_api/services/filings/validations/incorporation_application.py +++ b/legal-api/src/legal_api/services/filings/validations/incorporation_application.py @@ -298,15 +298,11 @@ def validate_cooperative_documents(incorporation_json: dict): rules_file_key = cooperative["rulesFileKey"] rules_file_key_path = "/filing/incorporationApplication/cooperative/rulesFileKey" - rules_err = validate_pdf(rules_file_key, rules_file_key_path) - if rules_err: - msg.extend(rules_err) + msg.extend(validate_pdf(rules_file_key, rules_file_key_path)) memorandum_file_key = cooperative["memorandumFileKey"] memorandum_file_key_path = "/filing/incorporationApplication/cooperative/memorandumFileKey" - memorandum_err = validate_pdf(memorandum_file_key, memorandum_file_key_path) - if memorandum_err: - msg.extend(memorandum_err) + msg.extend(validate_pdf(memorandum_file_key, memorandum_file_key_path)) return msg @@ -315,9 +311,7 @@ def validate_ia_court_order(filing: dict) -> list: """Validate court order.""" if court_order := filing.get("filing", {}).get("incorporationApplication", {}).get("courtOrder", None): court_order_path: Final = "/filing/incorporationApplication/courtOrder" - err = validate_court_order(court_order_path, court_order) - if err: - return err + return validate_court_order(court_order_path, court_order) return [] def _validate_incorporation_permission( diff --git a/legal-api/src/legal_api/services/filings/validations/put_back_off.py b/legal-api/src/legal_api/services/filings/validations/put_back_off.py index 0d5c0df15f..472559ed27 100644 --- a/legal-api/src/legal_api/services/filings/validations/put_back_off.py +++ b/legal-api/src/legal_api/services/filings/validations/put_back_off.py @@ -43,7 +43,5 @@ def _validate_court_order(filing): """Validate court order.""" if court_order := filing.get("filing", {}).get("putBackOff", {}).get("courtOrder", None): court_order_path: Final = "/filing/putBackOff/courtOrder" - err = validate_court_order(court_order_path, court_order) - if err: - return err + return validate_court_order(court_order_path, court_order) return [] diff --git a/legal-api/src/legal_api/services/filings/validations/put_back_on.py b/legal-api/src/legal_api/services/filings/validations/put_back_on.py index 00db826980..ccc5fb21fa 100644 --- a/legal-api/src/legal_api/services/filings/validations/put_back_on.py +++ b/legal-api/src/legal_api/services/filings/validations/put_back_on.py @@ -49,7 +49,5 @@ def _validate_court_order(filing): """Validate court order.""" if court_order := filing.get("filing", {}).get("putBackOn", {}).get("courtOrder", None): court_order_path: Final = "/filing/putBackOn/courtOrder" - err = validate_court_order(court_order_path, court_order) - if err: - return err + return validate_court_order(court_order_path, court_order) return [] diff --git a/legal-api/src/legal_api/services/filings/validations/registrars_notation.py b/legal-api/src/legal_api/services/filings/validations/registrars_notation.py index 55a1e6f819..85fecddeea 100644 --- a/legal-api/src/legal_api/services/filings/validations/registrars_notation.py +++ b/legal-api/src/legal_api/services/filings/validations/registrars_notation.py @@ -18,7 +18,6 @@ from business_model.models import Business from legal_api.errors import Error -from legal_api.services.utils import get_str def validate(business: Business, registrars_notation: dict) -> Error | None: @@ -27,16 +26,16 @@ def validate(business: Business, registrars_notation: dict) -> Error | None: return Error(HTTPStatus.BAD_REQUEST, [{"error": babel("A valid business and filing are required.")}]) msg = [] - effect_of_order = get_str(registrars_notation, "/filing/registrarsNotation/effectOfOrder") - if effect_of_order: + data = registrars_notation["filing"]["registrarsNotation"] + path = "/filing/registrarsNotation" + if effect_of_order := data.get("effectOfOrder"): if effect_of_order == "planOfArrangement": - file_number = get_str(registrars_notation, "/filing/registrarsNotation/fileNumber") - if not file_number: + if not data.get("fileNumber"): msg.append({"error": babel( "Court Order Number is required when this filing is pursuant to a Plan of Arrangement."), - "path": "/filing/registrarsNotation/fileNumber"}) + "path": f"{path}/fileNumber"}) else: - msg.append({"error": babel("Invalid effectOfOrder."), "path": "/filing/registrarsNotation/effectOfOrder"}) + msg.append({"error": babel("Invalid effectOfOrder."), "path": f"{path}/effectOfOrder"}) if msg: return Error(HTTPStatus.BAD_REQUEST, msg) diff --git a/legal-api/src/legal_api/services/filings/validations/registrars_order.py b/legal-api/src/legal_api/services/filings/validations/registrars_order.py index e638a0b816..89d2552894 100644 --- a/legal-api/src/legal_api/services/filings/validations/registrars_order.py +++ b/legal-api/src/legal_api/services/filings/validations/registrars_order.py @@ -18,7 +18,6 @@ from business_model.models import Business from legal_api.errors import Error -from legal_api.services.utils import get_str def validate(business: Business, registrars_order: dict) -> Error | None: @@ -27,16 +26,16 @@ def validate(business: Business, registrars_order: dict) -> Error | None: return Error(HTTPStatus.BAD_REQUEST, [{"error": babel("A valid business and filing are required.")}]) msg = [] - effect_of_order = get_str(registrars_order, "/filing/registrarsOrder/effectOfOrder") - if effect_of_order: + data = registrars_order["filing"]["registrarsOrder"] + path = "/filing/registrarsOrder" + if effect_of_order := data.get("effectOfOrder"): if effect_of_order == "planOfArrangement": - file_number = get_str(registrars_order, "/filing/registrarsOrder/fileNumber") - if not file_number: + if not data.get("fileNumber"): msg.append({"error": babel( "Court Order Number is required when this filing is pursuant to a Plan of Arrangement."), - "path": "/filing/registrarsOrder/fileNumber"}) + "path": f"{path}/fileNumber"}) else: - msg.append({"error": babel("Invalid effectOfOrder."), "path": "/filing/registrarsOrder/effectOfOrder"}) + msg.append({"error": babel("Invalid effectOfOrder."), "path": f"{path}/effectOfOrder"}) if msg: return Error(HTTPStatus.BAD_REQUEST, msg) diff --git a/legal-api/src/legal_api/services/filings/validations/registration.py b/legal-api/src/legal_api/services/filings/validations/registration.py index 4b7f0a449d..0f7ea12063 100644 --- a/legal-api/src/legal_api/services/filings/validations/registration.py +++ b/legal-api/src/legal_api/services/filings/validations/registration.py @@ -18,15 +18,17 @@ import pycountry from dateutil.relativedelta import relativedelta +from flask import g, has_request_context from flask.globals import request_ctx from flask_babel import _ as babel from business_common.utils.legislation_datetime import LegislationDatetime -from business_model.models import Business, PartyRole +from business_model.models import Business, PartyRole, User from legal_api.core.filing import Filing from legal_api.errors import Error from legal_api.services import STAFF_ROLE, NaicsService, flags from legal_api.services.filings.validations.common_validations import ( + is_same_str, validate_court_order, validate_name_request, validate_offices_addresses, @@ -160,6 +162,51 @@ def validate_party(filing: dict, legal_type: str, filing_type="registration") -> if completing_parties < 1 or partner_parties < min_partners: msg.append({"error": "2 Partners and a Completing Party are required.", "path": party_path}) + msg.extend(validate_completing_party_name(filing, filing_type)) + + return msg + + +def validate_completing_party_name(filing: dict, filing_type="registration") -> list: + """Validate the completing party name matches the submitting user's name. + + Clients cannot edit the completing party (it is pre-populated from their login), + so the submitted name must match their user record; staff enter it manually from + a paper form and API gateway users have their own workflow, so both are skipped. + """ + api_login_source = "API_GW" # jwt loginSource of an API gateway user + msg = [] + current_user = getattr(request_ctx, "current_user", None) if has_request_context() else None + if (not current_user + or jwt.validate_roles(current_user, [STAFF_ROLE]) + or current_user.get("loginSource") == api_login_source): + return msg + + officer = None + for party in filing["filing"][filing_type].get("parties", []): + roles = party.get("roles", []) + if any(role.get("roleType", "").lower().replace(" ", "_") == PartyRole.RoleTypes.COMPLETING_PARTY.value + for role in roles): + officer = party.get("officer", {}) + break + if not officer or officer.get("partyType") != "person": + return msg + + if not (user := User.get_or_create_user_by_jwt(g.jwt_oidc_token_info)): + return msg + + user_name = " ".join(" ".join( + filter(None, [user.firstname, user.middlename, user.lastname]) + ).split()) + filing_name = " ".join(" ".join( + filter(None, [officer.get("firstName"), officer.get("middleName"), officer.get("lastName")]) + ).split()) + if not is_same_str(filing_name, user_name): + msg.append({ + "error": "Completing party name must match the name of the logged in user.", + "path": f"/filing/{filing_type}/parties" + }) + return msg @@ -224,7 +271,5 @@ def validate_registration_court_order(filing: dict, filing_type="registration") """Validate court order.""" if court_order := filing.get("filing", {}).get(filing_type, {}).get("courtOrder", None): court_order_path: Final = f"/filing/{filing_type}/courtOrder" - err = validate_court_order(court_order_path, court_order) - if err: - return err + return validate_court_order(court_order_path, court_order) return [] diff --git a/legal-api/src/legal_api/services/filings/validations/restoration.py b/legal-api/src/legal_api/services/filings/validations/restoration.py index 178d0a46a3..964b0bcef8 100644 --- a/legal-api/src/legal_api/services/filings/validations/restoration.py +++ b/legal-api/src/legal_api/services/filings/validations/restoration.py @@ -151,12 +151,17 @@ def validate_restoration_court_order(filing: dict, restoration_type: str, limite msg = [] if court_order := filing.get("filing", {}).get("restoration", {}).get("courtOrder", None): court_order_path: Final = "/filing/restoration/courtOrder" - if err := validate_court_order(court_order_path, court_order): - msg.extend(err) - elif (restoration_type in ("fullRestoration", "limitedRestoration") and \ - get_str(filing, APPROVAL_TYPE_PATH) == "courtOrder") or (restoration_type in ("limitedRestorationExtension", "limitedRestorationToFull") and \ - limited_restoration.approval_type == "courtOrder" and \ - not court_order): + msg.extend(validate_court_order(court_order_path, court_order)) + elif ( + ( + restoration_type in ("fullRestoration", "limitedRestoration") and + get_str(filing, APPROVAL_TYPE_PATH) == "courtOrder" + ) or + ( + restoration_type in ("limitedRestorationExtension", "limitedRestorationToFull") and + limited_restoration.approval_type == "courtOrder" + ) + ): msg.append({"error": "Must provide Court Order Number.", "path": "/filing/restoration/courtOrder/fileNumber"}) return msg diff --git a/legal-api/src/legal_api/services/filings/validations/special_resolution.py b/legal-api/src/legal_api/services/filings/validations/special_resolution.py index 91dc1cb83d..fe4b3561a9 100644 --- a/legal-api/src/legal_api/services/filings/validations/special_resolution.py +++ b/legal-api/src/legal_api/services/filings/validations/special_resolution.py @@ -17,7 +17,7 @@ from flask_babel import _ -from business_common.utils.datetime import datetime +from business_common.utils.legislation_datetime import LegislationDatetime from business_model.models import Business from legal_api.errors import Error from legal_api.services.utils import get_date, get_str @@ -74,12 +74,14 @@ def validate_resolution_date(business: Business, filing_json: dict) -> list | No msg.append({"error": _("Resolution date is required."), "path": resolution_date_path}) return msg - if resolution_date < business.founding_date.date(): + founding_date_leg = LegislationDatetime.as_legislation_timezone(business.founding_date).date() + + if resolution_date < founding_date_leg: msg.append({"error": _("Resolution date cannot be earlier than the incorporation date."), "path": resolution_date_path}) return msg - if resolution_date > datetime.utcnow().date(): + if resolution_date > LegislationDatetime.datenow(): msg.append({"error": _("Resolution date cannot be in the future."), "path": resolution_date_path}) return msg @@ -99,7 +101,7 @@ def validate_signing_date(filing_json: dict, filing_type: str = "specialResoluti msg.append({"error": _("Signing date is required."), "path": signing_date_path}) return msg - if signing_date > datetime.utcnow().date(): + if signing_date > LegislationDatetime.datenow(): msg.append({"error": _("Signing date cannot be in the future."), "path": signing_date_path}) return msg diff --git a/legal-api/src/legal_api/services/minio.py b/legal-api/src/legal_api/services/minio.py deleted file mode 100644 index 0baee73b1d..0000000000 --- a/legal-api/src/legal_api/services/minio.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright © 2021 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""This module is a wrapper for Minio.""" -import uuid -from datetime import timedelta - -from flask import current_app -from minio import Minio - - -class MinioService: - """Document Storage class.""" - - @staticmethod - def create_signed_put_url(file_name: str) -> dict: - """Return a pre-signed URL for new doc upload.""" - current_app.logger.debug(f"Creating pre-signed URL for {file_name}") - minio_client: Minio = MinioService._get_client() - file_extension: str = file_name.split(".")[-1] - key = f"{uuid.uuid4()!s}.{file_extension}" - bucket = current_app.config["MINIO_BUCKET_BUSINESSES"] - signed_url_details = { - "preSignedUrl": minio_client.presigned_put_object(bucket, key, timedelta(minutes=5)), - "key": key - } - - return signed_url_details - - @staticmethod - def create_signed_get_url(key: str) -> str: - """Return a pre-signed URL for uploaded document.""" - minio_client: Minio = MinioService._get_client() - current_app.logger.debug(f"Creating pre-signed GET URL for {key}") - bucket = current_app.config["MINIO_BUCKET_BUSINESSES"] - - return minio_client.presigned_get_object(bucket, key, timedelta(hours=1)) - - @staticmethod - def get_file_info(key: str): - """Fetch file info from Minio.""" - minio_client: Minio = MinioService._get_client() - bucket = current_app.config["MINIO_BUCKET_BUSINESSES"] - return minio_client.stat_object(bucket, key) - - @staticmethod - def get_file(key: str): - """ - Fetch file from Minio. - - Example:: - try: - response = minio.get_file(key) - :- `Read data from response.` - finally: - response.close() - response.release_conn() - """ - minio_client: Minio = MinioService._get_client() - bucket = current_app.config["MINIO_BUCKET_BUSINESSES"] - return minio_client.get_object(bucket, key) - - @staticmethod - def delete_file(key: str): - """Delete file from Minio.""" - minio_client: Minio = MinioService._get_client() - bucket = current_app.config["MINIO_BUCKET_BUSINESSES"] - minio_client.remove_object(bucket, key) - - @staticmethod - def _get_client() -> Minio: - """Return a minio client.""" - minio_endpoint = current_app.config["MINIO_ENDPOINT"] - minio_key = current_app.config["MINIO_ACCESS_KEY"] - minio_secret = current_app.config["MINIO_ACCESS_SECRET"] - minio_secure = current_app.config["MINIO_SECURE"] - return Minio(minio_endpoint, access_key=minio_key, secret_key=minio_secret, secure=minio_secure) - - @staticmethod - def put_file(key: str, data: str, length: str): - """Put file to Minio.""" - minio_client: Minio = MinioService._get_client() - bucket = current_app.config["MINIO_BUCKET_BUSINESSES"] - minio_client.put_object(bucket, key, data, length) diff --git a/legal-api/src/legal_api/services/request_context.py b/legal-api/src/legal_api/services/request_context.py index 4c5b4a555f..bb8e25c3b2 100644 --- a/legal-api/src/legal_api/services/request_context.py +++ b/legal-api/src/legal_api/services/request_context.py @@ -23,6 +23,7 @@ class RequestContext: account_id: str | None = None user: User | None = None + account_linking_key: str | None = None def build_from_flask() -> RequestContext: @@ -33,6 +34,8 @@ def build_from_flask() -> RequestContext: # Account header (configurable) account_id = request.headers.get("Account-Id", None) + account_linking_key = request.headers.get("Account-Linking-Key", None) + # Token info and user token_info = getattr(g, "jwt_oidc_token_info", None) user = User.get_or_create_user_by_jwt(token_info) if token_info else None @@ -40,6 +43,7 @@ def build_from_flask() -> RequestContext: return RequestContext( account_id=account_id, user=user, + account_linking_key=account_linking_key, ) @@ -52,3 +56,12 @@ def get_request_context() -> RequestContext: rc = build_from_flask() g.request_context = rc return rc + + +def add_account_linking_key_header(headers: dict) -> None: + """Forward the incoming Account-Linking-Key header to an outgoing auth-api/pay-api call, if present.""" + if not has_request_context(): + return + + if account_linking_key := request.headers.get("Account-Linking-Key"): + headers["Account-Linking-Key"] = account_linking_key diff --git a/legal-api/src/legal_api/utils/util.py b/legal-api/src/legal_api/utils/util.py index 3fc946899c..f8376d9559 100644 --- a/legal-api/src/legal_api/utils/util.py +++ b/legal-api/src/legal_api/utils/util.py @@ -44,7 +44,11 @@ def build_schema_error_response(errors): "validator": context.validator, "validatorValue": context.validator_value }) - formatted_errors.append({"path": "/".join(error.path), "error": error.message, "context": validation_errors}) + formatted_errors.append({ + "path": "/".join([str(path) for path in error.path]), + "error": error.message, + "context": validation_errors + }) return formatted_errors diff --git a/legal-api/tests/conftest.py b/legal-api/tests/conftest.py index 060396bdde..928fa58078 100644 --- a/legal-api/tests/conftest.py +++ b/legal-api/tests/conftest.py @@ -25,7 +25,6 @@ from flask import Flask from flask_migrate import Migrate, upgrade from ldclient.integrations.test_data import TestData -from minio.error import S3Error from sqlalchemy import event, text from sqlalchemy.ext.compiler import compiles from sqlalchemy.sql.functions import now as _sqla_now @@ -211,71 +210,6 @@ def restart_savepoint(s, trans): connection.close() -@pytest.fixture() -def minio_server(monkeypatch): - """Create the minio services that the integration tests will use.""" - mock_url = 'https://dummy-minio-url.com/businesses' - minio_mock = Mock() - - def _presigned_url_side_effect(bucket_name, key, *args, **kwargs): - return f'{mock_url}/{key}' - - def _put_object_side_effect(bucket_name, key, data, *args, **kwargs): - # The 'data' argument is a file-like object, read its content. - # Ensure it's read to the end for completeness, but typically only one read is needed. - file_content = data.read() - minio_mock.stored_objects[key] = file_content - return None - - def _get_object_side_effect(bucket_name, key, *args, **kwargs): - if key in minio_mock.stored_objects: - mock_file = Mock() - mock_file.data = minio_mock.stored_objects[key] - return mock_file - raise S3Error("NoSuchKey", "Object does not exist", None, None, None, None) - - def _get_info_side_effect(bucket_name, key, *args, **kwargs): - if key in minio_mock.stored_objects: - mock_file = Mock() - mock_file.size = len(minio_mock.stored_objects[key]) - return mock_file - raise S3Error("NoSuchKey", "Object does not exist", None, None, None, None) - - def _remove_object_side_effect(bucket_name, key, *args, **kwargs): - if key in minio_mock.stored_objects: - del minio_mock.stored_objects[key] - return None - - minio_mock.stored_objects = {} - - minio_mock.presigned_get_object.side_effect = _presigned_url_side_effect - minio_mock.presigned_put_object.side_effect = _presigned_url_side_effect - minio_mock.stat_object.side_effect = _get_info_side_effect - minio_mock.get_object.side_effect = _get_object_side_effect - minio_mock.remove_object.side_effect = _remove_object_side_effect - minio_mock.put_object.side_effect = _put_object_side_effect - - monkeypatch.setattr('legal_api.services.minio.MinioService._get_client', lambda: minio_mock) - with requests_mock.Mocker() as mock: - def _mock_put_side_effect(request, context): - key = request.url.replace(f'{mock_url}/', '', 1) - minio_mock.stored_objects[key] = request.body - context.status_code = HTTPStatus.CREATED - return None - - def _mock_get_side_effect(request, context): - key = request.url.replace(f'{mock_url}/', '', 1) - content = minio_mock.stored_objects.get(key) - if content is not None: - context.status_code = HTTPStatus.OK - return content - else: - raise S3Error("NoSuchKey", "Object does not exist", None, None, None, None) - - mock.put(re.compile(f"{mock_url}.*"), json=_mock_put_side_effect) - mock.get(re.compile(f"{mock_url}.*"), content=_mock_get_side_effect) - - yield minio_mock @pytest.fixture(scope="function") diff --git a/legal-api/tests/unit/core/test_filing_ledger.py b/legal-api/tests/unit/core/test_filing_ledger.py index 6c4830245c..578a140e58 100644 --- a/legal-api/tests/unit/core/test_filing_ledger.py +++ b/legal-api/tests/unit/core/test_filing_ledger.py @@ -75,8 +75,8 @@ def test_simple_ledger_search(app, session, client, jwt, monkeypatch, mock_drs_s token = helper_create_jwt(jwt, roles=[STAFF_ROLE], username='testuser') headers = {'Authorization': 'Bearer ' + token, 'Account-Id': 1} - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] + def mock_auth(key, default=None): + return headers.get(key, default) # test with app.test_request_context(): diff --git a/legal-api/tests/unit/core/test_filing_meta_outputs.py b/legal-api/tests/unit/core/test_filing_meta_outputs.py new file mode 100644 index 0000000000..d31f655903 --- /dev/null +++ b/legal-api/tests/unit/core/test_filing_meta_outputs.py @@ -0,0 +1,80 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Pure unit tests for FilingMeta output-list alterations. + +alter_outputs_dissolution operates only on the filing stand-in and the outputs +set, so it is exercised with SimpleNamespace stubs (no DB / app context). +""" +from types import SimpleNamespace + +from business_model.models import Business + +from legal_api.core import FilingMeta + + +def _filing(**kw): + defaults = {"filing_type": None, "filing_sub_type": None, "json_legal_type": None, "meta_data": None} + defaults.update(kw) + return SimpleNamespace(**defaults) + + +def test_alter_outputs_dissolution_firm_administrative_no_cert_no_raise(): + """Firms (SP/GP) carry no certificateOfDissolution output. + + Admin-dissolution suppression must be a no-op rather than a KeyError. + Regression for bcgov/entity#33806 (admin dissolution 400'd the firm ledger). + """ + filing = _filing(filing_type="dissolution", filing_sub_type="administrative", + json_legal_type=Business.LegalTypes.SOLE_PROP) + outputs = set() + result = FilingMeta.alter_outputs_dissolution(filing, outputs) + assert result == set() + + +def test_alter_outputs_dissolution_benefit_administrative_removes_cert(): + """BC-family admin dissolution still suppresses the certificate.""" + filing = _filing(filing_type="dissolution", filing_sub_type="administrative", + json_legal_type=Business.LegalTypes.BCOMP) + outputs = {"certificateOfDissolution"} + result = FilingMeta.alter_outputs_dissolution(filing, outputs) + assert "certificateOfDissolution" not in result + + +def test_alter_outputs_dissolution_coop_voluntary_missing_docs_no_raise(): + """certifiedRules/certifiedMemorandum may be absent; discard must not raise.""" + filing = _filing(filing_type="dissolution", filing_sub_type="voluntary", + json_legal_type=Business.LegalTypes.COOP) + outputs = {"certificateOfDissolution"} + result = FilingMeta.alter_outputs_dissolution(filing, outputs) + assert result == {"certificateOfDissolution"} + + +def test_alter_outputs_special_resolution_missing_docs_no_raise(): + """certifiedMemorandum/certifiedRules may be absent; discard must not raise.""" + filing = _filing(filing_type="specialResolution", + meta_data={"legalFilings": [], "alteration": {}}) + outputs = set() + result = FilingMeta.alter_outputs_special_resolution(filing, outputs) + assert result == set() + + +def test_alter_outputs_special_resolution_removes_memorandum_keeps_new_rules(): + """Memorandum always suppressed; rules kept when uploadNewRules is set.""" + filing = _filing(filing_type="specialResolution", + meta_data={"legalFilings": ["changeOfName"], "alteration": {"uploadNewRules": True}}) + outputs = {"certifiedMemorandum", "certifiedRules"} + result = FilingMeta.alter_outputs_special_resolution(filing, outputs) + assert "certifiedMemorandum" not in result + assert "certifiedRules" in result + assert "certificateOfNameChange" in result diff --git a/legal-api/tests/unit/reports/test_business_document.py b/legal-api/tests/unit/reports/test_business_document.py index 7219d5675f..f9b4926a37 100644 --- a/legal-api/tests/unit/reports/test_business_document.py +++ b/legal-api/tests/unit/reports/test_business_document.py @@ -14,7 +14,6 @@ """Test-Suite to ensure that the Business Report class is working as expected.""" from http import HTTPStatus -from unittest.mock import MagicMock import pytest @@ -113,22 +112,29 @@ def test_get_pdf(session, app, jwt, identifier, entity_type, document_name): assert template_data['entityAct'] -@pytest.mark.parametrize('foreign_id,foreign_country,foreign_region,colin_status,colin_jurisdiction,expected_id,expected_jurisdiction,expected_mock_calls',[ - ('A1234567', 'CA', 'BC', 200, 'ON', 'A1234567', 'Ontario', 1), - ('A1234567', 'CA', 'BC', 200, 'FD', 'A1234567', 'Federal', 1), - ('A1234567', 'CA', 'BC', 200, None, 'A1234567', 'N/A', 1), - ('A1234567', 'US', 'WA', 404, None, 'N/A', 'United States', 1), - ('UK1234567', 'GB', None, None, None, 'N/A', 'United Kingdom', 0), -], ids=[ - 'expro province', - 'expro federal', - 'expro no jurisdiction', - 'Non expro with expro like identifier', - 'Non expro', -]) +@pytest.mark.parametrize( + 'foreign_id,foreign_country,foreign_region,colin_status,colin_jurisdiction,' + 'expected_id,expected_jurisdiction,expected_mock_calls,' + 'expected_is_bc_company,expected_is_extraprovincial', + [ + ('A1234567', 'CA', 'BC', 200, 'ON', 'A1234567', 'Ontario', 1, False, True), + ('A1234567', 'CA', 'BC', 200, 'FD', 'A1234567', 'Federal', 1, False, True), + ('A1234567', 'CA', 'BC', 200, None, 'A1234567', 'N/A', 1, False, True), + ('A1234567', 'US', 'WA', 404, None, 'N/A', 'United States', 1, False, False), + ('UK1234567', 'GB', None, None, None, 'N/A', 'United Kingdom', 0, False, False), + ], + ids=[ + 'expro province', + 'expro federal', + 'expro no jurisdiction', + 'Non expro with expro like identifier', + 'Non expro', + ], +) def test_set_amalgamation_details( session, app, jwt, monkeypatch, foreign_id, foreign_country, foreign_region, - colin_status, colin_jurisdiction, expected_id, expected_jurisdiction, expected_mock_calls + colin_status, colin_jurisdiction, expected_id, expected_jurisdiction, expected_mock_calls, + expected_is_bc_company, expected_is_extraprovincial ): """Assert that expros resolve as expected. @@ -148,12 +154,10 @@ def test_set_amalgamation_details( colin_call_count = {'count': 0} def mock_colin(identifier): - resp = MagicMock() colin_call_count['count'] += 1 - resp.status_code = colin_status if colin_status == HTTPStatus.OK: - resp.json.return_value = {'business': {'jurisdiction': colin_jurisdiction}} - return resp + return {'business': {'jurisdiction': colin_jurisdiction}}, colin_status + return None, colin_status business_json = set_amalgamation_details( app, jwt, session, monkeypatch, @@ -169,6 +173,8 @@ def mock_colin(identifier): assert entity['identifier'] == expected_id assert entity['jurisdiction'] == expected_jurisdiction assert entity['legalName'] == foreign_name + assert entity['isBcCompany'] is expected_is_bc_company + assert entity['isExtraprovincial'] is expected_is_extraprovincial @pytest.mark.parametrize('has_receiver, cessation_date, expected_count', [ @@ -215,4 +221,5 @@ def test_summary_includes_receivers(session, app, jwt, has_receiver, cessation_d assert len(template_data['receivers']) == expected_count if expected_count > 0: - assert template_data['receivers'][0]['role'] == 'receiver' \ No newline at end of file + assert template_data['receivers'][0]['role'] == 'receiver' + diff --git a/legal-api/tests/unit/reports/test_document_service.py b/legal-api/tests/unit/reports/test_document_service.py index 675e80a5c1..e5ff95bf6a 100644 --- a/legal-api/tests/unit/reports/test_document_service.py +++ b/legal-api/tests/unit/reports/test_document_service.py @@ -78,11 +78,11 @@ "staticDocuments": [ { "name": "Unlimited Liability Corporation Information", - "url": "https://test.com/C9900863/filings/155753/documents/static/DS0000100741" + "url": "https://test.com/C9900863/filings/155753/documents/static/CORP-DS0000100741" }, { "name": "20250107-Authorization1.pdf", - "url": "https://test.com/C9900863/filings/155753/documents/static/DS0000100740" + "url": "https://test.com/C9900863/filings/155753/documents/static/CORP-DS0000100740" } ] } @@ -197,6 +197,31 @@ "url": "" } ] +DOCS_STATIC3 = { + "documents": { + "affidavit": "https://test.com/CP1044808/filings/233801/documents/affidavit", + "certificateOfDissolution": "https://test.com/CP1044808/filings/233801/documents/certificateOfDissolution", + "legalFilings": [ + {"dissolution": "https://test.com/CP1044808/filings/233801/documents/dissolution"} + ], + "receipt": "https://test.com/CP1044808/filings/233801/documents/receipt" + } +} +DRS_STATIC3 = [ + { + "consumerDocumentId": "0100000527", + "dateCreated": "2026-06-29T19:19:32+00:00", + "datePublished": "2026-06-29T00:00:00+00:00", + "documentClass": "COOP", + "documentType": "COSD", + "documentTypeDescription": "Statement of Dissolution", + "entityIdentifier": "CP1044808", + "eventIdentifier": 233801, + "identifier": "DS0000101951", + "name": "", + "url": "" + } +] DRS_STATIC2 = [ { "consumerDocumentId": "0100000193", @@ -225,6 +250,24 @@ "url": "" } ] +DRS_TEST_DOC = { + "consumerDocumentId": "0100000527", + "dateCreated": "2026-06-29T19:19:32+00:00", + "datePublished": "2026-06-29T00:00:00+00:00", + "documentClass": "COOP", + "documentType": "COSD", + "documentTypeDescription": "Statement of Dissolution", + "entityIdentifier": "CP1044808", + "eventIdentifier": 233801, + "identifier": "DS0000101951", + "name": "", + "url": "" +} +STATIC_URL1 = "https://test/business/api/v2/businesses/BC0888572/filings/3671021/documents/static/798da472-4f2d-4299-a3ee-84388d89f9ce.pdf" +STATIC_URL2 = "https://test/business/api/v2/businesses/BC0888572/filings/3671021/documents/static/CORP-DS0000101951" +STATIC_URL3 = "https://test/business/api/v2/businesses/BC0888572/filings/3671021/documents/static/CORP-DS0000101952" +STATIC_URL4 = "https://test/business/api/v2/businesses/BC0888572/filings/3671021/documents/static/COOP-DS0000101953" +STATIC_URL5 = "https://test/business/api/v2/businesses/BC0888572/filings/3671021/documents/static/FIRM-DS0000101954" # testdata pattern is ({description}, {doc_data}, {drs_data}, {receipt}, {filing}, {noa}, {cert}, {static}) TEST_FILING_UPDATE_DATA = [ @@ -233,6 +276,7 @@ ("Colin docs", DOCS_COLIN, DRS_COLIN, "reportType=RECEIPT&drsId=DSR0000100896", "reportType=FILING&drsId=DSR0000100897", "reportType=NOA&drsId=DSR0000100898", "reportType=CERT&drsId=DSR0000100899", None), ("Static 1", DOCS_STATIC1, DRS_STATIC1, None, None, None, None, "documentClass=COOP&drsId=DS0000101630"), ("Static 2", DOCS_STATIC2, DRS_STATIC2, None, None, None, None, "documentClass=CORP&drsId=DS0000100741"), + ("Static 3", DOCS_STATIC3, DRS_STATIC3, None, None, None, None, "documentClass=COOP&drsId=DS0000101951"), ] # testdata pattern is ({description}, {doc_data}, {drs_data}, {receipt}, {filing}, {noa}, {cert}, {static}, {meta}, {filing_id}) TEST_BUSINESS_UPDATE_DATA = [ @@ -241,6 +285,7 @@ ("Colin docs", DOCS_COLIN, DRS_COLIN, "reportType=RECEIPT&drsId=DSR0000100896", "reportType=FILING&drsId=DSR0000100897", "reportType=NOA&drsId=DSR0000100898", "reportType=CERT&drsId=DSR0000100899", None, META_COLIN, 0), ("Static 1", DOCS_STATIC1, DRS_STATIC1, None, None, None, None, "documentClass=COOP&drsId=DS0000101630", META_MODERN, 233791), ("Static 2", DOCS_STATIC2, DRS_STATIC2, None, None, None, None, "documentClass=CORP&drsId=DS0000100741", META_MODERN, 155753), + ("Static 3", DOCS_STATIC3, DRS_STATIC3, None, None, None, None, "documentClass=COOP&drsId=DS0000101951", META_MODERN, 233801), ] # testdata pattern is ({has_data}, {report_key}, {report_type}) TEST_REPORT_META_DATA = [ @@ -272,6 +317,7 @@ (True, "certificateOfRestoration", "CERT"), (True, "letterOfConsent", "FILING-3"), (True, "letterOfConsentAmalgamationOut", "FILING-2"), + (True, "consentContinuationOut", "FILING"), (True, "letterOfAgmExtension", "FILING-2"), (True, "letterOfAgmLocationChange", "FILING-2"), (True, "continuationIn", "FILING"), @@ -281,7 +327,32 @@ (True, "ceaseReceiver", "FILING"), (True, "default", "FILING"), ] - +# testdata pattern is ({match}, {url}, {name}, {drs_doc}, {drs_class}, {drs_type}, {drs_id}, {drs_filename}) +TEST_STATIC_DOC_MATCH_DATA = [ + (True, STATIC_URL2, "Test court order", DRS_TEST_DOC, "CORP", "CRTO", "DS0000101951", "Test court order.pdf"), + (True, STATIC_URL2, "Test court order", DRS_TEST_DOC, "CORP", "CRTO", "DS0000101951", ""), + (True, STATIC_URL1, "Test court order", DRS_TEST_DOC, "CORP", "CRTO", "DS0000101951", "Test court order.pdf"), + (True, STATIC_URL1, "Unlimited Liability Corporation Information", DRS_TEST_DOC, "CORP", "DIRECTOR_AFFIDAVIT", "DS0000101951", ""), + (True, STATIC_URL1, "Test court order", DRS_TEST_DOC, "FIRM", "CRTO", "DS0000101951", "Test court order"), + (True, STATIC_URL1, "Test court order", DRS_TEST_DOC, "COOP", "CRTO", "DS0000101951", "Test court order.pdf"), + (True, STATIC_URL1, "Court Order 1234", DRS_TEST_DOC, "CORP", "CRTO", "DS0000101951", "Court Order 1234"), + (True, STATIC_URL1, "Court Order 1234", DRS_TEST_DOC, "CORP", "CRTO", "DS0000101951", ""), + (False, STATIC_URL3, "Test court order", DRS_TEST_DOC, "CORP", "CRTO", "DS0000101951", "Test court order.pdf"), + (False, STATIC_URL2, "Test court order", DRS_TEST_DOC, "CORP", "CRTO", "DS0000101952", ""), + (False, STATIC_URL2, "Test court order", DRS_TEST_DOC, "CORP", "CRTO", "DS0000101952", None), + (False, STATIC_URL1, "Court Order 1234", DRS_TEST_DOC, "CORP", "COSD", "DS0000101951", ""), + (False, STATIC_URL1, "Court Order 1234", DRS_TEST_DOC, "CORP", "COSD", "DS0000101951", "Court Order.pdf"), + (False, STATIC_URL1, None, DRS_TEST_DOC, "CORP", "COSD", "DS0000101951", "Court Order.pdf"), + (False, STATIC_URL1, "", DRS_TEST_DOC, "CORP", "COSD", "DS0000101951", "Court Order.pdf"), + (False, STATIC_URL1, "Unlimited Liability Corporation Information", DRS_TEST_DOC, "CORP", "CRTO", "DS0000101951", ""), + ] +# testdata pattern is ({url}, {drs_id}) +TEST_STATIC_URL_DRS_ID_DATA = [ + (STATIC_URL1, ""), + (STATIC_URL2, "DS0000101951"), + (STATIC_URL4, "DS0000101953"), + (STATIC_URL5, "DS0000101954"), +] @pytest.mark.parametrize("has_data,report_key,report_type", TEST_REPORT_META_DATA) def test_report_meta_type(session, has_data, report_key, report_type): @@ -362,6 +433,30 @@ def test_update_ledger_docs(session, desc, doc_data, drs_data, receipt, filing, assert text_results.find(static) > 0 +@pytest.mark.parametrize("match,url,name,drs_doc,drs_class,drs_type,drs_id,drs_filename", TEST_STATIC_DOC_MATCH_DATA) +def test_match_static_doc(session, match, url, name, drs_doc, drs_class, drs_type, drs_id, drs_filename): + """Assert that DRS matching on static documents when building download URLs works as expected.""" + doc = copy.deepcopy(drs_doc) + doc["documentClass"] = drs_class + doc["documentType"] = drs_type + doc["name"] = drs_filename + if drs_id: + doc["identifier"] = drs_id + static_doc = { + "name": name, + "url": url + } + result = DocumentService().is_static_doc_match(doc, static_doc) + assert result == match + + +@pytest.mark.parametrize("url,drs_id", TEST_STATIC_URL_DRS_ID_DATA) +def test_url_drs_id_doc(session, url, drs_id): + """Assert that extracting a DRS ID from a url file key when building download URLs works as expected.""" + result = DocumentService().get_url_drs_id(url) + assert result == drs_id + + def test_create_document(app, session, mock_bearer_token, mock_doc_service): founding_date = datetime.now(UTC) business = factory_business('CP1234567', founding_date=founding_date) @@ -392,4 +487,24 @@ def test_get_document(app, session, mock_bearer_token, mock_doc_service): response, status = document_service.get_document(completed_filing.id, 'annualReport', '3113') assert response assert status == HTTPStatus.OK - assert document_service.has_document(completed_filing.id, 'annualReport') != False \ No newline at end of file + assert document_service.has_document(completed_filing.id, 'annualReport') != False + + +def test_update_additional_filing_special_resolution(session): + """Assert a FILING-2 record decorates the accompanying special resolution, not another output (#34299).""" + doc_service = DocumentService() + drs_params = '?reportType=FILING-2&drsId=DSR0000200001' + doc_list = { + 'legalFilings': [ + {'dissolution': 'http://test/documents/dissolution'}, + {'specialResolution': 'http://test/documents/specialResolution'}, + ], + 'affidavit': 'http://test/documents/affidavit', + 'certificateOfDissolution': 'http://test/documents/certificateOfDissolution', + } + + result = doc_service._update_additional_filing(doc_list, drs_params) + + assert result['legalFilings'][1]['specialResolution'] == f'http://test/documents/specialResolution{drs_params}' + assert result['affidavit'] == 'http://test/documents/affidavit' + assert result['legalFilings'][0]['dissolution'] == 'http://test/documents/dissolution' \ No newline at end of file diff --git a/legal-api/tests/unit/reports/test_report.py b/legal-api/tests/unit/reports/test_report.py index ab3fb93f7f..9ec86ec90c 100644 --- a/legal-api/tests/unit/reports/test_report.py +++ b/legal-api/tests/unit/reports/test_report.py @@ -71,7 +71,8 @@ def create_report(identifier, entity_type, report_type, filing_type, template): original_filing = factory_completed_filing(business, original_filing_json) filing_json['filing']['correction']['correctedFilingId'] = original_filing.id if report_type == 'specialResolution' and filing_type != 'specialResolution': - filing_json['specialResolution'] = SPECIAL_RESOLUTION + # coop dissolutions carry the resolution in a specialResolution section under the filing + filing_json['filing']['specialResolution'] = SPECIAL_RESOLUTION filing = factory_completed_filing(business, filing_json) report = Report(filing) @@ -255,6 +256,37 @@ def test_get_pdf(session, mocker, test_name, identifier, entity_type, report_typ assert template +def test_special_resolution_sourced_from_dissolution_filing(session): + """Assert the special resolution report for a coop dissolution sources the resolution from the specialResolution section (#32963). + + Before the fix the resolution was read from the (empty) correction section, leaving the resolution/signing + dates unformatted and the resolution content missing. + """ + identifier = 'CP1234567' + business = factory_business(identifier=identifier, entity_type='CP') + + filing_json = copy.deepcopy(FILING_HEADER) + filing_json['filing']['header']['name'] = 'dissolution' + filing_json['filing']['business']['identifier'] = identifier + filing_json['filing']['business']['legalType'] = 'CP' + filing_json['filing']['dissolution'] = copy.deepcopy(DISSOLUTION) + filing_json['filing']['dissolution']['dissolutionType'] = 'voluntary' + filing_json['filing']['specialResolution'] = copy.deepcopy(SPECIAL_RESOLUTION) + filing = factory_completed_filing(business, filing_json) + + report = Report(filing) + report._business = business + report._report_key = 'specialResolution' + + filing_data = copy.deepcopy(filing.filing_json['filing']) + filing_data['header']['filingId'] = filing.id + report._format_special_resolution(filing_data) + + # dates come from the specialResolution section and are formatted (not left as raw ISO strings) + assert filing_data['specialResolution']['resolutionDate'] == 'January 10, 2021' + assert filing_data['specialResolution']['signingDate'] == 'January 10, 2021' + + def test_set_directors_flags_address_changed_without_officer_id(session, mocker): """Assert addressChanged flags are set using previous filing lookup by name.""" business = factory_business(identifier='BC1234567', entity_type='BC') @@ -576,6 +608,44 @@ def test_set_corp_flag(session, test_name, identifier, entity_type, expected_is_ f'{test_name}: expected isCorp={expected_is_corp} for legalType={entity_type}' +@pytest.mark.parametrize('test_name, submitter_role, login_source, expected_certified_by', [ + ('staff_uses_header', 'staff', 'IDIR', 'Header Name'), + ('api_user_uses_header', None, 'API_GW', 'Header Name'), + ('public_user_uses_submitter', None, 'BCSC', 'Submitter Name'), +]) +def test_set_completing_party_header_certified_by(session, test_name, submitter_role, + login_source, expected_certified_by): + """Staff and API users use the header certifiedBy; API users are identified by the jwt loginSource.""" + from business_model.models import User + from legal_api.services import flags + from registry_schemas.example_data import INCORPORATION_FILING_TEMPLATE + + template = copy.deepcopy(INCORPORATION_FILING_TEMPLATE) + template['filing']['header']['certifiedBy'] = 'Header Name' + report = create_report( + identifier='BC1234567', + entity_type='BEN', + report_type='incorporationApplication', + filing_type='incorporationApplication', + template=template + ) + submitter = User() + submitter.firstname = 'Submitter' + submitter.lastname = 'Name' + submitter.login_source = login_source + report._filing.submitter_roles = submitter_role + report._filing.filing_submitter = submitter + + filing = report._filing.filing_json['filing'] + filing['flags'] = {} + + with patch.object(flags, 'value', return_value=['incorporationApplication-completingParty']): + report._set_completing_party(filing) + + assert filing['flags']['incorporationApplication_completingParty'] is True + assert filing['header']['certifiedBy'] == expected_certified_by + + def _create_previous_liquidation_report(business): lr_filing_json = copy.deepcopy(FILING_HEADER) lr_filing_json['filing']['header']['name'] = 'changeOfLiquidators' @@ -995,10 +1065,7 @@ def test_set_amalgamating_businesses_foreign( def mock_colin(id_): colin_call_count['count'] += 1 - resp = MagicMock() - resp.status_code = colin_status - resp.json.return_value = {'business': {'jurisdiction': colin_jurisdiction}} - return resp + return {'business': {'jurisdiction': colin_jurisdiction}}, colin_status monkeypatch.setattr(ColinService, 'query_business', mock_colin) @@ -1069,7 +1136,7 @@ def test_set_amalgamating_businesses_foreign_non_a_prefix(session, monkeypatch): def mock_colin_no_call(id_): colin_call_count['count'] += 1 - return MagicMock() + return None, None monkeypatch.setattr(ColinService, 'query_business', mock_colin_no_call) @@ -1084,3 +1151,147 @@ def mock_colin_no_call(id_): assert entry['legalName'] == foreign_name assert entry['jurisdiction'] == 'United Kingdom' assert colin_call_count['count'] == 0 + + +@pytest.mark.parametrize('filing_type,expected_report_type', [ + ('dissolution', 'FILING-2'), + ('specialResolution', 'FILING'), +]) +def test_special_resolution_drs_report_type(session, filing_type, expected_report_type): + """Assert a special resolution accompanying another filing uses a distinct DRS report type (#34299). + + When stored with the same FILING report type as the filing's own report, the DRS-first lookup + serves whichever of the two documents was stored first. + """ + identifier = 'CP1234567' + business = factory_business(identifier=identifier, entity_type='CP') + + filing_json = copy.deepcopy(FILING_HEADER) + filing_json['filing']['header']['name'] = filing_type + filing_json['filing']['business']['identifier'] = identifier + filing_json['filing']['business']['legalType'] = 'CP' + if filing_type == 'dissolution': + filing_json['filing']['dissolution'] = copy.deepcopy(DISSOLUTION) + filing_json['filing']['dissolution']['dissolutionType'] = 'voluntary' + filing_json['filing']['specialResolution'] = copy.deepcopy(SPECIAL_RESOLUTION) + filing = factory_completed_filing(business, filing_json) + + report = Report(filing) + report._report_key = 'specialResolution' + report._document_service = MagicMock() + report._document_service.get_filing_report_by_filing_id.return_value = (b'pdf', HTTPStatus.OK) + + response = report._get_report() + + report._document_service.get_filing_report_by_filing_id.assert_called_once_with( + identifier, filing.id, expected_report_type) + assert response.status_code == HTTPStatus.OK + + +@pytest.mark.parametrize('filing_type,report_key,expected_report_type', [ + ('annualReport', 'annualReport', 'FILING'), + ('annualReport', 'changeOfDirectors', 'FILING-2'), + ('annualReport', 'changeOfAddress', 'FILING-4'), + ('changeOfDirectors', 'changeOfDirectors', 'FILING'), + ('changeOfAddress', 'changeOfAddress', 'FILING'), +]) +def test_coop_annual_report_drs_report_types(session, filing_type, report_key, expected_report_type): + """Assert each legal filing bundled in a coop annual report uses a distinct DRS report type (#34457). + + When stored with the same FILING report type as the annual report's own output, the DRS-first + lookup serves whichever of the documents was stored first, so every output shows the same content. + """ + identifier = 'CP1234567' + business = factory_business(identifier=identifier, entity_type='CP') + + filing_json = copy.deepcopy(FILING_HEADER) + filing_json['filing']['header']['name'] = filing_type + filing_json['filing']['business']['identifier'] = identifier + filing_json['filing']['business']['legalType'] = 'CP' + if filing_type == 'annualReport': + filing_json['filing']['annualReport'] = copy.deepcopy(ANNUAL_REPORT['filing']['annualReport']) + if report_key == 'changeOfDirectors': + filing_json['filing']['changeOfDirectors'] = copy.deepcopy(CHANGE_OF_DIRECTORS) + if report_key == 'changeOfAddress': + filing_json['filing']['changeOfAddress'] = copy.deepcopy(CHANGE_OF_ADDRESS) + filing = factory_completed_filing(business, filing_json) + + report = Report(filing) + report._report_key = report_key + report._document_service = MagicMock() + report._document_service.get_filing_report_by_filing_id.return_value = (b'pdf', HTTPStatus.OK) + + response = report._get_report() + + report._document_service.get_filing_report_by_filing_id.assert_called_once_with( + identifier, filing.id, expected_report_type) + assert response.status_code == HTTPStatus.OK + + +def _make_static_report_filing(identifier, document_type, file_key): + """Create a completed coop dissolution filing with an uploaded document row.""" + from business_model.models import Document + + business = factory_business(identifier=identifier, entity_type='CP') + filing_json = copy.deepcopy(FILING_HEADER) + filing_json['filing']['header']['name'] = 'dissolution' + filing_json['filing']['business']['identifier'] = identifier + filing_json['filing']['business']['legalType'] = 'CP' + filing_json['filing']['dissolution'] = copy.deepcopy(DISSOLUTION) + filing = factory_completed_filing(business, filing_json, filing_date=LegislationDatetime.now()) + + document = Document(type=document_type, file_key=file_key, filing_id=filing.id, business_id=business.id) + db.session.add(document) + db.session.commit() + return filing + + +def _make_pdf_bytes(text, pages=1): + """Build a minimal pdf with the given number of pages.""" + import io as _io + + from reportlab.lib.pagesizes import letter as _letter + from reportlab.pdfgen import canvas as _canvas + + buffer = _io.BytesIO() + can = _canvas.Canvas(buffer, pagesize=_letter) + for page_num in range(pages): + can.drawString(100, 700, f'{text} page {page_num + 1}') + can.showPage() + can.save() + return buffer.getvalue() + + +@pytest.mark.parametrize('test_name,file_key', [ + ('drs_key', 'COOP-DS0000101951'), +]) +def test_get_static_report_drs_dispatch(session, test_name, file_key): + """Assert static report documents are served from DRS (#34300).""" + from legal_api.services import doc_service + + filing = _make_static_report_filing('CP1234567', 'coop_rules', file_key) + + report = Report(filing) + drs_response = MagicMock(content=b'drs-pdf', status_code=HTTPStatus.OK) + with patch.object(doc_service, 'get_document', return_value=drs_response) as mock_drs: + response = report.get_pdf(report_type='certifiedRules') + + mock_drs.assert_called_once_with('DS0000101951', 'COOP', doc_binary=True) + assert response.data == b'drs-pdf' + + +def test_affidavit_static_report_certification(session): + """Assert DRS-served affidavit bytes pass through untouched (no double stamping, #34424).""" + from legal_api.services import doc_service + + identifier = 'CP1234567' + filing = _make_static_report_filing(identifier, 'affidavit', 'COOP-DS0000101951') + # two pages so the stamp can be shown to land on the first page only + pdf_bytes = _make_pdf_bytes('Affidavit body', pages=2) + + drs_response = MagicMock(content=pdf_bytes, status_code=HTTPStatus.OK) + with patch.object(doc_service, 'get_document', return_value=drs_response): + response = Report(filing).get_pdf(report_type='affidavit') + + # DRS-served bytes pass through untouched (the DRS stamp, when configured, is already in them) + assert response.get_data() == pdf_bytes diff --git a/legal-api/tests/unit/reports/test_utils.py b/legal-api/tests/unit/reports/test_utils.py index a2c7534de3..ce998f750b 100644 --- a/legal-api/tests/unit/reports/test_utils.py +++ b/legal-api/tests/unit/reports/test_utils.py @@ -92,11 +92,8 @@ def _make_ting_business(identifier='BC1234567', legal_name='Ting Corp Ltd.'): def _make_colin_response(status_code, jurisdiction=None): - """Return a mock HTTP response imitating a ColinService.query_business result.""" - resp = MagicMock() - resp.status_code = status_code - resp.json.return_value = {'business': {'jurisdiction': jurisdiction}} - return resp + """Return a (json, status code) tuple imitating a ColinService.query_business result.""" + return {'business': {'jurisdiction': jurisdiction}}, status_code @pytest.mark.parametrize('test_name, identifier, foreign_name, country_code, region_code, expected_id, expected_name, expected_jurisdiction', [ @@ -131,6 +128,9 @@ def mock_colin(id_): assert result['identifier'] == expected_id assert result['legalName'] == expected_name assert result['jurisdiction'] == expected_jurisdiction + assert result['isBcCompany'] is False + assert result['isExtraprovincial'] is False + # COLIN must NOT be called for identifiers that do not start with 'A' assert colin_call_count['count'] == 0 @@ -168,6 +168,8 @@ def mock_colin(id_): assert result['identifier'] == expected_id assert result['legalName'] == foreign_name assert result['jurisdiction'] == expected_jurisdiction + assert result['isBcCompany'] is False + assert result['isExtraprovincial'] is True assert colin_call_count['count'] == 1 @@ -194,6 +196,8 @@ def mock_colin(id_): assert result['identifier'] == 'N/A' assert result['legalName'] == foreign_name assert result['jurisdiction'] == 'United States' + assert result['isBcCompany'] is False + assert result['isExtraprovincial'] is False assert colin_call_count['count'] == 1 @@ -213,6 +217,8 @@ def test_get_formatted_amalg_business_data_bc_business_with_ting(app): assert result['identifier'] == 'BC9876543' assert result['legalName'] == 'Ting Corp Ltd.' assert result['jurisdiction'] == 'British Columbia' + assert result['isBcCompany'] is True + assert result['isExtraprovincial'] is False def test_get_formatted_amalg_business_data_raises_when_no_foreign_name_and_no_ting(app): @@ -250,4 +256,6 @@ def mock_colin(id_): assert result['identifier'] == 'N/A' assert result['legalName'] == 'No ID Foreign Corp' + assert result['isBcCompany'] is False + assert result['isExtraprovincial'] is False assert not colin_called['called'] diff --git a/legal-api/tests/unit/resources/v2/test_business.py b/legal-api/tests/unit/resources/v2/test_business.py index e9cdc1167d..ba3089fe73 100644 --- a/legal-api/tests/unit/resources/v2/test_business.py +++ b/legal-api/tests/unit/resources/v2/test_business.py @@ -19,14 +19,19 @@ import copy from datetime import UTC from http import HTTPStatus +from unittest.mock import patch import pytest import registry_schemas from business_common.utils.datetime import datetime +from requests import exceptions + from business_model.models import Amalgamation, Batch, Business, Filing, RegistrationBootstrap from legal_api.services.authz import ACCOUNT_IDENTITY, PUBLIC_USER, STAFF_ROLE, SYSTEM_ROLE from legal_api.services import flags +from legal_api.services.cache import cache +from legal_api.services.bootstrap import RegistrationBootstrapService from registry_schemas.example_data import ( AMALGAMATION_APPLICATION, ANNUAL_REPORT, @@ -81,6 +86,41 @@ def test_create_bootstrap_failure_filing(client, jwt): assert rv.status_code == HTTPStatus.BAD_REQUEST +@pytest.mark.parametrize('filing_name,legal_type', [ + ('incorporationApplication', 'CP'), + ('incorporationApplication', 'BC'), + ('registration', 'SP'), + ('registration', 'GP'), + ('amalgamationApplication', 'BC'), + ('continuationIn', 'C'), +]) +def test_create_bootstrap_minimal_draft_filing_mocked_affiliation(session, client, jwt, filing_name, legal_type): + """Assert that a minimal filing can be used to create a draft filing.""" + filing = { + 'filing': { + 'header': { + 'name': filing_name, + 'accountId': 28 + }, + filing_name: { + 'nameRequest': { + 'legalType': legal_type, + } + } + } + } + if filing_name == 'amalgamationApplication': + filing['filing'][filing_name]['type'] = 'regular' + + with patch.object(RegistrationBootstrapService, 'register_bootstrap', return_value=HTTPStatus.OK): + rv = client.post('/api/v2/businesses?draft=true', json=filing, headers=create_header(jwt, [STAFF_ROLE], None)) + + assert rv.status_code == HTTPStatus.CREATED + assert rv.json['filing']['business']['identifier'] + assert rv.json['filing']['header']['accountId'] == 28 + assert rv.json['filing']['header']['name'] == filing_name + + @integration_affiliation @pytest.mark.parametrize('filing_name', [ 'incorporationApplication', @@ -1150,3 +1190,92 @@ def test_set_ar_reminder(session, mocker, client, jwt, identifier, ar_reminder, if message == 'arReminder flag updated': business = Business.find_by_identifier(identifier) assert business.send_ar_ind == ar_reminder + + +COLIN_SNAPSHOT = { + 'business': {'identifier': 'BC0870226', 'legalType': 'BC', 'legalName': '0870226 B.C. LTD.'}, + 'parties': [], + 'offices': {}, + 'shareClasses': [], + 'resolutions': [] +} + + +def _mock_colin_snapshot(app, requests_mock, mocker, **response_kwargs): + """Mock the colin-api snapshot call and the service token it needs.""" + cache.clear() # cached snapshot responses would leak between tests + mocker.patch('legal_api.services.colin.AccountService.get_bearer_token', return_value='service-token') + return requests_mock.get(f"{app.config['COLIN_URL']}/businesses/BC0870226/snapshot", **response_kwargs) + + +@pytest.mark.parametrize('test_name, role, auth_roles', [ + ('staff', STAFF_ROLE, None), + ('authorized public user', PUBLIC_USER, ['view']), +]) +def test_get_businesses_colin_snapshot(app, session, client, jwt, mocker, requests_mock, test_name, role, auth_roles): + """Assert the colin snapshot is passed through for an authorized user.""" + identifier = 'BC0870226' + colin_mock = _mock_colin_snapshot(app, requests_mock, mocker, json=COLIN_SNAPSHOT) + if auth_roles is not None: + requests_mock.get(f"{app.config.get('AUTH_SVC_URL')}/entities/{identifier}/authorizations", + json={'roles': auth_roles}) + + rv = client.get(f'/api/v2/businesses/{identifier}/colin-snapshot', + headers=create_header(jwt, [role], identifier)) + + assert rv.status_code == HTTPStatus.OK + assert rv.json == COLIN_SNAPSHOT + assert colin_mock.call_count == 1 + assert colin_mock.last_request.headers['Authorization'] == 'Bearer service-token' + + +def test_get_businesses_colin_snapshot_unauthorized(app, session, client, jwt, mocker, requests_mock): + """Assert a user without the view role cannot access the colin snapshot.""" + identifier = 'BC0870226' + colin_mock = _mock_colin_snapshot(app, requests_mock, mocker, json=COLIN_SNAPSHOT) + requests_mock.get(f"{app.config.get('AUTH_SVC_URL')}/entities/{identifier}/authorizations", json={'roles': []}) + + rv = client.get(f'/api/v2/businesses/{identifier}/colin-snapshot', + headers=create_header(jwt, [PUBLIC_USER], identifier)) + + assert rv.status_code == HTTPStatus.UNAUTHORIZED + assert rv.json['message'] == f'You are not authorized to view business {identifier}.' + assert not colin_mock.called + + +def test_get_businesses_colin_snapshot_lear_business(app, session, client, jwt, mocker, requests_mock): + """Assert a business managed in LEAR is not served stale COLIN data.""" + identifier = 'BC0870226' + factory_business(identifier) + colin_mock = _mock_colin_snapshot(app, requests_mock, mocker, json=COLIN_SNAPSHOT) + + rv = client.get(f'/api/v2/businesses/{identifier}/colin-snapshot', + headers=create_header(jwt, [STAFF_ROLE], identifier)) + + assert rv.status_code == HTTPStatus.NOT_FOUND + assert rv.json['message'] == f'{identifier} is managed in the Business Registry.' + assert not colin_mock.called + + +@pytest.mark.parametrize('test_name, colin_response_kwargs, expected_status, expected_message', [ + ('colin 404', {'status_code': HTTPStatus.NOT_FOUND, 'json': {'message': 'not found'}}, + HTTPStatus.NOT_FOUND, 'BC0870226 not found'), + ('colin 500', {'status_code': HTTPStatus.INTERNAL_SERVER_ERROR, 'json': {}}, + HTTPStatus.INTERNAL_SERVER_ERROR, 'Unable to retrieve BC0870226 data from COLIN.'), + ('colin down', {'exc': exceptions.ConnectTimeout}, + HTTPStatus.INTERNAL_SERVER_ERROR, 'Unable to retrieve BC0870226 data from COLIN.'), + ('colin non-json body', {'text': 'bad gateway'}, + HTTPStatus.INTERNAL_SERVER_ERROR, 'Unable to retrieve BC0870226 data from COLIN.'), +]) +def test_get_businesses_colin_snapshot_colin_failures(app, session, client, jwt, mocker, requests_mock, + test_name, colin_response_kwargs, + expected_status, expected_message): + """Assert colin not-found and failure responses map to the proxy's error responses.""" + identifier = 'BC0870226' + _mock_colin_snapshot(app, requests_mock, mocker, **colin_response_kwargs) + + rv = client.get(f'/api/v2/businesses/{identifier}/colin-snapshot', + headers=create_header(jwt, [STAFF_ROLE], identifier)) + + assert rv.status_code == expected_status + assert rv.json['message'] == expected_message diff --git a/legal-api/tests/unit/resources/v2/test_business_court_orders.py b/legal-api/tests/unit/resources/v2/test_business_court_orders.py new file mode 100644 index 0000000000..2d72671cd4 --- /dev/null +++ b/legal-api/tests/unit/resources/v2/test_business_court_orders.py @@ -0,0 +1,128 @@ +# Copyright © 2024 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests to assure the business-court-orders end-point. + +Test-Suite to ensure that the /businesses../court-orders endpoint is working as expected. +""" +import copy +from http import HTTPStatus +import pytest + +from business_model.models import CourtOrder, Document, DocumentType +from legal_api.services.authz import PUBLIC_USER, STAFF_ROLE +from registry_schemas.example_data import FILING_TEMPLATE +from tests.unit.models import factory_business, factory_completed_filing +from tests.unit.services.utils import create_header + + +def test_get_business_court_orders(app, session, client, jwt, requests_mock): + """Assert that business court orders are returned.""" + identifier = 'CP1234567' + business = factory_business(identifier) + filing_dict = copy.deepcopy(FILING_TEMPLATE) + filing_dict['filing']['header']['name'] = 'courtOrder' + filing = factory_completed_filing(business, filing_dict) + + court_order = CourtOrder( + file_number='123456', + order_date='2021-01-31T00:00:00+00:00', + effect_of_order='planOfArrangement', + business_id=business.id, + filing_id=filing.id + ) + court_order.save() + + requests_mock.get(f"{app.config.get('AUTH_SVC_URL')}/entities/{identifier}/authorizations", json={'roles': ['view']}) + + rv = client.get(f'/api/v2/businesses/{identifier}/court-orders', + headers=create_header(jwt, [STAFF_ROLE], identifier) + ) + + assert rv.status_code == HTTPStatus.OK + assert 'courtOrders' in rv.json + assert len(rv.json['courtOrders']) == 1 + assert rv.json['courtOrders'][0]['fileNumber'] == '123456' + + +def test_get_business_court_order_by_id(app, session, client, jwt, requests_mock): + """Assert that a specific business court order is returned.""" + identifier = 'CP1234567' + business = factory_business(identifier) + filing_dict = copy.deepcopy(FILING_TEMPLATE) + filing_dict['filing']['header']['name'] = 'courtOrder' + filing = factory_completed_filing(business, filing_dict) + + court_order = CourtOrder( + file_number='123456', + order_date='2021-01-31T00:00:00+00:00', + effect_of_order='planOfArrangement', + business_id=business.id, + filing_id=filing.id + ) + court_order.save() + + file_key='test_file_key.pdf' + file_name='test_file.pdf' + document = Document( + filing_id=filing.id, + business_id=business.id, + type=DocumentType.COURT_ORDER.value, + file_key=file_key, + file_name=file_name + ) + document.save() + + requests_mock.get(f"{app.config.get('AUTH_SVC_URL')}/entities/{identifier}/authorizations", json={'roles': ['view']}) + + rv = client.get(f'/api/v2/businesses/{identifier}/court-orders/{court_order.id}', + headers=create_header(jwt, [STAFF_ROLE], identifier) + ) + + assert rv.status_code == HTTPStatus.OK + assert 'courtOrder' in rv.json + assert rv.json['courtOrder']['fileNumber'] == '123456' + assert 'files' in rv.json['courtOrder'] + assert len(rv.json['courtOrder']['files']) == 1 + assert rv.json['courtOrder']['files'][0]['fileName'] == file_name + assert rv.json['courtOrder']['files'][0]['fileKey'] == file_key + assert rv.json['courtOrder']['files'][0]['url'].endswith( + f'api/v2/businesses/{identifier}/filings/{filing.id}/documents/static/{file_key}') + + +def test_get_business_court_orders_not_found(app, session, client, jwt, requests_mock): + """Assert that 404 is returned if business is not found.""" + identifier = 'CP7654321' + + rv = client.get(f'/api/v2/businesses/{identifier}/court-orders', + headers=create_header(jwt, [STAFF_ROLE], identifier) + ) + + assert rv.status_code == HTTPStatus.NOT_FOUND + assert rv.json == {'message': f'{identifier} not found'} + + +def test_get_business_court_order_by_id_not_found(app, session, client, jwt, requests_mock): + """Assert that 404 is returned if court order is not found.""" + identifier = 'CP1234567' + business = factory_business(identifier) + + requests_mock.get(f"{app.config.get('AUTH_SVC_URL')}/entities/{identifier}/authorizations", json={'roles': ['view']}) + + rv = client.get(f'/api/v2/businesses/{identifier}/court-orders/99999', + headers=create_header(jwt, [STAFF_ROLE], identifier) + ) + + assert rv.status_code == HTTPStatus.NOT_FOUND + assert rv.json == {'message': f'{identifier} court order not found'} diff --git a/legal-api/tests/unit/resources/v2/test_business_extended.py b/legal-api/tests/unit/resources/v2/test_business_extended.py new file mode 100644 index 0000000000..6753715f58 --- /dev/null +++ b/legal-api/tests/unit/resources/v2/test_business_extended.py @@ -0,0 +1,267 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests to assure the business extended end-point. + +Test-Suite to ensure that the /businesses../extended endpoint is working as expected. +""" + +import copy +from http import HTTPStatus +import pytest + +from business_model.models import Amalgamation, AmalgamatingBusiness, Business, Jurisdiction +from business_model.utils.legislation_datetime import LegislationDatetime +from legal_api.services.authz import PUBLIC_USER, STAFF_ROLE +from registry_schemas.example_data import ( + AMALGAMATION_APPLICATION, + AMALGAMATION_OUT, + CONTINUATION_IN, + CONTINUATION_OUT, + INCORPORATION, + FILING_HEADER, +) + +from tests.unit.models import factory_business, factory_business_mailing_address, factory_completed_filing +from tests.unit.services.utils import create_header + + +@pytest.mark.parametrize('legal_type, filing_type, dependent_filing_type', [ + ("C", "continuationIn", None), + ("BC", "amalgamationApplication", None), + ("BC", "amalgamationOut", None), + ("BC", "continuationOut", None), + ("C", "amalgamationOut", "continuationIn"), + ("C", "continuationOut", "continuationIn"), + ("BC", "amalgamationOut", "amalgamationApplication"), + ("BC", "continuationOut", "amalgamationApplication") +]) +def test_get_business_extended_without_filing_type(app, session, client, jwt, legal_type, filing_type, dependent_filing_type): + """Assert that business extended data is returned.""" + identifier = 'BC7654321' if legal_type == 'BC' else 'C7654321' + business = factory_business(identifier, entity_type=legal_type) + data = {} + if "continuationIn" in (dependent_filing_type, filing_type): + data["continuationIn"] = _create_continuation_in_business(business) + + if "amalgamationApplication" in (dependent_filing_type, filing_type): + data["amalgamation"] = _create_amalgation_business(business, True) + + if "continuationOut" == filing_type: + data["continuationOut"] = _create_out_business(business, "continuationOut") + elif "amalgamationOut" == filing_type: + data["amalgamationOut"] = _create_out_business(business, "amalgamationOut") + + + rv = client.get(f'/api/v2/businesses/{identifier}/extended?forCorrection=true', + headers=create_header(jwt, [STAFF_ROLE], identifier) + ) + + assert rv.status_code == HTTPStatus.OK + assert rv.json == data + + +@pytest.mark.parametrize('filing_type', ["amalgamationOut", "continuationOut"]) +def test_get_business_extended_out(app, session, client, jwt, filing_type): + """Assert that business extended data is returned.""" + identifier = 'BC7654321' + business = factory_business(identifier, entity_type='BC') + data = _create_out_business(business, filing_type) + + rv = client.get(f'/api/v2/businesses/{identifier}/extended/{filing_type}', + headers=create_header(jwt, [STAFF_ROLE], identifier) + ) + + assert rv.status_code == HTTPStatus.OK + assert rv.json[filing_type] == data + + +def test_get_business_extended_in(app, session, client, jwt): + """Assert that business continuation in data is returned.""" + identifier = 'C7654321' + filing_type = "continuationIn" + business = factory_business(identifier, entity_type='C') + data = _create_continuation_in_business(business) + rv = client.get(f'/api/v2/businesses/{identifier}/extended/{filing_type}', + headers=create_header(jwt, [STAFF_ROLE], identifier) + ) + + assert rv.status_code == HTTPStatus.OK + assert rv.json[filing_type] == data + + +@pytest.mark.parametrize('for_correction', [True, False]) +def test_get_business_extended_amalgamation(app, session, client, jwt, for_correction): + """Assert that business amalgamation data is returned.""" + identifier = 'BC7654321' + business = factory_business(identifier, entity_type='BC') + data = _create_amalgation_business(business, for_correction) + query_string = f'?forCorrection={for_correction}' if for_correction else '' + rv = client.get(f'/api/v2/businesses/{identifier}/extended/amalgamationApplication{query_string}', + headers=create_header(jwt, [STAFF_ROLE], identifier) + ) + + assert rv.status_code == HTTPStatus.OK + assert rv.json['amalgamation'] == data + + +def test_get_business_extended_bad_request(app, session, client, jwt): + """Assert that 400 is returned if filing type does not support.""" + identifier = 'BC7654321' + filing_type = 'incorporationApplication' + business = factory_business(identifier) + + rv = client.get(f'/api/v2/businesses/{identifier}/extended/{filing_type}', + headers=create_header(jwt, [STAFF_ROLE], identifier) + ) + + assert rv.status_code == HTTPStatus.BAD_REQUEST + assert rv.json == {'message': f'{filing_type} not supported'} + + +@pytest.mark.parametrize('filing_type', ["continuationIn", "amalgamationOut", "continuationOut"]) +def test_get_business_extended_not_found(app, session, client, jwt, filing_type): + """Assert that 404 is returned if business is not found.""" + identifier = 'BC7654321' + business = factory_business(identifier) + + rv = client.get(f'/api/v2/businesses/{identifier}/extended/{filing_type}', + headers=create_header(jwt, [STAFF_ROLE], identifier) + ) + + assert rv.status_code == HTTPStatus.NOT_FOUND + assert rv.json == {'message': f"Could not find {filing_type} data for {identifier}"} + + +def _create_out_business(business, filing_type): + continuation_out_filing = copy.deepcopy(FILING_HEADER) + continuation_out_filing['filing'][filing_type] = copy.deepcopy(CONTINUATION_OUT if filing_type == 'continuationOut' else AMALGAMATION_OUT) + continuation_out_filing['filing']['header']['name'] = filing_type + + filing = factory_completed_filing(business, continuation_out_filing) + data = { + 'date': '2023-06-19', + 'country': 'CA', + 'region': 'AB', + 'legalName': 'HAULER SERVICES', + } + business.state = Business.State.HISTORICAL + business.state_filing_id = filing.id + business.jurisdiction = data["country"] + business.foreign_jurisdiction_region = data["region"] + business.foreign_legal_name = data["legalName"] + if filing_type == 'continuationOut': + business.continuation_out_date = LegislationDatetime.as_utc_timezone_from_legislation_date_str(data["date"]) + elif filing_type == 'amalgamationOut': + business.amalgamation_out_date = LegislationDatetime.as_utc_timezone_from_legislation_date_str(data["date"]) + business.save() + return data + + +def _create_continuation_in_business(business): + filing_type = "continuationIn" + filing_json = copy.deepcopy(FILING_HEADER) + filing_json['filing'][filing_type] = copy.deepcopy(CONTINUATION_IN) + filing_json['filing']['header']['name'] = filing_type + + filing = factory_completed_filing(business, filing_json) + data = { + 'country': 'CA', + 'region': 'AB', + 'legalName': 'HAULER SERVICES', + 'identifier': 'AB1234567', + 'incorporationDate': '2020-01-01', + 'expro': { + 'identifier': 'A0077779', + 'legalName': 'Test Company Inc.' + } + } + jurisdiction = Jurisdiction( + filing_id = filing.id, + country=data['country'], + region=data['region'], + identifier=data['identifier'], + legal_name=data['legalName'], + incorporation_date=LegislationDatetime.as_utc_timezone_from_legislation_date_str(data['incorporationDate']), + expro_identifier=data['expro']['identifier'], + expro_legal_name=data['expro']['legalName'] + ) + business.jurisdictions.append(jurisdiction) + business.save() + return data + + +def _create_amalgation_business(business, for_correction=False): + amalgamating_identifier = 'BC1234567' + amalgamating_business = factory_business(amalgamating_identifier, entity_type='BC') + factory_business_mailing_address(amalgamating_business) + filing_type = "amalgamationApplication" + filing_json = copy.deepcopy(FILING_HEADER) + filing_json['filing'][filing_type] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing_json['filing']['header']['name'] = filing_type + + filing = factory_completed_filing(business, filing_json) + amalgamating_business_json = [ + { + 'role': 'amalgamating', + 'identifier': amalgamating_identifier + }, + { + 'role': 'amalgamating', + 'foreignJurisdiction': { + 'country': 'CA', + 'region': 'AB', + }, + 'legalName': 'HAULER SERVICES', + 'identifier': 'AB1234567' + } + ] + if for_correction: + amalgamating_business_json[0]['legalName'] = amalgamating_business.legal_name + amalgamating_business_json[0]['legalType'] = amalgamating_business.legal_type + amalgamating_business_json[0]['mailingAddress'] = amalgamating_business.mailing_address.one_or_none().json + + data = { + 'courtApproval': False, + 'amalgamatingBusinesses':amalgamating_business_json + } + amalgamation = Amalgamation( + amalgamation_type=Amalgamation.AmalgamationTypes.regular, + amalgamation_date=LegislationDatetime.now(), + court_approval=False, + filing_id=filing.id, + business_id=business.id, + ) + amalgamation.save() + + ting1 = AmalgamatingBusiness( + role=AmalgamatingBusiness.Role.amalgamating, + business_id=amalgamating_business.id, + amalgamation_id=amalgamation.id + ) + ting1.save() + amalgamating_business_json[0]['id'] = ting1.id + + ting2 = AmalgamatingBusiness( + role=AmalgamatingBusiness.Role.amalgamating, + foreign_jurisdiction=amalgamating_business_json[1]['foreignJurisdiction']['country'], + foreign_jurisdiction_region=amalgamating_business_json[1]['foreignJurisdiction']['region'], + foreign_name=amalgamating_business_json[1]['legalName'], + foreign_identifier=amalgamating_business_json[1]['identifier'], + amalgamation_id=amalgamation.id + ) + ting2.save() + amalgamating_business_json[1]['id'] = ting2.id + + return data diff --git a/legal-api/tests/unit/resources/v2/test_business_filings/test_filing_documents.py b/legal-api/tests/unit/resources/v2/test_business_filings/test_filing_documents.py index e4cf4ca9df..f96ae2956f 100644 --- a/legal-api/tests/unit/resources/v2/test_business_filings/test_filing_documents.py +++ b/legal-api/tests/unit/resources/v2/test_business_filings/test_filing_documents.py @@ -36,6 +36,7 @@ CHANGE_OF_OFFICERS, CHANGE_OF_RECEIVERS, CHANGE_OF_REGISTRATION, + CONSENT_CONTINUATION_OUT, CONTINUATION_IN, CONTINUATION_OUT, CORRECTION_AR, @@ -70,6 +71,11 @@ } +def mock_auth(headers: dict): + """Return a stub for flask.request.headers.get.""" + return lambda key, default=None: headers.get(key, default) + + def basic_test_helper(): identifier = 'CP7654321' business = factory_business(identifier) @@ -201,20 +207,6 @@ def test_unpaid_filing(session, client, jwt): }, HTTPStatus.OK, '2017-10-01' ), - ('specres_court_completed', 'CP7654321', Business.LegalTypes.COOP.value, - 'specialResolution', SPECIAL_RESOLUTION, 'courtOrder', COURT_ORDER, Filing.Status.COMPLETED, - {'documents': { - 'certifiedRules': f'{base_url}/api/v2/businesses/CP7654321/filings/1/documents/certifiedRules', - 'legalFilings': [ - {'courtOrder': f'{base_url}/api/v2/businesses/CP7654321/filings/1/documents/courtOrder'}, - {'specialResolution': f'{base_url}/api/v2/businesses/CP7654321/filings/1/documents/specialResolution'}, - ], - 'receipt': f'{base_url}/api/v2/businesses/CP7654321/filings/1/documents/receipt', - 'specialResolutionApplication': f'{base_url}/api/v2/businesses/CP7654321/filings/1/documents/specialResolutionApplication', - } - }, - HTTPStatus.OK, '2017-10-01' - ), ('special_res_correction', 'CP7654321', Business.LegalTypes.COOP.value, 'correction', CORRECTION_CP_SPECIAL_RESOLUTION, None, None, Filing.Status.COMPLETED, {'documents': { @@ -1070,6 +1062,33 @@ def test_unpaid_filing(session, client, jwt): }, HTTPStatus.OK, '2017-10-01' ), + ('ben_consentContinuationOut_completed', 'BC7654321', + Business.LegalTypes.BCOMP.value, 'consentContinuationOut', CONSENT_CONTINUATION_OUT, + None, None, Filing.Status.COMPLETED, + {'documents': { + 'legalFilings': [ + {'consentContinuationOut': + f'{base_url}/api/v2/businesses/BC7654321/filings/1/documents/consentContinuationOut'}, + ], + 'letterOfConsent': f'{base_url}/api/v2/businesses/BC7654321/filings/documents/letterOfConsent', + 'receipt': f'{base_url}/api/v2/businesses/BC7654321/filings/1/documents/receipt' + } + }, + HTTPStatus.OK, '2017-10-01' + ), + ('ben_consentContinuationOut_paid', 'BC7654321', + Business.LegalTypes.BCOMP.value, 'consentContinuationOut', CONSENT_CONTINUATION_OUT, + None, None, Filing.Status.PAID, + {'documents': { + 'legalFilings': [ + {'consentContinuationOut': + f'{base_url}/api/v2/businesses/BC7654321/filings/1/documents/consentContinuationOut'}, + ], + 'receipt': f'{base_url}/api/v2/businesses/BC7654321/filings/1/documents/receipt' + } + }, + HTTPStatus.OK, '2017-10-01' + ), ('ben_amalgamation_completed', 'BC7654321', Business.LegalTypes.BCOMP.value, 'amalgamationApplication', AMALGAMATION_APPLICATION, None, None, Filing.Status.COMPLETED, @@ -1553,7 +1572,13 @@ def test_unpaid_filing(session, client, jwt): } }, HTTPStatus.OK, '2024-09-26' - ) + ), + ('ben_court_order_completed', 'BC7654321', Business.LegalTypes.BCOMP.value, + 'courtOrder', COURT_ORDER, None, None, Filing.Status.COMPLETED, + {'documents': {'receipt': f'{base_url}/api/v2/businesses/BC7654321/filings/1/documents/receipt'} + }, + HTTPStatus.OK, '2017-10-01' + ), ]) def test_document_list_for_various_filing_states(app, session, mocker, client, jwt, monkeypatch, mock_drs_service, test_name, @@ -1606,15 +1631,20 @@ def test_document_list_for_various_filing_states(app, session, mocker, client, j 'name': file.get('fileName'), 'url': f'{base_url}/api/v2/businesses/{identifier}/filings/1/documents/static/{file_key}' }) + elif filing_name_1 == 'courtOrder': + for file in meta_data['courtOrder']['files']: + file_key = file.get('fileKey') + expected_msg['documents']['staticDocuments'] = [{ + 'name': file.get('fileName'), + 'url': f'{base_url}/api/v2/businesses/{identifier}/filings/1/documents/static/{file_key}' + }] + account_id: str = '1' headers=create_header(jwt, [STAFF_ROLE], identifier, account_id=account_id) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) account_products_mock = [] with requests_mock.Mocker() as m: mock_url = f"{app.config['AUTH_SVC_URL']}/orgs/{account_id}/products?include_hidden=true" @@ -1644,6 +1674,32 @@ def filer_action(filing_name, filing_json, meta_data, business): meta_data['continuationIn'] = {} meta_data['continuationIn']['affidavitFileKey'] = continuation_in['foreignJurisdiction']['affidavitFileKey'] meta_data['continuationIn']['authorizationFiles'] = continuation_in['authorization']['files'] + + court_order = ( + filing_json['filing'][filing_name] if filing_name == 'courtOrder' + else filing_json['filing'][filing_name].get('courtOrder') + ) + if court_order: + file_number = court_order['fileNumber'] + effect_of_order = court_order.get('effectOfOrder') + order_date = court_order.get('orderDate') + + court_order_meta = {"fileNumber": file_number} + if effect_of_order: + court_order_meta["effectOfOrder"] = effect_of_order + if order_date: + court_order_meta["orderDate"] = order_date + + meta_data["courtOrder"] = court_order_meta + + if filing_name == 'courtOrder': + meta_data["courtOrder"]["orderDetails"] = court_order.get('orderDetails') + meta_data["courtOrder"]["files"] = [ + { + "fileKey": court_order.get('fileKey'), + "fileName": f"Court Order {court_order['fileNumber']}" + } + ] if filing_name == 'correction' and business.legal_type == 'CP': meta_data['correction'] = {} @@ -1711,11 +1767,8 @@ def test_continuation_out_uploaded_documents(app, session, client, jwt, monkeypa account_id = '1' headers = create_header(jwt, [STAFF_ROLE], identifier, account_id=account_id) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) with requests_mock.Mocker() as m: mock_url = f"{app.config['AUTH_SVC_URL']}/orgs/{account_id}/products?include_hidden=true" m.get(mock_url, json=[], status_code=HTTPStatus.OK) @@ -1780,11 +1833,8 @@ def test_continuation_out_uploaded_documents_returned_for_non_staff(non_staff_ro account_id = '1' headers = create_header(jwt, [non_staff_role], identifier, account_id=account_id) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) with requests_mock.Mocker() as m: m.get(f"{app.config.get('AUTH_SVC_URL')}/entities/{identifier}/authorizations", json={'roles': ['view']}, status_code=HTTPStatus.OK) @@ -1893,12 +1943,9 @@ def test_temp_document_list_for_various_filing_states(app, mocker, session, clie filing.save() account_id: str = '1' headers=create_header(jwt, [STAFF_ROLE], temp_identifier, account_id=account_id) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) account_products_mock = [] with requests_mock.Mocker() as m: mock_url = f"{app.config['AUTH_SVC_URL']}/orgs/{account_id}/products?include_hidden=true" @@ -1915,7 +1962,7 @@ def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library me assert rv_data == expected -def test_get_receipt(session, client, jwt, requests_mock): +def test_get_receipt(session, client, jwt, requests_mock, mocker): """Assert that a receipt is generated.""" from legal_api.resources.v2.business.business_filings.business_documents import _get_receipt @@ -1943,15 +1990,17 @@ def test_get_receipt(session, client, jwt, requests_mock): json={'foo': 'bar'}, status_code=HTTPStatus.CREATED) - token = helper_create_jwt(jwt, roles=[STAFF_ROLE], username='username') + service_token = 'service-token-abc' + mocker.patch('business_account.AccountService.get_bearer_token', return_value=service_token) - content, status_code = _get_receipt(business, filing_core, token) + content, status_code = _get_receipt(business, filing_core) assert status_code == HTTPStatus.CREATED assert requests_mock.called_once + assert requests_mock.last_request.headers.get('Authorization') == f'Bearer {service_token}' -def test_get_receipt_request_mock(session, client, jwt, requests_mock): +def test_get_receipt_request_mock(session, client, jwt, requests_mock, mocker): """Assert that a receipt is generated.""" from legal_api.resources.v2.business.business_filings.business_documents import _get_receipt @@ -1978,6 +2027,9 @@ def test_get_receipt_request_mock(session, client, jwt, requests_mock): json={'foo': 'bar'}, status_code=HTTPStatus.CREATED) + service_token = 'service-token-abc' + mocker.patch('business_account.AccountService.get_bearer_token', return_value=service_token) + rv = client.get(f'/api/v2/businesses/{identifier}/filings/{filing.id}/documents/receipt', headers=create_header(jwt, [STAFF_ROLE], @@ -1986,7 +2038,46 @@ def test_get_receipt_request_mock(session, client, jwt, requests_mock): ) assert rv.status_code == HTTPStatus.CREATED - assert requests_mock.called_once + assert requests_mock.called + assert requests_mock.last_request.headers.get('Authorization') == f'Bearer {service_token}' + + +def test_get_receipt_forwards_account_linking_key(session, client, jwt, requests_mock, mocker): + """Assert that the receipt call forwards the Account-Linking-Key header when present on the request.""" + identifier = 'CP7654321' + business = factory_business(identifier) + filing_name = 'incorporationApplication' + payment_id = '12345' + + filing_json = copy.deepcopy(FILING_HEADER) + filing_json['filing']['header']['name'] = filing_name + filing_json['filing'][filing_name] = INCORPORATION + filing_json['filing'].pop('business') + + filing_date = datetime.now(UTC) + filing = factory_filing(business, filing_json, filing_date=filing_date) + filing.skip_status_listener = True + filing._status = 'PAID' + filing._payment_token = payment_id + filing._payment_completion_date = filing_date + filing.save() + + pay_mock = requests_mock.post(f"{current_app.config.get('PAYMENT_SVC_URL')}/{payment_id}/receipts", + json={'foo': 'bar'}, + status_code=HTTPStatus.CREATED) + + mocker.patch('business_account.AccountService.get_bearer_token', return_value='service-token') + + rv = client.get(f'/api/v2/businesses/{identifier}/filings/{filing.id}/documents/receipt', + headers=create_header(jwt, + [STAFF_ROLE], + identifier, + **{'accept': 'application/pdf', + 'Account-Linking-Key': 'test-linking-key'}) + ) + + assert rv.status_code == HTTPStatus.CREATED + assert pay_mock.last_request.headers.get('Account-Linking-Key') == 'test-linking-key' def test_get_receipt_no_receipt_ca(session, client, jwt, requests_mock): @@ -2078,12 +2169,9 @@ def test_temp_document_list_for_now(app, mocker, session, client, jwt, monkeypat filing.save() account_id: str = '1' headers=create_header(jwt, [STAFF_ROLE], temp_identifier, account_id=account_id) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) account_products_mock = [] with requests_mock.Mocker() as m: mock_url = f"{app.config['AUTH_SVC_URL']}/orgs/{account_id}/products?include_hidden=true" @@ -2101,7 +2189,7 @@ def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library me filing._status = Filing.Status.COMPLETED filing.save() - mocker.patch('legal_api.core.filing.has_roles', return_value=True) + mocker.patch('legal_api.core.filing.has_any_roles', return_value=True) rv = client.get(f'/api/v2/businesses/{temp_identifier}/filings/{filing.id}/documents', headers=create_header(jwt, [STAFF_ROLE], temp_identifier)) @@ -2111,3 +2199,74 @@ def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library me assert rv.status_code == expected_http_code assert rv_data == expected + + +def test_get_static_document_by_file_key_shape(session, client, jwt, mocker): + """Assert static documents are served from DRS.""" + from unittest.mock import MagicMock, patch + + from business_model.models import Document + from legal_api.resources.v2.business.business_filings import business_documents + + file_key = 'CORP-DS0000101951' + identifier = 'CP7654321' + business = factory_business(identifier) + filing_json = copy.deepcopy(FILING_HEADER) + filing_json['filing']['header']['name'] = 'continuationIn' + filing = factory_filing(business, filing_json) + + document = Document(type='authorization_file', file_key=file_key, filing_id=filing.id, business_id=business.id) + document.save() + + mocker.patch.object(business_documents, '_is_document_available', return_value=True) + drs_response = MagicMock(content=b'drs-pdf', status_code=HTTPStatus.OK) + + with patch.object(business_documents.client_doc_service, 'get_document', return_value=drs_response) as mock_drs: + rv = client.get(f'/api/v2/businesses/{identifier}/filings/{filing.id}/documents/static/{file_key}', + headers=create_header(jwt, [STAFF_ROLE], identifier, **{'accept': 'application/pdf'})) + + assert rv.status_code == HTTPStatus.OK + mock_drs.assert_called_once_with('DS0000101951', 'CORP', doc_binary=True) + assert rv.data == b'drs-pdf' + + +def test_admin_dissolution_firm_documents_no_certificate(app, session, client, jwt, monkeypatch, mock_drs_service): + """Firm (SP/GP) administrative dissolution must build its document list without error. + + Regression for bcgov/entity#33806: alter_outputs_dissolution did + outputs.remove('certificateOfDissolution') for administrative dissolutions, but + firms carry no certificate output, so the set.remove raised KeyError and the + documents/ledger endpoint returned HTTP 400 (surfacing as "deleted" ledger history). + """ + identifier = 'FM7654321' + business = factory_business(identifier, entity_type='SP') + + filing_json = copy.deepcopy(FILING_HEADER) + filing_json['filing']['header']['name'] = 'dissolution' + filing_json['filing']['business']['legalType'] = 'SP' + filing_json['filing']['dissolution'] = ADMIN_DISSOLUTION + + filing = factory_filing(business, filing_json, filing_date=datetime.now(UTC)) + filing.skip_status_listener = True + filing._status = Filing.Status.COMPLETED + filing._payment_completion_date = '2017-10-01' + lf = [list(x.keys()) for x in filing.legal_filings()] + legal_filings = [item for sublist in lf for item in sublist] + filing._meta_data = {'legalFilings': legal_filings} + filing.save() + + account_id = '1' + headers = create_header(jwt, [STAFF_ROLE], identifier, account_id=account_id) + with app.test_request_context(): + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) + with requests_mock.Mocker() as m: + m.get(f"{app.config['AUTH_SVC_URL']}/orgs/{account_id}/products?include_hidden=true", + json=[], status_code=HTTPStatus.OK) + rv = client.get(f'/api/v2/businesses/{identifier}/filings/{filing.id}/documents', + headers=headers) + + assert rv.status_code == HTTPStatus.OK + documents = rv.json['documents'] + assert 'certificateOfDissolution' not in documents + assert {'dissolution': f'{base_url}/api/v2/businesses/{identifier}/filings/{filing.id}/documents/dissolution'} \ + in documents['legalFilings'] diff --git a/legal-api/tests/unit/resources/v2/test_business_filings/test_filings.py b/legal-api/tests/unit/resources/v2/test_business_filings/test_filings.py index 6631f4b4cd..c710e95e58 100644 --- a/legal-api/tests/unit/resources/v2/test_business_filings/test_filings.py +++ b/legal-api/tests/unit/resources/v2/test_business_filings/test_filings.py @@ -28,7 +28,6 @@ import pytest from dateutil.parser import parse from flask import current_app -from minio.error import S3Error from reportlab.lib.pagesizes import letter from business_common.utils.legislation_datetime import LegislationDatetime @@ -48,7 +47,6 @@ from legal_api.resources.v2.business.business_filings.business_filings import ListFilingResource from legal_api.services.authz import BASIC_USER, PUBLIC_USER, STAFF_ROLE from legal_api.services.bootstrap import RegistrationBootstrapService -from legal_api.services.minio import MinioService from registry_schemas.example_data import ( ALTERATION_FILING_TEMPLATE, AMALGAMATION_APPLICATION, @@ -74,7 +72,8 @@ REGISTRATION, RESTORATION, SPECIAL_RESOLUTION, - TRANSITION_FILING_TEMPLATE + TRANSITION_FILING_TEMPLATE, + get_filing_template, ) from registry_schemas.example_data.schema_data import COURT_ORDER_FILING_TEMPLATE, RESTORATION from tests import integration_payment @@ -89,10 +88,120 @@ factory_pending_filing, factory_user, ) -from tests.unit.services.filings.test_utils import _upload_file from tests.unit.services.utils import create_header +def test_trim_strings_whitespace_only_becomes_empty(): + """Assert that a whitespace-only string is trimmed down to an empty string.""" + data = {'field': ' '} + ListFilingResource._trim_strings(data) + assert data['field'] == '' + + +def test_trim_strings_various_whitespace_characters(): + """Assert tabs, newlines, and mixed whitespace are all stripped, not just spaces.""" + data = {'field': '\t\n value with internal space \n\t'} + ListFilingResource._trim_strings(data) + assert data['field'] == 'value with internal space' + + +def test_trim_strings_empty_dict_and_list_untouched(): + """Assert empty containers don't error and are left as empty containers.""" + data = {'a': {}, 'b': [], 'c': ' x '} + ListFilingResource._trim_strings(data) + assert data == {'a': {}, 'b': [], 'c': 'x'} + + +def test_trim_strings_is_idempotent(): + """Assert running trim twice on the same data produces the same result.""" + data = {'a': ' x ', 'b': {'c': [' y ', ' z ']}} + ListFilingResource._trim_strings(data) + first_pass = copy.deepcopy(data) + ListFilingResource._trim_strings(data) + assert data == first_pass + + +def test_trim_strings_returns_same_object_reference(): + """Assert the function mutates in place and returns the same object (not a copy).""" + data = {'a': ' x '} + result = ListFilingResource._trim_strings(data) + assert result is data + + +def test_post_ar_trims_padded_routing_slip_number(session, client, jwt): + """Assert that a routingSlipNumber padded with whitespace is trimmed before validation, + so it passes even though the padded (untrimmed) length would exceed the schema limit.""" + identifier = 'CP7654321' + factory_business(identifier, + founding_date=(datetime.now(UTC) - datedelta.datedelta(years=2)), + last_ar_date=datetime(datetime.now(UTC).year - 1, 4, 20).date()) + + ar = copy.deepcopy(ANNUAL_REPORT) + ar['filing']['business'] = {'identifier': identifier} + annual_report_date = datetime(datetime.now(UTC).year, 2, 20).date() + if annual_report_date > LegislationDatetime.now().date(): + annual_report_date = LegislationDatetime.now().date() + ar['filing']['annualReport']['annualReportDate'] = annual_report_date.isoformat() + ar['filing']['annualReport']['annualGeneralMeetingDate'] = datetime.now(UTC).date().isoformat() + ar['filing']['header']['routingSlipNumber'] = ' 123131332 ' + + rv = client.post(f'/api/v2/businesses/{identifier}/filings?only_validate=true', + json=ar, + headers=create_header(jwt, [STAFF_ROLE], identifier) + ) + + assert rv.status_code == HTTPStatus.OK + assert not rv.json.get('errors') + + +def test_post_change_of_directors_with_whitespace_trimmed(session, client, jwt): + """Assert that whitespace padding in nested officer/address fields is trimmed + before the changeOfDirectors filing is validated and saved.""" + identifier = 'CP7654321' + factory_business(identifier) + + cod = get_filing_template('changeOfDirectors', identifier) + cod['filing']['changeOfDirectors'] = copy.deepcopy(CHANGE_OF_DIRECTORS) + cod['filing']['changeOfDirectors']['directors'][0]['officer']['firstName'] = ' Peter ' + cod['filing']['changeOfDirectors']['directors'][0]['officer']['lastName'] = ' Griffin ' + cod['filing']['changeOfDirectors']['directors'][0]['deliveryAddress']['postalCode'] = ' H0H0H0 ' + cod['filing']['changeOfDirectors']['directors'][1]['title'] = ' Treasurer ' + + rv = client.post(f'/api/v2/businesses/{identifier}/filings?draft=true', + json=cod, + headers=create_header(jwt, [STAFF_ROLE], identifier) + ) + + assert rv.status_code == HTTPStatus.CREATED + directors = rv.json['filing']['changeOfDirectors']['directors'] + assert directors[0]['officer']['firstName'] == 'Peter' + assert directors[0]['officer']['lastName'] == 'Griffin' + assert directors[0]['deliveryAddress']['postalCode'] == 'H0H0H0' + assert directors[1]['title'] == 'Treasurer' + assert directors[0]['cessationDate'] is None + assert directors[1]['cessationDate'] is None + + +def test_post_ar_preserves_numeric_and_boolean_types(session, client, jwt): + """Assert numbers/booleans in the filing json aren't coerced to strings or otherwise altered.""" + identifier = 'CP7654321' + factory_business(identifier) + + ar = copy.deepcopy(ANNUAL_REPORT) + ar['filing']['business'] = {'identifier': identifier} + ar['filing']['header']['priority'] = True + ar['filing']['header']['waiveFees'] = False + + rv = client.post(f'/api/v2/businesses/{identifier}/filings?draft=true', + json=ar, + headers=create_header(jwt, [STAFF_ROLE], identifier) + ) + + assert rv.status_code == HTTPStatus.CREATED + assert rv.json['filing']['header']['priority'] is True + assert rv.json['filing']['header']['waiveFees'] is False + + @pytest.mark.parametrize( 'legal_type, filing_type, filing_json', [ @@ -112,7 +221,7 @@ def test_get_temp_business_filing(session, client, jwt, legal_type, filing_type, temp_reg.save() json_data = copy.deepcopy(FILING_HEADER) json_data['filing']['header']['name'] = filing_type - del json_data['filing']['business'] + json_data['filing']['business'] = {'identifier': identifier} filing_json = copy.deepcopy(filing_json) filing_json['nameRequest']['legalType'] = legal_type json_data['filing'][filing_type] = filing_json @@ -303,6 +412,16 @@ def test_get_one_business_filing_by_id_public_json(session, client, jwt, test_na } })) filing.save() + expected_dissolution_date = '2023-01-19' + if filing_name == 'dissolution': + # Filer adds this value into the meta_data so we need to do this manually here as part of the setup + filing._meta_data = json.loads(json.dumps( + { + filing_name: { + 'dissolutionDate': expected_dissolution_date + } + })) + filing.save() rv = client.get(f'/api/v2/businesses/{identifier}/filings/{filing.id}?public=true', headers=create_header(jwt, [PUBLIC_USER], identifier)) @@ -316,10 +435,13 @@ def test_get_one_business_filing_by_id_public_json(session, client, jwt, test_na if filing_name == 'putBackOff': assert rv.json['filing'][filing_name].get('reason') == expected_reason assert rv.json['filing'][filing_name].get('expiryDate') == expected_expiry_date + if filing_name == 'dissolution': + assert rv.json['filing'][filing_name].get('dissolutionDate') == expected_dissolution_date assert not any([key for key in rv.json['filing'] if key not in ['header', filing_name]]) assert not any([key for key in rv.json['filing']['header'] if key not in ['name', 'effectiveDate']]) - assert not any([key for key in rv.json['filing'][filing_name] if key not in ['expiryDate', 'type', 'reason']]) + assert not any( + [key for key in rv.json['filing'][filing_name] if key not in ['expiryDate', 'type', 'reason', 'dissolutionDate']]) def test_get_404_when_business_invalid_filing_id(session, client, jwt): @@ -366,17 +488,18 @@ def test_post_fail_if_given_filing_id(session, client, jwt): f'Illegal to attempt to create a duplicate filing for {identifier}.'} -def test_post_filing_no_business(session, client, jwt): - """Assert that a filing cannot be created against non-existent business.""" +def test_post_filing_business_identifier(session, client, jwt): + """Assert that a filing requires a matching business identifier specified.""" identifier = 'CP7654321' - + filing = copy.deepcopy(ANNUAL_REPORT) + filing['filing']['business']['identifier'] = 'BC1234567' rv = client.post(f'/api/v2/businesses/{identifier}/filings', - json=ANNUAL_REPORT, + json=filing, headers=create_header(jwt, [STAFF_ROLE], identifier) ) assert rv.status_code == HTTPStatus.BAD_REQUEST - assert rv.json['errors'][0] == {'message': 'A valid business is required.'} + assert rv.json['errors'][0] == {'message': 'filing/business/identifier does not equal the identifier in the request path.'} def test_post_empty_annual_report_to_a_business(session, client, jwt): @@ -402,9 +525,9 @@ def test_post_authorized_draft_ar(session, client, jwt): """Assert that a unpaid filing can be posted.""" identifier = 'CP7654321' factory_business(identifier) - + data = get_filing_template('annualReport', identifier) rv = client.post(f'/api/v2/businesses/{identifier}/filings?draft=true', - json=ANNUAL_REPORT, + json=data, headers=create_header(jwt, [STAFF_ROLE], identifier) ) @@ -415,9 +538,9 @@ def test_post_not_authorized_draft_ar(session, client, jwt): """Assert that a unpaid filing can be posted.""" identifier = 'CP7654321' factory_business(identifier) - + data = get_filing_template('annualReport', identifier) rv = client.post(f'/api/v2/businesses/{identifier}/filings?draft=true', - json=ANNUAL_REPORT, + json=data, headers=create_header(jwt, [BASIC_USER], 'WRONGUSER') ) @@ -428,9 +551,9 @@ def test_post_not_allowed_historical(session, client, jwt): """Assert that a filing is not allowed for historical business.""" identifier = 'CP7654321' factory_business(identifier, state=Business.State.HISTORICAL) - + data = get_filing_template('annualReport', identifier) rv = client.post(f'/api/v2/businesses/{identifier}/filings', - json=ANNUAL_REPORT, + json=data, headers=create_header(jwt, [BASIC_USER], 'WRONGUSER') ) @@ -441,9 +564,9 @@ def test_post_allowed_historical(session, client, jwt): """Assert that a filing is allowed for historical business.""" identifier = 'BC7654321' factory_business(identifier, state=Business.State.HISTORICAL) - + co = get_filing_template('courtOrder', identifier) rv = client.post(f'/api/v2/businesses/{identifier}/filings?draft=true', - json=COURT_ORDER_FILING_TEMPLATE, + json=co, headers=create_header(jwt, [STAFF_ROLE], 'user') ) @@ -452,11 +575,10 @@ def test_post_allowed_historical(session, client, jwt): def test_special_resolution_sanitation(session, client, jwt): """Assert that script tags can't be passed into special resolution resolution field.""" - identifier = 'BC7654399' + identifier = 'CP7654399' factory_business(identifier, state=Business.State.ACTIVE) - data = copy.deepcopy(FILING_HEADER) - data['filing']['header']['name'] = 'specialResolution' + data = get_filing_template('specialResolution', identifier) data['filing']['specialResolution'] = copy.deepcopy(SPECIAL_RESOLUTION) data['filing']['specialResolution']['resolution'] = """

Hello this is great

@@ -471,7 +593,7 @@ def test_special_resolution_sanitation(session, client, jwt): headers=create_header(jwt, [STAFF_ROLE], 'user') ) assert rv.status_code == HTTPStatus.CREATED - assert rv.json['filing']['specialResolution']['resolution'] == '

Hello this is great

' + assert rv.json['filing']['specialResolution']['resolution'] == '

Hello this is great

' def test_post_draft_ar(session, client, jwt): @@ -479,8 +601,10 @@ def test_post_draft_ar(session, client, jwt): identifier = 'CP7654321' factory_business(identifier) + ar = copy.deepcopy(ANNUAL_REPORT) + ar['filing']['business'] = {'identifier': identifier} rv = client.post(f'/api/v2/businesses/{identifier}/filings?draft=true', - json=ANNUAL_REPORT, + json=ar, headers=create_header(jwt, [STAFF_ROLE], identifier) ) @@ -497,6 +621,7 @@ def test_post_only_validate_ar(session, client, jwt): last_ar_date=datetime(datetime.now(UTC).year - 1, 4, 20).date()) ar = copy.deepcopy(ANNUAL_REPORT) + ar['filing']['business']['identifier'] = identifier annual_report_date = datetime(datetime.now(UTC).year, 2, 20).date() if annual_report_date > LegislationDatetime.now().date(): annual_report_date = LegislationDatetime.now().date() @@ -520,6 +645,7 @@ def test_post_validate_ar_using_last_ar_date(session, client, jwt): founding_date=(datetime.now(UTC) - datedelta.datedelta(years=2)) # founding date = 2 years ago ) ar = copy.deepcopy(ANNUAL_REPORT) + ar['filing']['business']['identifier'] = identifier annual_report_date = datetime(datetime.now(UTC).year, 2, 20).date() if annual_report_date > LegislationDatetime.now().date(): annual_report_date = LegislationDatetime.now().date() @@ -561,6 +687,7 @@ def test_post_only_validate_ar_invalid_routing_slip(session, client, jwt): factory_business(identifier) ar = copy.deepcopy(ANNUAL_REPORT) + ar['filing']['business']['identifier'] = identifier ar['filing']['header']['routingSlipNumber'] = '1231313329988888' rv = client.post(f'/api/v2/businesses/{identifier}/filings?only_validate=true', @@ -591,6 +718,7 @@ def test_post_validate_ar_valid_routing_slip(session, client, jwt): last_ar_date=datetime(datetime.now(UTC).year - 1, 4, 20).date()) ar = copy.deepcopy(ANNUAL_REPORT) + ar['filing']['business'] = {'identifier': identifier} annual_report_date = datetime(datetime.now(UTC).year, 2, 20).date() if annual_report_date > LegislationDatetime.now().date(): annual_report_date = LegislationDatetime.now().date() @@ -613,8 +741,7 @@ def test_post_cod_with_empty_directors_array(session, client, jwt, only_validate identifier = 'CP7654321' factory_business(identifier) - cod = copy.deepcopy(FILING_HEADER) - cod['filing']['header']['name'] = 'changeOfDirectors' + cod = get_filing_template('changeOfDirectors', identifier) cod['filing']['changeOfDirectors'] = copy.deepcopy(CHANGE_OF_DIRECTORS) # Set empty directors array - this should fail validation due to minItems: 1 in schema cod['filing']['changeOfDirectors']['directors'] = [] @@ -890,11 +1017,11 @@ def test_payment_failed(session, client, jwt): def test_update_draft_ar(session, client, jwt): """Assert that a valid filing can be updated to a paid filing.""" - import copy identifier = 'CP7654321' b = factory_business(identifier) filings = factory_filing(b, ANNUAL_REPORT) ar = copy.deepcopy(ANNUAL_REPORT) + ar['filing']['business']['identifier'] = identifier rv = client.put(f'/api/v2/businesses/{identifier}/filings/{filings.id}?draft=true', json=ar, @@ -982,8 +1109,10 @@ def test_delete_draft_now_filing(session, client, jwt): assert 'noticeOfWithdrawal' not in rv.json['filing'] -def test_delete_coop_ia_filing_in_draft_with_file_in_minio(session, client, jwt, minio_server): - """Assert that a draft filing can be deleted.""" +def test_delete_coop_ia_filing_in_draft_with_file_in_drs(session, client, jwt): + """Assert that a draft filing can be deleted and DRS files are removed.""" + from legal_api.resources.v2.business.business_filings import business_filings + identifier = 'T1234567' temp_reg = RegistrationBootstrap() temp_reg._identifier = identifier @@ -999,8 +1128,8 @@ def test_delete_coop_ia_filing_in_draft_with_file_in_minio(session, client, jwt, 'cooperativeAssociationType': 'CP' } - rules_file_key = _upload_file(letter, invalid=False) - memorandum_file_key = _upload_file(letter, invalid=False) + rules_file_key = 'COOP-DS0000101951' + memorandum_file_key = 'COOP-DS0000101952' filing_json['filing']['incorporationApplication']['cooperative']['rulesFileKey'] = rules_file_key filing_json['filing']['incorporationApplication']['cooperative']['memorandumFileKey'] = memorandum_file_key filing = factory_filing(Business(), filing_json, filing_type='incorporationApplication') @@ -1008,24 +1137,19 @@ def test_delete_coop_ia_filing_in_draft_with_file_in_minio(session, client, jwt, filing.save() headers = create_header(jwt, [STAFF_ROLE], identifier) - with patch.object(RegistrationBootstrapService, 'deregister_bootstrap', return_value=HTTPStatus.OK): - with patch.object(RegistrationBootstrapService, 'delete_bootstrap', return_value=HTTPStatus.OK): - rv = client.delete(f'/api/v2/businesses/{identifier}/filings/{filing.id}', headers=headers) + with patch.object(RegistrationBootstrapService, 'deregister_bootstrap', return_value=HTTPStatus.OK), \ + patch.object(RegistrationBootstrapService, 'delete_bootstrap', return_value=HTTPStatus.OK), \ + patch.object(business_filings.doc_service, 'delete_document') as mock_drs: + rv = client.delete(f'/api/v2/businesses/{identifier}/filings/{filing.id}', headers=headers) - assert rv.status_code == HTTPStatus.OK - try: - MinioService.get_file_info(rules_file_key) - except S3Error as ex: - assert ex.code == 'NoSuchKey' + assert rv.status_code == HTTPStatus.OK + assert mock_drs.call_count == 2 - try: - MinioService.get_file_info(memorandum_file_key) - except S3Error as ex: - assert ex.code == 'NoSuchKey' +def test_delete_continuation_in_filing_with_authorization_files_in_draft(session, client, jwt): + """Assert that a draft continuationIn filing can be deleted and authorization files are removed from DRS.""" + from legal_api.resources.v2.business.business_filings import business_filings -def test_delete_continuation_in_filing_with_authorization_files_in_draft(session, client, jwt, minio_server): - """Assert that a draft continuationIn filing can be deleted and authorization files are removed from Minio.""" identifier = 'CP1234568' b = factory_business(identifier) @@ -1038,26 +1162,26 @@ def test_delete_continuation_in_filing_with_authorization_files_in_draft(session "files": [] } } - file_key_1 = _upload_file(letter, invalid=False) - file_key_2 = _upload_file(letter, invalid=False) + file_key_1 = 'CORP-DS0000101951' + file_key_2 = 'CORP-DS0000101952' filing_json['filing']['continuationIn']['authorization']['files'] = [ {"fileKey": file_key_1}, {"fileKey": file_key_2} ] filing = factory_filing(b, filing_json, filing_type='continuationIn') headers = create_header(jwt, [STAFF_ROLE], identifier) - rv = client.delete(f'/api/v2/businesses/{identifier}/filings/{filing.id}', headers=headers) + + with patch.object(business_filings.doc_service, 'delete_document') as mock_drs: + rv = client.delete(f'/api/v2/businesses/{identifier}/filings/{filing.id}', headers=headers) assert rv.status_code == HTTPStatus.OK - for file_key in [file_key_1, file_key_2]: - try: - MinioService.get_file_info(file_key) - except S3Error as ex: - assert ex.code == 'NoSuchKey' + assert mock_drs.call_count == 2 + +def test_delete_continuation_in_filing_with_affidavit_in_draft(session, client, jwt): + """Assert that a draft continuationIn filing can be deleted and the affidavit file is removed from DRS.""" + from legal_api.resources.v2.business.business_filings import business_filings -def test_delete_continuation_in_filing_with_affidavit_in_draft(session, client, jwt, minio_server): - """Assert that a draft continuationIn filing can be deleted and the affidavit file is removed from Minio.""" identifier = 'CP1234567' b = factory_business(identifier) @@ -1070,21 +1194,22 @@ def test_delete_continuation_in_filing_with_affidavit_in_draft(session, client, "files": [] } } - file_key = _upload_file(letter, invalid=False) + file_key = 'CORP-DS0000101953' filing_json['filing']['continuationIn']['foreignJurisdiction']['affidavitFileKey'] = file_key filing = factory_filing(b, filing_json, filing_type='continuationIn') headers = create_header(jwt, [STAFF_ROLE], identifier) - rv = client.delete(f'/api/v2/businesses/{identifier}/filings/{filing.id}', headers=headers) + + with patch.object(business_filings.doc_service, 'delete_document') as mock_drs: + rv = client.delete(f'/api/v2/businesses/{identifier}/filings/{filing.id}', headers=headers) assert rv.status_code == HTTPStatus.OK - try: - MinioService.get_file_info(file_key) - except S3Error as ex: - assert ex.code == 'NoSuchKey' + mock_drs.assert_called_once() -def test_delete_dissolution_filing_in_draft_with_file_in_minio(session, client, jwt, minio_server): - """Assert that a draft filing can be deleted.""" +def test_delete_dissolution_filing_in_draft_with_file_in_drs(session, client, jwt): + """Assert that a draft filing can be deleted and DRS file is removed.""" + from legal_api.resources.v2.business.business_filings import business_filings + identifier = 'CP7654321' b = factory_business(identifier) @@ -1092,17 +1217,16 @@ def test_delete_dissolution_filing_in_draft_with_file_in_minio(session, client, filing_json['filing']['header']['name'] = 'dissolution' filing_json['filing']['business']['legalType'] = 'CP' filing_json['filing']['dissolution'] = copy.deepcopy(DISSOLUTION) - file_key = _upload_file(letter, invalid=False) + file_key = 'COOP-DS0000101954' filing_json['filing']['dissolution']['affidavitFileKey'] = file_key filing = factory_filing(b, filing_json, filing_type='dissolution') headers = create_header(jwt, [STAFF_ROLE], identifier) - rv = client.delete(f'/api/v2/businesses/{identifier}/filings/{filing.id}', headers=headers) + + with patch.object(business_filings.doc_service, 'delete_document') as mock_drs: + rv = client.delete(f'/api/v2/businesses/{identifier}/filings/{filing.id}', headers=headers) assert rv.status_code == HTTPStatus.OK - try: - MinioService.get_file_info(file_key) - except S3Error as ex: - assert ex.code == 'NoSuchKey' + mock_drs.assert_called_once() def test_delete_filing_block_completed(session, client, jwt): @@ -1214,6 +1338,7 @@ def test_update_ar_with_a_missing_filing_id_fails(session, client, jwt): ) factory_business_mailing_address(business) ar = copy.deepcopy(ANNUAL_REPORT) + ar['filing']['business'] = {'identifier': identifier} annual_report_date = datetime(datetime.now(UTC).year, 2, 20).date() if annual_report_date > datetime.now(UTC).date(): annual_report_date = datetime.now(UTC).date() @@ -1234,11 +1359,37 @@ def test_update_ar_with_a_missing_business_id_fails(session, client, jwt): """Assert that updating to a non-existant business fails.""" import copy identifier = 'CP7654321' + invalid_business_identifier = 'CP0000001' business = factory_business(identifier, founding_date=(datetime.now(UTC) - datedelta.YEAR) ) factory_business_mailing_address(business) ar = copy.deepcopy(ANNUAL_REPORT) + ar['filing']['business'] = {'identifier': invalid_business_identifier} + ar['filing']['annualReport']['annualReportDate'] = datetime.now(UTC).date().isoformat() + ar['filing']['annualReport']['annualGeneralMeetingDate'] = datetime.now(UTC).date().isoformat() + + filings = factory_completed_filing(business, ar) + + rv = client.put(f'/api/v2/businesses/{invalid_business_identifier}/filings/{filings.id+1}', + json=ar, + headers=create_header(jwt, [STAFF_ROLE], identifier) + ) + + assert rv.status_code == HTTPStatus.BAD_REQUEST + assert rv.json['errors'][0] == {'message': 'A valid business is required.'} + + +def test_update_ar_with_a_different_business_id_fails(session, client, jwt): + """Assert that updating to a non-existant business fails.""" + import copy + identifier = 'CP7654321' + business = factory_business(identifier, + founding_date=(datetime.now(UTC) - datedelta.YEAR) + ) + factory_business_mailing_address(business) + ar = copy.deepcopy(ANNUAL_REPORT) + ar['filing']['business'] = {'identifier': identifier} ar['filing']['annualReport']['annualReportDate'] = datetime.now(UTC).date().isoformat() ar['filing']['annualReport']['annualGeneralMeetingDate'] = datetime.now(UTC).date().isoformat() @@ -1251,7 +1402,7 @@ def test_update_ar_with_a_missing_business_id_fails(session, client, jwt): ) assert rv.status_code == HTTPStatus.BAD_REQUEST - assert rv.json['errors'][0] == {'message': 'A valid business is required.'} + assert rv.json['errors'][0] == {'message': 'filing/business/identifier does not equal the identifier in the request path.'} def test_update_ar_with_missing_json_body_fails(session, client, jwt): @@ -1283,6 +1434,7 @@ def test_file_ar_no_agm_coop(session, client, jwt): ) factory_business_mailing_address(business) ar = copy.deepcopy(ANNUAL_REPORT) + ar['filing']['business'] = {'identifier': identifier} annual_report_date = datetime(datetime.now(UTC).year, 2, 20).date() if annual_report_date > LegislationDatetime.now().date(): annual_report_date = LegislationDatetime.now().date() @@ -1664,6 +1816,34 @@ def test_coa(session, requests_mock, client, jwt, test_name, legal_type, identif assert 'futureEffectiveDate' not in rv.json['filing']['header'] +def test_create_invoice_forwards_account_linking_key(session, requests_mock, client, jwt): + """Assert that create_invoice forwards the Account-Linking-Key header when present on the request.""" + identifier = 'CP1234567' + coa = copy.deepcopy(FILING_HEADER) + coa['filing']['header']['name'] = 'changeOfAddress' + coa['filing']['changeOfAddress'] = CHANGE_OF_ADDRESS + coa['filing']['changeOfAddress']['offices']['registeredOffice']['deliveryAddress']['addressCountry'] = 'CA' + coa['filing']['changeOfAddress']['offices']['registeredOffice']['mailingAddress']['addressCountry'] = 'CA' + coa['filing']['business']['identifier'] = identifier + + b = factory_business(identifier, (datetime.now(UTC) - datedelta.YEAR), None, Business.LegalTypes.COOP.value) + factory_business_mailing_address(b) + + pay_mock = requests_mock.post(current_app.config.get('PAYMENT_SVC_URL'), + json={'id': 21322, + 'statusCode': 'COMPLETED', + 'isPaymentActionRequired': False}, + status_code=HTTPStatus.CREATED) + rv = client.post(f'/api/v2/businesses/{identifier}/filings', + json=coa, + headers=create_header(jwt, [STAFF_ROLE], identifier, + **{'Account-Linking-Key': 'test-linking-key'}) + ) + + assert rv.status_code == HTTPStatus.CREATED + assert pay_mock.last_request.headers.get('Account-Linking-Key') == 'test-linking-key' + + def test_rules_memorandum_in_sr(session, mocker, requests_mock, client, jwt, ): """Assert if both rules update in sr, and rules file key is provided""" mocker.patch('legal_api.services.filings.validations.alteration.validate_pdf', @@ -1728,6 +1908,7 @@ def mockFlagsValue(flag, _user=None, _account_id=None): temp_reg._identifier = identifier temp_reg.save() json_data = copy.deepcopy(CONTINUATION_IN_FILING_TEMPLATE) + json_data['filing']['business'] = {'identifier': identifier} del json_data['filing']['continuationIn']['parties'][1] filing = factory_filing(None, json_data) filing.temp_reg = identifier @@ -1767,10 +1948,10 @@ def mockFlagsValue(flag, _user=None, _account_id=None): 'legal_api.resources.v2.business.business_filings.business_filings.ListFilingResource.check_and_update_nr', return_value=None) mocker.patch('legal_api.resources.v2.business.business_filings.business_filings.publish_to_queue', return_value=None) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=None) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=[]) mocker.patch('legal_api.services.filings.validations.continuation_in.validate_name_request', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_business_in_colin', + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_continuation_in_expro_business_in_colin', return_value=[]) if filing_status == Filing.Status.APPROVED.value: @@ -1833,6 +2014,7 @@ def test_resubmit_filing_failed(session, client, jwt, filing_status, review_stat temp_reg._identifier = identifier temp_reg.save() json_data = copy.deepcopy(CONTINUATION_IN_FILING_TEMPLATE) + json_data['filing']['business'] = {'identifier': identifier} filing = factory_filing(None, json_data) filing.temp_reg = identifier filing._status = filing_status @@ -1878,7 +2060,7 @@ def test_notice_of_withdrawal_filing(session, client, jwt, test_name, legal_type temp_reg.save() json_data = copy.deepcopy(FILING_HEADER) json_data['filing']['header']['name'] = filing_type - del json_data['filing']['business'] + json_data['filing']['business'] = {'identifier': identifier} new_bus_filing_json = copy.deepcopy(filing_json) new_bus_filing_json['nameRequest']['legalType'] = legal_type json_data['filing'][filing_type] = new_bus_filing_json @@ -2316,4 +2498,38 @@ def test_ta(session, requests_mock, client, jwt, monkeypatch, test_name, legal_t else: assert rv.status_code == HTTPStatus.FORBIDDEN assert rv.json[0]['message'] == 'Permission Denied - transition filing is currently not available for this user and/or account.' - \ No newline at end of file + + +def test_delete_uploaded_file_dispatch(session): + """Assert uploaded files are deleted from DRS.""" + from legal_api.resources.v2.business.business_filings import business_filings + + file_key = 'COOP-DS0000101951' + with patch.object(business_filings.doc_service, 'delete_document') as mock_drs: + ListFilingResource.delete_uploaded_file(file_key) + + mock_drs.assert_called_once() + assert mock_drs.call_args[0][0].file_key == file_key + + +def test_delete_dissolution_filing_in_draft_with_drs_file(session, client, jwt): + """Assert that deleting a draft dissolution removes its DRS-backed affidavit from the DRS.""" + from legal_api.resources.v2.business.business_filings import business_filings + + identifier = 'CP7654321' + file_key = 'COOP-DS0000101951' + b = factory_business(identifier) + filing_json = copy.deepcopy(FILING_HEADER) + filing_json['filing']['header']['name'] = 'dissolution' + filing_json['filing']['business']['legalType'] = 'CP' + filing_json['filing']['dissolution'] = copy.deepcopy(DISSOLUTION) + filing_json['filing']['dissolution']['affidavitFileKey'] = file_key + filing = factory_filing(b, filing_json, filing_type='dissolution') + headers = create_header(jwt, [STAFF_ROLE], identifier) + + with patch.object(business_filings.doc_service, 'delete_document') as mock_drs: + rv = client.delete(f'/api/v2/businesses/{identifier}/filings/{filing.id}', headers=headers) + + assert rv.status_code == HTTPStatus.OK + mock_drs.assert_called_once() + assert mock_drs.call_args[0][0].file_key == file_key diff --git a/legal-api/tests/unit/resources/v2/test_business_filings/test_filings_ledger.py b/legal-api/tests/unit/resources/v2/test_business_filings/test_filings_ledger.py index af1c094b75..a137cdca9d 100644 --- a/legal-api/tests/unit/resources/v2/test_business_filings/test_filings_ledger.py +++ b/legal-api/tests/unit/resources/v2/test_business_filings/test_filings_ledger.py @@ -24,7 +24,7 @@ import datedelta import pytest -from business_model.models import Business, Comment, Filing as FilingStorage, UserRoles +from business_model.models import Business, Comment, CourtOrder, Filing as FilingStorage, UserRoles from legal_api.core import Filing, FILINGS from legal_api.services.authz import STAFF_ROLE from registry_schemas.example_data import ( @@ -47,6 +47,13 @@ from tests.unit.services.utils import create_header REGISTER_CORRECTION_APPLICATION = 'Register Correction Application' + + +def mock_auth(headers: dict): + """Return a stub for flask.request.headers.get.""" + return lambda key, default=None: headers.get(key, default) + + def test_get_all_business_filings_only_one_in_ledger(app, session, client, jwt, monkeypatch, mock_drs_service, mocker): """Assert that the business info can be received in a valid JSONSchema format.""" import copy @@ -60,12 +67,9 @@ def test_get_all_business_filings_only_one_in_ledger(app, session, client, jwt, print('test_get_all_business_filings - filing:', filings) headers=create_header(jwt, [STAFF_ROLE], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) @@ -90,12 +94,9 @@ def test_get_all_business_filings_multi_in_ledger(app, session, client, jwt, mon datetime.date(add_years(datetime(2001, 8, 5, 7, 7, 58, 272362), i)).isoformat() factory_filing(b, ar) headers=create_header(jwt, [STAFF_ROLE], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) @@ -111,12 +112,9 @@ def test_ledger_search(app, session, client, jwt, monkeypatch, mock_drs_service, business = factory_business(identifier=identifier, founding_date=founding_date, last_ar_date=None, entity_type=Business.LegalTypes.BCOMP.value) num_of_files = load_ledger(business, founding_date) headers=create_header(jwt, [UserRoles.system], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) @@ -177,12 +175,9 @@ def test_ledger_comment_count(app, session, client, jwt, monkeypatch, mock_drs_s filing_storage.comments.append(comment) filing_storage.save() headers=create_header(jwt, [UserRoles.system], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) @@ -222,12 +217,9 @@ def test_get_all_business_filings_permitted_statuses(app, session, client, jwt, filing_storage.skip_status_listener = True filing_storage.save() headers=create_header(jwt, [UserRoles.system], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) @@ -255,21 +247,33 @@ def test_ledger_court_order(app, session, client, jwt, test_name, file_number, o """Assert that the ledger returns court_order values.""" # setup identifier = 'BC1234567' - business, filing_storage = ledger_element_setup_help(identifier) + business, filing = ledger_element_setup_help(identifier, filing_name='courtOrder') - filing_storage.court_order_file_number = file_number - filing_storage.court_order_date = order_date - filing_storage.court_order_effect_of_order = effect_of_order - filing_storage.order_details = order_details + if expected: + filing.court_orders.append(CourtOrder( + business_id=business.id, + file_number=file_number, + effect_of_order=effect_of_order, + order_details=order_details, + order_date=order_date + )) + court_order = {"fileNumber": file_number} + if effect_of_order: + court_order["effectOfOrder"] = effect_of_order + if order_date: + court_order["orderDate"] = order_date.isoformat() + if order_details: + court_order["orderDetails"] = order_details + + filing._meta_data = { + "courtOrder": court_order + } + filing.save() - filing_storage.save() headers=create_header(jwt, [UserRoles.system], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) @@ -299,12 +303,9 @@ def test_ledger_display_name_annual_report(app, session, client, jwt, monkeypatc filing_storage._meta_data = meta_data filing_storage.save() headers=create_header(jwt, [UserRoles.system], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) @@ -325,12 +326,9 @@ def test_ledger_display_unknown_name(app, session, client, jwt, monkeypatch, moc filing_storage._meta_data = meta_data filing_storage.save() headers=create_header(jwt, [UserRoles.system], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) @@ -356,12 +354,9 @@ def test_ledger_display_alteration_report(app, session, client, jwt, monkeypatch filing_storage._meta_data = meta_data filing_storage.save() headers=create_header(jwt, [UserRoles.system], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) @@ -400,12 +395,9 @@ def test_ledger_display_restoration(app, session, client, jwt, restoration_type, factory_completed_filing(business, filing, filing_date=filing_date) headers=create_header(jwt, [UserRoles.system], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) @@ -453,12 +445,9 @@ def test_ledger_display_incorporation(app, session, client, jwt, test_name, enti } f._meta_data = {**{'applicationDate': today}, **ia_meta} headers=create_header(jwt, [UserRoles.system], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) @@ -476,12 +465,9 @@ def test_ledger_display_corrected_incorporation(app, session, client, jwt, monke original.parent_filing_id = correction.id original.save() headers=create_header(jwt, [UserRoles.system], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) @@ -516,12 +502,9 @@ def test_ledger_display_corrected_annual_report(app, session, client, jwt, monke correction._meta_data = {**{'applicationDate': today}, **correction_meta} correction.save() headers=create_header(jwt, [UserRoles.system], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) @@ -582,12 +565,9 @@ def test_ledger_redaction(app, session, client, jwt, test_name, submitter_role, setattr(new_filing, 'skip_status_listener', True) # skip status listener new_filing.save() headers=create_header(jwt, [jwt_role], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) except Exception as err: @@ -635,12 +615,9 @@ def test_ledger_display_special_resolution_correction(app, session, client, jwt, correction_2._meta_data = {**{'applicationDate': today}, **correction_2_meta} correction_2.save() headers=create_header(jwt, [UserRoles.system], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) @@ -677,12 +654,9 @@ def test_ledger_display_non_special_resolution_correction_name(app, session, cli correction._meta_data = {**{'applicationDate': today}, **correction_meta} correction.save() headers=create_header(jwt, [UserRoles.system], identifier) - def mock_auth(one, two): # pylint: disable=unused-argument; mocks of library methods - return headers[one] - # test with app.test_request_context(): - monkeypatch.setattr('flask.request.headers.get', mock_auth) + monkeypatch.setattr('flask.request.headers.get', mock_auth(headers)) rv = client.get(f'/api/v2/businesses/{identifier}/filings', headers=headers) diff --git a/legal-api/tests/unit/resources/v2/test_business_tasks.py b/legal-api/tests/unit/resources/v2/test_business_tasks.py index f40c94388f..2c04f942c7 100644 --- a/legal-api/tests/unit/resources/v2/test_business_tasks.py +++ b/legal-api/tests/unit/resources/v2/test_business_tasks.py @@ -230,6 +230,35 @@ def test_get_tasks_error_filings(session, client, jwt): assert rv.json['tasks'][0]['task']['filing']['header']['filingId'] == filing.id +def test_get_tasks_forwards_account_linking_key(app, session, client, jwt, requests_mock): + """Assert that the pay-api call forwards the Account-Linking-Key header when present on the request.""" + from business_model.models import Filing + from tests.unit.models import AR_FILING + + identifier = 'CP7654321' + b = factory_business(identifier, last_ar_date=datetime(2019, 8, 13)) + factory_business_mailing_address(b) + + filing = Filing() + filing.business_id = b.id + filing.filing_date = datetime(2019, 8, 5, 7, 7, 58, 272362) + filing.filing_json = AR_FILING + filing.payment_token = 2 + filing.payment_status_code = 'CREATED' + filing.save() + + pay_mock = requests_mock.get(f"{app.config.get('PAYMENT_SVC_URL')}/{filing.payment_token}", + json={'isPaymentActionRequired': False, 'paymentMethod': 'DIRECT_PAY'}, + status_code=HTTPStatus.OK) + + rv = client.get(f'/api/v2/businesses/{identifier}/tasks', + headers=create_header(jwt, [STAFF_ROLE], identifier, + **{'Account-Linking-Key': 'test-linking-key'})) + + assert rv.status_code == HTTPStatus.OK + assert pay_mock.last_request.headers.get('Account-Linking-Key') == 'test-linking-key' + + def test_get_tasks_pending_correction_filings(session, client, jwt): """Assert that to-do list returns the error filings.""" from freezegun import freeze_time diff --git a/legal-api/tests/unit/resources/v2/test_document.py b/legal-api/tests/unit/resources/v2/test_document.py index f398787b38..316ee12cce 100644 --- a/legal-api/tests/unit/resources/v2/test_document.py +++ b/legal-api/tests/unit/resources/v2/test_document.py @@ -19,16 +19,234 @@ from http import HTTPStatus +import pytest +from flask import current_app + +from legal_api.resources.v2 import document from legal_api.services.authz import STAFF_ROLE from tests.unit.services.utils import create_header -def test_documents_signature_get_returns_200(client, jwt, session, minio_server): # pylint:disable=unused-argument - """Assert get documents/filename/signatures endpoint returns 200.""" +MOCK_DRS_URL = "https://test.api.connect.gov.bc.ca/mockTarget/doc/api/v1" +TEST_DATAFILE = 'tests/unit/services/test_doc.pdf' +PATCH_VALID_PAYLOAD = { + "businessIdentifier": "BC9901019", + "filingId": "1199642", + "filingDate": "2024-08-02T19:00:00+00:00" +} +# testdata pattern is ({filekey}, {drs_id}) +TEST_DRS_ID_DATA = [ + ("", ""), + ("DS0100001003", "DS0100001003"), + ("CORP-DS0100001003", "DS0100001003"), + ("COOP-DS0100001003", "DS0100001003"), + ("FIRM-DS0100001003", "DS0100001003"), +] +# testdata pattern is ({filekey}, {doc_class}) +TEST_DRS_CLASS_DATA = [ + ("", "CORP"), + ("DS0100001003", "CORP"), + ("CORP-DS0100001003", "CORP"), + ("COOP-DS0100001003", "COOP"), + ("FIRM-DS0100001003", "FIRM"), +] +# testdata pattern is ({filing_type}, {entity_type}, {doc_type}, {drs_class}, {drs_type}) +TEST_CREATE_INFO_DATA = [ + ("JUNK", "JUNK", "JUNK", "CORP", "COSD"), + ("specialResolution", "CP", "coop_memorandum", "COOP", "COOP_MEMORANDUM"), + ("specialResolution", "CP", "coop_rules", "COOP", "COOP_RULES"), + ("incorporationApplication", "CP", "coop_memorandum", "COOP", "COOP_MEMORANDUM"), + ("incorporationApplication", "CP", "coop_rules", "COOP", "COOP_RULES"), + ("correction", "CP", "coop_memorandum", "COOP", "COOP_MEMORANDUM"), + ("correction", "CP", "coop_rules", "COOP", "COOP_RULES"), + ("dissolution", "CP", "affidavit", "COOP", "COSD"), + ("courtOrder", "BC", "court_order", "CORP", "CRTO"), + ("courtOrder", "BEN", "court_order", "CORP", "CRTO"), + ("courtOrder", "C", "court_order", "CORP", "CRTO"), + ("courtOrder", "CC", "court_order", "CORP", "CRTO"), + ("courtOrder", "CUL", "court_order", "CORP", "CRTO"), + ("courtOrder", "ULC", "court_order", "CORP", "CRTO"), + ("courtOrder", "CP", "court_order", "COOP", "CRTO"), + ("courtOrder", "SP", "court_order", "FIRM", "CRTO"), + ("courtOrder", "GP", "court_order", "FIRM", "CRTO"), + ("continuationOut", "BC", "continuation_out", "CORP", "CNTO"), + ("continuationOut", "BEN", "continuation_out", "CORP", "CNTO"), + ("continuationOut", "C", "continuation_out", "CORP", "CNTO"), + ("continuationIn", "C", "authorization_file", "CORP", "CNTA"), + ("continuationIn", "C", "director_affidavit", "CORP", "DIRECTOR_AFFIDAVIT"), + ("continuationIn", "CBEN", "authorization_file", "CORP", "CNTA"), + ("continuationIn", "CBEN", "director_affidavit", "CORP", "DIRECTOR_AFFIDAVIT"), + ("continuationIn", "CCC", "authorization_file", "CORP", "CNTA"), + ("continuationIn", "CCC", "director_affidavit", "CORP", "DIRECTOR_AFFIDAVIT"), + ("continuationIn", "CUL", "authorization_file", "CORP", "CNTA"), + ("continuationIn", "CUL", "director_affidavit", "CORP", "DIRECTOR_AFFIDAVIT"), +] +# testdata pattern is ({filename}, {identifier}, {filedate}, {filing_id}) +TEST_CREATE_INFO_DATA_EXTRA = [ + (None, None, None, None), + ("filename.pdf", "BC1500000", "2024-08-01T23:03:45+00:00", 2000000), +] +# testdata pattern is ({description}, {file_key}, {accept}) +TEST_CLIENT_GET_DATA = [ + ("Valid file key binary", "CORP-DS0100001000", None), + ("Valid file key JSON", "CORP-DS0100001000", "application/json"), +] +# testdata pattern is ({description}, {file_key}, {payload}) +TEST_CLIENT_PATCH_DATA = [ + ("Valid Request", "CORP-DS0100001003", PATCH_VALID_PAYLOAD), +] +# testdata pattern is ({description}, {file_key}) +TEST_CLIENT_DELETE_DATA = [ + ("Valid Request", "CORP-DS0100001000"), +] +# testdata pattern is ({description}, {file_key}) +TEST_CLIENT_PUT_DATA = [ + ("Valid Request", "CORP-DS0100001000"), +] +# testdata pattern is ({description}, {filing_type}, {entity_type}, {doc_type}) +TEST_CLIENT_POST_DATA = [ + ("Valid Request", "courtOrder", "BC", "court_order"), +] + + + +@pytest.mark.parametrize('desc,filing_type,entity_type,doc_type', TEST_CLIENT_POST_DATA) +def test_create_client_document(session, client, jwt, desc, filing_type, entity_type, doc_type): + """Assert that adding/replacing a document by file key returns the expected result.""" + # setup + current_app.config.update(DOCUMENT_SVC_URL=MOCK_DRS_URL) + headers = create_header(jwt, [STAFF_ROLE]) + raw_data = None + with open(TEST_DATAFILE, 'rb') as data_file: + raw_data = data_file.read() + data_file.close() + + rv = client.post( + f"/api/v2/documents/client/{filing_type}/{entity_type}/{doc_type}", + headers=headers, + content_type="application/pdf", + data=raw_data) + # check + assert rv.status_code == HTTPStatus.CREATED + assert rv.json + assert rv.json.get("key") + + +@pytest.mark.parametrize('desc,file_key,accept', TEST_CLIENT_GET_DATA) +def test_get_client_document(session, client, jwt, desc, file_key, accept): + """Assert that get document by file key returns the expected result.""" + # setup + current_app.config.update(DOCUMENT_SVC_URL=MOCK_DRS_URL) headers = create_header(jwt, [STAFF_ROLE]) - file_name = 'test_file.jpeg' - rv = client.get(f'/api/v2/documents/{file_name}/signatures', headers=headers, content_type='application/json') + if accept: + headers["Accept"] = accept + rv = client.get(f"/api/v2/documents/client/{file_key}", headers=headers, content_type="application/json") + # check + assert rv + assert rv.status_code == HTTPStatus.OK + +@pytest.mark.parametrize('desc,file_key', TEST_CLIENT_DELETE_DATA) +def test_delete_client_document(session, client, jwt, desc, file_key): + """Assert that delete document by file key returns the expected result.""" + # setup + current_app.config.update(DOCUMENT_SVC_URL=MOCK_DRS_URL) + headers = create_header(jwt, [STAFF_ROLE]) + + rv = client.delete(f"/api/v2/documents/client/{file_key}", headers=headers, content_type="application/json") + # check + assert rv assert rv.status_code == HTTPStatus.OK - assert 'key' in rv.json and 'preSignedUrl' in rv.json + + +@pytest.mark.parametrize('desc,file_key', TEST_CLIENT_PUT_DATA) +def test_replace_client_document(session, client, jwt, desc, file_key): + """Assert that adding/replacing a document by file key returns the expected result.""" + # setup + current_app.config.update(DOCUMENT_SVC_URL=MOCK_DRS_URL) + headers = create_header(jwt, [STAFF_ROLE]) + raw_data = None + with open(TEST_DATAFILE, 'rb') as data_file: + raw_data = data_file.read() + data_file.close() + + rv = client.put( + f"/api/v2/documents/client/{file_key}", + headers=headers, + content_type="application/pdf", + data=raw_data) + # check + assert rv + assert rv.status_code in (HTTPStatus.OK, HTTPStatus.ACCEPTED) + + +@pytest.mark.parametrize('desc,file_key,payload', TEST_CLIENT_PATCH_DATA) +def test_update_client_document_info(session, client, jwt, desc, file_key, payload): + """Assert that update document record information works as expected.""" + # setup + current_app.config.update(DOCUMENT_SVC_URL=MOCK_DRS_URL) + headers = create_header(jwt, [STAFF_ROLE]) + rv = client.patch( + f"/api/v2/documents/client/{file_key}", + headers=headers, + content_type="application/json", + json=payload + ) + # check + assert rv + assert rv.status_code in (HTTPStatus.OK, HTTPStatus.ACCEPTED) + + +@pytest.mark.parametrize('filing_type,entity_type,doc_type,drs_class,drs_type', TEST_CREATE_INFO_DATA) +def test_create_info(session, jwt, filing_type, entity_type, doc_type, drs_class, drs_type): + """Assert that building the DRS info from the created document request produces the expected result.""" + request_params = {} + info: dict = document.build_create_info(filing_type, entity_type, doc_type, request_params) + assert drs_class == info.get("documentClass") + assert drs_type == info.get("documentType") + + +@pytest.mark.parametrize('filename,identifier,filedate,filing_id', TEST_CREATE_INFO_DATA_EXTRA) +def test_create_info_extra(session, jwt, filename, identifier, filedate, filing_id): + """Assert that building the DRS info from the created document request produces the expected result.""" + request_params = {} + if filename: + request_params[document.PARAM_FILENAME] = filename + if identifier: + request_params[document.PARAM_IDENTIFIER] = identifier + if filedate: + request_params[document.PARAM_FILEDATE] = filedate + if filing_id: + request_params[document.PARAM_FILING_ID] = filing_id + info: dict = document.build_create_info("specialResolution", "CP", "coop_memorandum", request_params) + if filename: + assert info.get("consumerFilename") == filename + else: + assert not info.get("consumerFilename") + if identifier: + assert info.get("consumerIdentifier") == identifier + else: + assert not info.get("consumerIdentifier") + if filedate: + assert info.get("consumerFilingDate") == filedate + else: + assert not info.get("consumerFilingDate") + if filing_id: + assert info.get("consumerReferenceId") == str(filing_id) + else: + assert not info.get("consumerReferenceId") + + +@pytest.mark.parametrize('filekey,drs_id', TEST_DRS_ID_DATA) +def test_get_drs_id(session, jwt, filekey, drs_id): + """Assert that extracting the DRS id from a business documents record filekey produces the expected result.""" + result = document.get_drs_id_from_key(filekey) + assert result == drs_id + + +@pytest.mark.parametrize('filekey,doc_class', TEST_DRS_CLASS_DATA) +def test_get_drs_class(session, jwt, filekey, doc_class): + """Assert that extracting the DRS doc class from a business documents record filekey produces the expected result.""" + result = document.get_drs_class_from_key(filekey) + assert result == doc_class diff --git a/legal-api/tests/unit/services/filings/test_schemas.py b/legal-api/tests/unit/services/filings/test_schemas.py index c45a50ae17..6b95bcf978 100644 --- a/legal-api/tests/unit/services/filings/test_schemas.py +++ b/legal-api/tests/unit/services/filings/test_schemas.py @@ -55,15 +55,12 @@ def test_validate_schema_bad_cr(app): """Assert that an invalid AR returns an error.""" # validate_schema(json_data: Dict = None) -> Tuple(int, str): cr = copy.deepcopy(CORRECTION_REGISTRATION) - cr['filing']['correction']['parties'][0]['officer']['firstName'] = \ - 'this_first_name_maximum_length_is_over' - cr['filing']['correction']['parties'][0]['officer']['middleName'] = \ - 'this_first_name_maximum_length_is_over' + cr['filing']['correction']['parties'][0]['officer']['firstName'] = 'a' * 61 with app.app_context(): err = schemas.validate_against_schema(cr) - assert {'error': "", 'path': 'filing', 'context': []}.keys() == err.msg[0].keys() - assert "is not valid under any of the given schemas" in err.msg[0]['error'] + assert "is too long" in err.msg[0]['error'] + assert err.msg[0]['path'] == 'correction/parties/0/officer/firstName' assert err.code == HTTPStatus.UNPROCESSABLE_ENTITY diff --git a/legal-api/tests/unit/services/filings/test_utils.py b/legal-api/tests/unit/services/filings/test_utils.py index 88fbb18790..08f0b215f3 100644 --- a/legal-api/tests/unit/services/filings/test_utils.py +++ b/legal-api/tests/unit/services/filings/test_utils.py @@ -13,17 +13,19 @@ # limitations under the License. """Test suite to ensure the Common Utilities are working correctly.""" import io +import random from datetime import date -import requests from hypothesis import example, given from hypothesis.strategies import text from reportlab.lib.pagesizes import letter from reportlab.pdfgen import canvas -from legal_api.services import MinioService from legal_api.services.utils import get_date, get_str +# In-memory store mapping DRS file key -> PDF bytes, used by _upload_file and drs_document_mock. +_drs_store: dict = {} + @given(f=text(), p=text()) @example(f={'filing': {'header': {'date': '2001-08-05'}}}, @@ -50,13 +52,31 @@ def test_get_str(f, p): def _upload_file(page_size, invalid): - signed_url = MinioService.create_signed_put_url('cooperative-test.pdf') - key = signed_url.get('key') - pre_signed_put = signed_url.get('preSignedUrl') + """Create a PDF, store it in _drs_store keyed by a DRS file key, and return the key.""" + pdf_data = _create_pdf_file(page_size, invalid).read() + drs_id = f'DS{random.randint(1000000000, 9999999999)}' + file_key = f'COOP-{drs_id}' + _drs_store[file_key] = pdf_data + return file_key + + +def mock_drs_get_document(monkeypatch): + """Monkeypatch doc_service.get_document to serve PDF bytes from _drs_store. + + Call this at the start of any test that uses _upload_file and then calls validate_pdf. + Works with pytest monkeypatch fixture. + """ + from unittest.mock import MagicMock + + import legal_api.services.filings.validations.common_validations as cv + + def _side_effect(drs_id, doc_class, doc_binary=True): + file_key = f'{doc_class}-{drs_id}' + pdf_bytes = _drs_store.get(file_key, b'') + return MagicMock(ok=True, content=pdf_bytes) - requests.put(pre_signed_put, data=_create_pdf_file(page_size, invalid).read(), - headers={'Content-Type': 'application/octet-stream'}) - return key + monkeypatch.setattr(cv, 'doc_service', + MagicMock(get_document=MagicMock(side_effect=_side_effect))) def _create_pdf_file(page_size, invalid): diff --git a/legal-api/tests/unit/services/filings/validations/alteration/test_court_order.py b/legal-api/tests/unit/services/filings/validations/alteration/test_court_order.py index 0bdd3a14ca..3421b35408 100644 --- a/legal-api/tests/unit/services/filings/validations/alteration/test_court_order.py +++ b/legal-api/tests/unit/services/filings/validations/alteration/test_court_order.py @@ -12,9 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. """Tests to assure the Court Order is validated properly.""" +import copy import pytest -from legal_api.services.filings.validations.common_validations import validate_court_order +from registry_schemas.example_data import COURT_ORDER_FILING_TEMPLATE +from legal_api.services.filings import validate +from legal_api.services.filings.validations.court_order import validate_court_order + +from tests.unit.models import factory_business @pytest.mark.parametrize('invalid_court_order', [ @@ -40,10 +45,13 @@ ]) def test_validate_invalid_court_orders(session, invalid_court_order): """Assert not valid court orders.""" - msg = validate_court_order('/filing/alteration/courtOrder', invalid_court_order) + business = factory_business('BC1234567') + filing = copy.deepcopy(COURT_ORDER_FILING_TEMPLATE) + filing['filing']['courtOrder'] = invalid_court_order + err = validate(business, filing) - assert msg - assert len(msg) > 0 + assert err + assert len(err.msg) > 0 @pytest.mark.parametrize('valid_court_order', [ diff --git a/legal-api/tests/unit/services/filings/validations/test_admin_freeze.py b/legal-api/tests/unit/services/filings/validations/test_admin_freeze.py index f7cff09148..eb12dfa3f9 100644 --- a/legal-api/tests/unit/services/filings/validations/test_admin_freeze.py +++ b/legal-api/tests/unit/services/filings/validations/test_admin_freeze.py @@ -22,7 +22,6 @@ from legal_api.services.filings.validations.admin_freeze import validate from tests.unit.models import factory_business -from tests.unit.services.filings.test_utils import _upload_file from tests.unit.services.filings.validations import lists_are_equal diff --git a/legal-api/tests/unit/services/filings/validations/test_agm_extension.py b/legal-api/tests/unit/services/filings/validations/test_agm_extension.py index c72b3e64b9..eae617ec84 100644 --- a/legal-api/tests/unit/services/filings/validations/test_agm_extension.py +++ b/legal-api/tests/unit/services/filings/validations/test_agm_extension.py @@ -53,7 +53,15 @@ HTTPStatus.BAD_REQUEST, 'Allotted period to request extension has expired.'), ('FAIL_SUBSEQUENT_AGM_MORE_EXT_EXCEED_LIMIT', '2022-10-01', {'year': '2023','isFirstAgm': False, 'extReqForAgmYear': True, 'prevAgmRefDate':'2023-06-01', 'expireDateCurrExt':'2024-06-01'}, - HTTPStatus.BAD_REQUEST, 'Company has received the maximum 12 months of allowable extensions.') + HTTPStatus.BAD_REQUEST, 'Company has received the maximum 12 months of allowable extensions.'), + ('SUCCESS_AGM_YEAR_EQUALS_INCORPORATION', '2023-10-01', + {'year': '2023', 'isFirstAgm': True, 'extReqForAgmYear': False, + 'totalApprovedExt': 6, 'extensionDuration': 6}, + None, None), + ('FAIL_AGM_YEAR_BEFORE_INCORPORATION', '2023-10-01', + {'year': '2022', 'isFirstAgm': True, 'extReqForAgmYear': False, + 'totalApprovedExt': 6, 'extensionDuration': 6}, + HTTPStatus.BAD_REQUEST, 'AGM year cannot be earlier than the incorporation year.'), ] ) def test_validate_agm_extension(session, mocker, test_name, founding_date, agm_ext_json, expected_code, message, monkeypatch): diff --git a/legal-api/tests/unit/services/filings/validations/test_agm_location_change.py b/legal-api/tests/unit/services/filings/validations/test_agm_location_change.py index 2793eeeb95..c08952b404 100644 --- a/legal-api/tests/unit/services/filings/validations/test_agm_location_change.py +++ b/legal-api/tests/unit/services/filings/validations/test_agm_location_change.py @@ -28,41 +28,48 @@ # rejected by schema validation (HTTP 422) instead of the legal-api business check (HTTP 400). SCHEMA_REJECTED = 'SCHEMA_REJECTED' +# Set far enough in the past that founding date never becomes the binding floor +OLD_FOUNDING_DATE = datetime.utcnow().replace(year=datetime.utcnow().year - 10) + @pytest.mark.parametrize( 'test_name, expected_code, message', [ ('INVALID_YEAR', HTTPStatus.UNPROCESSABLE_ENTITY, SCHEMA_REJECTED), - ('FAIL_YEAR-3', HTTPStatus.BAD_REQUEST, 'AGM year must be between -2 or +1 year from current year.'), - ('FAIL_YEAR+2', HTTPStatus.BAD_REQUEST, 'AGM year must be between -2 or +1 year from current year.'), + ('FAIL_YEAR-3', HTTPStatus.BAD_REQUEST, None), + ('FAIL_YEAR+2', HTTPStatus.BAD_REQUEST, None), ('SUCCESS-2', None, None), ('SUCCESS+1', None, None), ('SUCCESS', None, None) ] ) def test_validate_agm_year(session, mocker, test_name, expected_code, message, monkeypatch): - """Assert validate agm year.""" + """Assert validate agm year for an established business (founding date well outside the lookback window).""" monkeypatch.setattr( 'legal_api.services.flags.value', lambda flag, default=None: "BC BEN CC ULC C CBEN CCC CUL" if flag == 'supported-agm-location-change-entities' else default ) - business = factory_business(identifier='BC1234567', entity_type='BC', founding_date=datetime.utcnow()) + business = factory_business(identifier='BC1234567', entity_type='BC', founding_date=OLD_FOUNDING_DATE) filing = copy.deepcopy(FILING_HEADER) filing['filing']['agmLocationChange'] = copy.deepcopy(AGM_LOCATION_CHANGE) filing['filing']['header']['name'] = 'agmLocationChange' + current_year = LegislationDatetime.now().year + expected_min = current_year - 2 + expected_max = current_year + 1 + if test_name == 'INVALID_YEAR': filing['filing']['agmLocationChange']['year'] = 'invalid' elif test_name == 'FAIL_YEAR-3': - filing['filing']['agmLocationChange']['year'] = str(LegislationDatetime.now().year - 3) + filing['filing']['agmLocationChange']['year'] = str(current_year - 3) elif test_name == 'FAIL_YEAR+2': - filing['filing']['agmLocationChange']['year'] = str(LegislationDatetime.now().year + 2) + filing['filing']['agmLocationChange']['year'] = str(current_year + 2) elif test_name == 'SUCCESS-2': - filing['filing']['agmLocationChange']['year'] = str(LegislationDatetime.now().year - 2) + filing['filing']['agmLocationChange']['year'] = str(current_year - 2) elif test_name == 'SUCCESS+1': - filing['filing']['agmLocationChange']['year'] = str(LegislationDatetime.now().year + 1) + filing['filing']['agmLocationChange']['year'] = str(current_year + 1) elif test_name == 'SUCCESS': - filing['filing']['agmLocationChange']['year'] = str(LegislationDatetime.now().year) + filing['filing']['agmLocationChange']['year'] = str(current_year) err = validate(business, filing) # validate outcomes @@ -71,11 +78,54 @@ def test_validate_agm_year(session, mocker, test_name, expected_code, message, m assert err.code == HTTPStatus.UNPROCESSABLE_ENTITY elif not test_name.startswith('SUCCESS'): assert expected_code == err.code - if message: - assert message == err.msg[0]['error'] + assert f'AGM year must be between {expected_min} and {expected_max}.' == err.msg[0]['error'] + else: + assert not err + + +@pytest.mark.parametrize( + 'test_name, founding_years_ago, year_offset_from_now', + [ + # business founded less than 2 years ago: founding year should raise the floor + # above the default lookback year + ('FAIL_BEFORE_FOUNDING_YEAR', 1, -2), + ('SUCCESS_AT_FOUNDING_YEAR', 1, -1), + # business founded this year: floor should equal the current year + ('FAIL_BEFORE_FOUNDING_THIS_YEAR', 0, -1), + ('SUCCESS_FOUNDING_THIS_YEAR', 0, 0), + ] +) +def test_validate_agm_year_founding_date_floor( + session, mocker, test_name, founding_years_ago, year_offset_from_now, monkeypatch +): + """Assert AGM year cannot be set to a year before the business's founding year.""" + monkeypatch.setattr( + 'legal_api.services.flags.value', + lambda flag, default=None: "BC BEN CC ULC C CBEN CCC CUL" if flag == 'supported-agm-location-change-entities' else default + ) + current_year = LegislationDatetime.now().year + founding_date = datetime.utcnow().replace(year=current_year - founding_years_ago) + business = factory_business(identifier='BC1234567', entity_type='BC', founding_date=founding_date) + + filing = copy.deepcopy(FILING_HEADER) + filing['filing']['agmLocationChange'] = copy.deepcopy(AGM_LOCATION_CHANGE) + filing['filing']['header']['name'] = 'agmLocationChange' + filing['filing']['agmLocationChange']['year'] = str(current_year + year_offset_from_now) + + err = validate(business, filing) + + founding_year = current_year - founding_years_ago + expected_min = max(current_year - 2, founding_year) + expected_max = current_year + 1 + + if test_name.startswith('FAIL'): + assert err is not None + assert err.code == HTTPStatus.BAD_REQUEST + assert f'AGM year must be between {expected_min} and {expected_max}.' == err.msg[0]['error'] else: assert not err + @pytest.mark.parametrize( 'test_name, reason, expected_code, message', [ @@ -91,7 +141,7 @@ def test_validate_agm_reason(session, mocker, test_name, reason, expected_code, 'legal_api.services.flags.value', lambda flag, default=None: "BC BEN CC ULC C CBEN CCC CUL" if flag == 'supported-agm-location-change-entities' else default ) - business = factory_business(identifier='BC1234567', entity_type='BC', founding_date=datetime.utcnow()) + business = factory_business(identifier='BC1234567', entity_type='BC', founding_date=OLD_FOUNDING_DATE) filing = copy.deepcopy(FILING_HEADER) filing['filing']['agmLocationChange'] = copy.deepcopy(AGM_LOCATION_CHANGE) filing['filing']['header']['name'] = 'agmLocationChange' diff --git a/legal-api/tests/unit/services/filings/validations/test_alteration.py b/legal-api/tests/unit/services/filings/validations/test_alteration.py index ee7265291a..9ceb889428 100644 --- a/legal-api/tests/unit/services/filings/validations/test_alteration.py +++ b/legal-api/tests/unit/services/filings/validations/test_alteration.py @@ -30,7 +30,7 @@ from legal_api.services.permissions import ListActionsPermissionsAllowed, PermissionService from registry_schemas.example_data import ALTERATION_FILING_TEMPLATE from tests.unit.models import factory_business -from tests.unit.services.filings.test_utils import _upload_file +from tests.unit.services.filings.test_utils import _upload_file, mock_drs_get_document from tests.unit.services.filings.validations import lists_are_equal @@ -513,9 +513,10 @@ def test_alteration_share_classes_optional(session): }]), ]) @patch.object(PermissionService, 'check_user_permission', MagicMock(return_value=None)) -def test_validate_cooperative_documents(session, mocker, minio_server, test_name, key, scenario, expected_code, +def test_validate_cooperative_documents(session, monkeypatch, mocker, test_name, key, scenario, expected_code, expected_msg): """Assert that validator validates cooperative documents correctly.""" + mock_drs_get_document(monkeypatch) identifier = 'CP1234567' business = factory_business(identifier) @@ -694,11 +695,18 @@ def test_alteration_share_class_series_validation(mock_get_parties, session, leg ('SUCCESS', '2020-09-18T00:00:00+00:00', None, None), ('SUCCESS', None, None, None), ('FAIL_INVALID_DATE_TIME_FORMAT', '2020-09-44T00:00:00z', - HTTPStatus.UNPROCESSABLE_CONTENT, [{ - 'path': 'filing/header/effectiveDate', - 'error': "'2020-09-44T00:00:00z' is not a 'date-time'", - 'context': [] - }]), + HTTPStatus.UNPROCESSABLE_CONTENT, [ + { + 'path': 'filing/header/effectiveDate', + 'error': "'2020-09-44T00:00:00z' is not a 'date-time'", + 'context': [] + }, + { + 'path': 'filing/header/effectiveDate', + 'error': "'2020-09-44T00:00:00z' is not a 'date-time'", + 'context': [] + } + ]), ('FAIL_INVALID_DATE_TIME_MINIMUM', '2020-09-17T00:01:00+00:00', HTTPStatus.BAD_REQUEST, [{ 'error': 'Invalid Datetime, effective date must be a minimum of 2 minutes ahead.', diff --git a/legal-api/tests/unit/services/filings/validations/test_amalgamation_application.py b/legal-api/tests/unit/services/filings/validations/test_amalgamation_application.py index 0698a33fbe..7d192fa46e 100644 --- a/legal-api/tests/unit/services/filings/validations/test_amalgamation_application.py +++ b/legal-api/tests/unit/services/filings/validations/test_amalgamation_application.py @@ -20,14 +20,26 @@ import pytest from freezegun import freeze_time -from business_model.models import AmalgamatingBusiness, Amalgamation, Business, Filing +from business_model.models import ( + Address, + AmalgamatingBusiness, + Amalgamation, + Business, + Filing, + Office, + Party, + PartyRole, + Resolution, + ShareClass, + ShareSeries, +) from legal_api.errors import Error from legal_api.services import NameXService, STAFF_ROLE, BASIC_USER, flags -from legal_api.services.filings.validations.amalgamation_application import _validate_foreign_businesses from legal_api.services.filings.validations.validation import validate from legal_api.services.permissions import PermissionService from registry_schemas.example_data import AMALGAMATION_APPLICATION +from tests.unit.models import factory_address, factory_business, factory_business_office from tests.unit.services.filings.validations import create_party, create_party_address, lists_are_equal from tests.unit.services.utils import jwt_request_context @@ -35,9 +47,10 @@ class MockResponse: """Mock http response.""" - def __init__(self, json_data): + def __init__(self, json_data, status_code=HTTPStatus.OK): """Initialize mock http response.""" self.json_data = json_data + self.status_code = status_code def json(self): """Return mock json data.""" @@ -57,13 +70,20 @@ def _mock_nr_response(legal_type): }) -def test_invalid_nr_amalgamation(mocker, app, session): - """Assert that nr is invalid.""" +def _get_amalg_template(): + """Return the Amalgamation Application filing template.""" filing = {'filing': {}} filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', 'certifiedBy': 'full name', 'authorizationReceived': True, 'email': 'no_one@never.get', 'filingId': 1} + filing['filing']['business'] = {'identifier': 'T1234567'} filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + return filing + + +def test_invalid_nr_amalgamation(mocker, app, session): + """Assert that nr is invalid.""" + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' invalid_nr_response = { @@ -93,11 +113,7 @@ def test_invalid_nr_amalgamation(mocker, app, session): ) def test_short_form_amalgamation_rejects_nr(mocker, app, session, amalgamation_type): """Assert short-form amalgamations reject an nrNumber.""" - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['type'] = amalgamation_type filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' @@ -121,11 +137,7 @@ def test_short_form_amalgamation_rejects_nr(mocker, app, session, amalgamation_t ) def test_amalgamation_parties_missing_role(mocker, app, session, amalgamation_type, expected_msg): """Assert that amalgamation party roles can be validated for missing roles.""" - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['type'] = amalgamation_type filing['filing']['amalgamationApplication']['parties'] = [] mocker.patch('legal_api.services.filings.validations.amalgamation_application.validate_name_request', @@ -165,6 +177,7 @@ def test_amalgamation_parties_invalid_role(mocker, app, session, parties, expect 'email': 'no_one@never.get', 'filingId': 1 } + filing['filing']['business'] = {'identifier': 'T1234567'} filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' filing['filing']['amalgamationApplication']['type'] = Amalgamation.AmalgamationTypes.regular.name @@ -366,10 +379,7 @@ def test_validate_amalgamation_office(session, mocker, test_name, legal_type, de delivery_country, mailing_region, mailing_country, expected_code, expected_msg): """Assert that amalgamation offices can be validated.""" - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} + filing = _get_amalg_template() filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) filing['filing']['amalgamationApplication']['nameRequest'] = {} @@ -416,6 +426,7 @@ def test_amalgamation_regular_requires_at_least_one_share_class(mocker, app, ses 'email': 'no_one@never.get', 'filingId': 1 } + filing['filing']['business'] = {'identifier': 'T1234567'} filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) filing['filing']['amalgamationApplication']['type'] = Amalgamation.AmalgamationTypes.regular.name filing['filing']['amalgamationApplication']['shareStructure'] = {'shareClasses': []} @@ -680,11 +691,7 @@ def test_validate_incorporation_share_classes(session, mocker, test_name, legal_ class_name_2, series_name_2, expected_code, expected_msg): """Assert that validator validates share class correctly.""" - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest'] = {} filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' @@ -730,16 +737,13 @@ def test_validate_incorporation_share_classes(session, mocker, test_name, legal_ 'amalgamation_type, expected_code', [ (Amalgamation.AmalgamationTypes.regular.name, HTTPStatus.UNPROCESSABLE_ENTITY), - (Amalgamation.AmalgamationTypes.vertical.name, None), - (Amalgamation.AmalgamationTypes.horizontal.name, None), + (Amalgamation.AmalgamationTypes.vertical.name, HTTPStatus.UNPROCESSABLE_ENTITY), + (Amalgamation.AmalgamationTypes.horizontal.name, HTTPStatus.UNPROCESSABLE_ENTITY), ] ) def test_validate_amalgamation_office_or_share_required(session, mocker, amalgamation_type, expected_code): - """Assert that amalgamation offices/shareStructure required can be validated.""" - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} + """Assert the schema requires offices/shareStructure for every amalgamation type.""" + filing = _get_amalg_template() filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) filing['filing']['amalgamationApplication']['type'] = amalgamation_type @@ -770,10 +774,7 @@ def test_validate_amalgamation_office_or_share_required(session, mocker, amalgam def test_amalgamation_court_orders(mocker, app, session, test_status, file_number, effect_of_order, expected_code, expected_msg): """Assert valid court orders.""" - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} + filing = _get_amalg_template() filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' @@ -806,11 +807,7 @@ def test_amalgamation_court_orders(mocker, app, session, def test_is_business_historical(mocker, app, session, jwt, test_status, expected_code, expected_msg): """Assert valid amalgamating businesses is historical.""" account_id = '123456' - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' def mock_find_by_identifier(identifier): @@ -837,6 +834,41 @@ def mock_find_by_identifier(identifier): assert expected_msg == err.msg[0]['error'] +@pytest.mark.parametrize( + 'test_status, expected_code, expected_msg', + [ + ('FAIL', HTTPStatus.BAD_REQUEST, 'BC1234567 is frozen.'), + ('SUCCESS', None, None) + ] +) +def test_is_business_frozen(mocker, app, session, jwt, test_status, expected_code, expected_msg): + """Assert an admin-frozen amalgamating business cannot amalgamate, staff included.""" + filing = _get_amalg_template() + filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' + + def mock_find_by_identifier(identifier): + return Business(identifier=identifier, + legal_type=Business.LegalTypes.BCOMP.value, + state=Business.State.ACTIVE, + admin_freeze=test_status == 'FAIL') + + mocker.patch('legal_api.services.filings.validations.amalgamation_application.validate_name_request', + return_value=[]) + mocker.patch('legal_api.services.filings.validations.amalgamation_application._has_pending_filing', + return_value=False) + mocker.patch('business_model.models.business.Business.find_by_identifier', side_effect=mock_find_by_identifier) + + with jwt_request_context(app, jwt, [STAFF_ROLE]): + err = validate(None, filing) + + # validate outcomes + if test_status == 'SUCCESS': + assert not err + else: + assert expected_code == err.code + assert expected_msg == err.msg[0]['error'] + + @pytest.mark.parametrize( 'test_status, expected_code, expected_msg', [ @@ -846,11 +878,7 @@ def mock_find_by_identifier(identifier): ) def test_has_pending_filing(mocker, app, session, jwt, test_status, expected_code, expected_msg): """Assert valid amalgamating businesses has draft, pending or future effective filing.""" - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' mocker.patch('legal_api.services.filings.validations.amalgamation_application.validate_name_request', @@ -882,11 +910,7 @@ def test_has_pending_filing(mocker, app, session, jwt, test_status, expected_cod def test_in_future_effective_amalgamation_filing(mocker, app, session, jwt, test_status, expected_code, expected_msg): """Assert valid amalgamating businesses is part of a future effective amalgamation filing.""" - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' mocker.patch('legal_api.services.filings.validations.amalgamation_application.validate_name_request', @@ -920,11 +944,7 @@ def test_in_future_effective_amalgamation_filing(mocker, app, session, jwt, def test_is_business_affliated(mocker, app, session, jwt, test_status, flag_enabled, has_permission, expected_code, expected_msg): """Assert valid amalgamating businesses is affliated.""" account_id = '123456' - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' filing['filing']['amalgamationApplication']['amalgamatingBusinesses'] = [ { @@ -989,11 +1009,7 @@ def mock_find_by_identifier(identifier): def test_is_business_in_good_standing(mocker, app, session, jwt, test_status, flag_enabled, has_permission, expected_code, expected_msg): """Assert valid amalgamating businesses is in good standing.""" account_id = '123456' - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' filing['filing']['amalgamationApplication']['amalgamatingBusinesses'] = [ { @@ -1058,11 +1074,7 @@ def mock_find_by_identifier(identifier): def test_is_business_not_found(mocker, app, session, jwt, test_status, expected_code, expected_msg): """Assert valid amalgamating businesses not found.""" account_id = '123456' - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' filing['filing']['amalgamationApplication']['amalgamatingBusinesses'] = [ { @@ -1090,6 +1102,9 @@ def mock_find_by_identifier(identifier): mocker.patch('legal_api.services.filings.validations.amalgamation_application._is_business_affliated', return_value=True) mocker.patch('business_model.models.business.Business.find_by_identifier', side_effect=mock_find_by_identifier) + # a business not in LEAR is looked up in COLIN - not there either + mocker.patch('legal_api.services.filings.validations.amalgamation_application.colin.get_snapshot', + return_value=({'message': 'not found'}, HTTPStatus.NOT_FOUND)) with jwt_request_context(app, jwt, [BASIC_USER]): err = validate(None, filing, account_id) @@ -1116,11 +1131,7 @@ def mock_find_by_identifier(identifier): def test_amalgamating_foreign_business(mocker, app, session, jwt, test_status, role, flag_enabled, has_permission, expected_code, expected_msg): """Assert valid amalgamating foreign business.""" account_id = '123456' - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' def mock_find_by_identifier(identifier): @@ -1168,11 +1179,7 @@ def test_amalgamating_foreign_business_with_bc_company_to_ulc(mocker, app, sessi test_status, role, expected_code, expected_msg): """Assert valid amalgamating foreign business with bc company to form ulc.""" account_id = '123456' - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' if test_status == 'FAIL': filing['filing']['amalgamationApplication']['nameRequest']['legalType'] = 'ULC' @@ -1212,11 +1219,7 @@ def test_amalgamating_foreign_business_with_ulc_company(mocker, app, session, jw test_status, role, expected_code, expected_msg): """Assert valid amalgamating foreign business with ulc company.""" account_id = '123456' - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' def mock_find_by_identifier(identifier): @@ -1255,11 +1258,7 @@ def test_amalgamating_cc_to_cc(mocker, app, session, jwt, test_status, expected_code, expected_msg): """Assert valid amalgamating cc to cc.""" account_id = '123456' - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['legalType'] = 'CC' filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' @@ -1299,11 +1298,7 @@ def mock_find_by_identifier(identifier): def test_amalgamating_expro_to_cc_or_ulc(mocker, app, session, jwt, test_status, legal_type): """Assert valid amalgamating expro with bc company to cc or ulc.""" account_id = '123456' - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['legalType'] = legal_type filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' filing['filing']['amalgamationApplication']['amalgamatingBusinesses'] = [ @@ -1358,11 +1353,7 @@ def mock_find_by_identifier(identifier): def test_regular_amalgamation_adoptable_name(mocker, app, session, jwt, test_status, legal_type): """Assert valid regular amalgamation adoptable name.""" account_id = '123456' - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['legalType'] = legal_type adoptable_name = f'Test adoptable name {legal_type}' filing['filing']['amalgamationApplication']['nameRequest']['legalName'] = adoptable_name @@ -1428,11 +1419,7 @@ def test_duplicate_amalgamating_businesses(mocker, app, session, jwt, test_statu expected_code, expected_msg): """Assert duplicate amalgamating businesses.""" account_id = '123456' - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' filing['filing']['amalgamationApplication']['amalgamatingBusinesses'] = amalgamating_businesses @@ -1514,11 +1501,7 @@ def test_amalgamating_business_roles(mocker, app, session, jwt, amalgamation_typ amalgamating_businesses, expected_code, expected_msg): """Assert amalgamating business roles are valid.""" account_id = '123456' - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['type'] = amalgamation_type if amalgamation_type == Amalgamation.AmalgamationTypes.regular.name: filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' @@ -1530,6 +1513,8 @@ def mock_find_by_identifier(identifier): mocker.patch('legal_api.services.filings.validations.amalgamation_application.validate_name_request', return_value=[]) + mocker.patch('legal_api.services.filings.validations.amalgamation_application.validate_primary_or_holding_match', + return_value=[]) mocker.patch('legal_api.services.filings.validations.amalgamation_application._has_pending_filing', return_value=False) mocker.patch('business_model.models.business.Business.find_by_identifier', side_effect=mock_find_by_identifier) @@ -1568,11 +1553,7 @@ def test_amalgamation_legal_type_mismatch(mocker, app, session, jwt, legal_type, amalgamation_type, expected_code): """Assert amalgamation legal type validation for short form.""" account_id = '123456' - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['legalType'] = legal_type filing['filing']['amalgamationApplication']['type'] = amalgamation_type filing['filing']['amalgamationApplication']['amalgamatingBusinesses'] = [ @@ -1587,6 +1568,8 @@ def mock_find_by_identifier(identifier): mocker.patch('legal_api.services.filings.validations.amalgamation_application.validate_name_request', return_value=[]) + mocker.patch('legal_api.services.filings.validations.amalgamation_application.validate_primary_or_holding_match', + return_value=[]) mocker.patch('legal_api.services.filings.validations.amalgamation_application._has_pending_filing', return_value=False) mocker.patch('business_model.models.business.Business.find_by_identifier', side_effect=mock_find_by_identifier) @@ -1614,11 +1597,7 @@ def mock_find_by_identifier(identifier): def test_horizontal_amalgamation(mocker, app, session, jwt, test_name, expected_code, expected_msg): """Assert horizontal amalgamation are valid.""" account_id = '123456' - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['type'] = Amalgamation.AmalgamationTypes.horizontal.name filing['filing']['amalgamationApplication']['amalgamatingBusinesses'][0]['role'] = \ AmalgamatingBusiness.Role.primary.name @@ -1631,6 +1610,8 @@ def mock_find_by_identifier(identifier): mocker.patch('legal_api.services.filings.validations.amalgamation_application.validate_name_request', return_value=[]) + mocker.patch('legal_api.services.filings.validations.amalgamation_application.validate_primary_or_holding_match', + return_value=[]) mocker.patch('legal_api.services.filings.validations.amalgamation_application._has_pending_filing', return_value=False) mocker.patch('business_model.models.business.Business.find_by_identifier', side_effect=mock_find_by_identifier) @@ -1654,12 +1635,13 @@ def mock_find_by_identifier(identifier): (Amalgamation.AmalgamationTypes.regular.name, Business.LegalTypes.BCOMP.value, True, True, True), (Amalgamation.AmalgamationTypes.regular.name, Business.LegalTypes.BCOMP.value, True, False, True), - (Amalgamation.AmalgamationTypes.horizontal.name, Business.LegalTypes.BCOMP.value, False, True, True), + # structural share checks apply to short-form too - inherited bad data must be fixed at the source + (Amalgamation.AmalgamationTypes.horizontal.name, Business.LegalTypes.BCOMP.value, False, True, False), (Amalgamation.AmalgamationTypes.horizontal.name, Business.LegalTypes.BCOMP.value, False, False, True), (Amalgamation.AmalgamationTypes.horizontal.name, Business.LegalTypes.BCOMP.value, True, True, True), (Amalgamation.AmalgamationTypes.horizontal.name, Business.LegalTypes.BCOMP.value, True, False, True), - (Amalgamation.AmalgamationTypes.vertical.name, Business.LegalTypes.BCOMP.value, False, True, True), + (Amalgamation.AmalgamationTypes.vertical.name, Business.LegalTypes.BCOMP.value, False, True, False), (Amalgamation.AmalgamationTypes.vertical.name, Business.LegalTypes.BCOMP.value, False, False, True), (Amalgamation.AmalgamationTypes.vertical.name, Business.LegalTypes.BCOMP.value, True, True, True), (Amalgamation.AmalgamationTypes.vertical.name, Business.LegalTypes.BCOMP.value, True, False, True), @@ -1669,10 +1651,7 @@ def test_amalgamation_share_class_series_validation(mocker, app, session, jwt, a has_rights_or_restrictions, has_series, should_pass): """Test share class/series validation in amalgamation application with different amalgamation types.""" filing = {'filing': {}} - filing['filing']['header'] = {'name': 'amalgamationApplication', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing = _get_amalg_template() filing['filing']['amalgamationApplication']['nameRequest']['legalType'] = legal_type filing['filing']['amalgamationApplication']['type'] = amalgamation_type @@ -1713,11 +1692,17 @@ def test_amalgamation_share_class_series_validation(mocker, app, session, jwt, a ('SUCCESS', '2020-09-18T00:00:00+00:00', None, None), ('SUCCESS', None, None, None), ('FAIL_INVALID_DATE_TIME_FORMAT', '2020-09-44T00:00:00z', - HTTPStatus.UNPROCESSABLE_CONTENT, [{ - 'path': 'filing/header/effectiveDate', - 'error': "'2020-09-44T00:00:00z' is not a 'date-time'", - 'context': [] - }]), + HTTPStatus.UNPROCESSABLE_CONTENT, [ + { + 'path': 'filing/header/effectiveDate', + 'error': "'2020-09-44T00:00:00z' is not a 'date-time'", + 'context': [] + }, + { + 'path': 'filing/header/effectiveDate', + 'error': "'2020-09-44T00:00:00z' is not a 'date-time'", + 'context': [] + }]), ('FAIL_INVALID_DATE_TIME_MINIMUM', '2020-09-17T00:01:00+00:00', HTTPStatus.BAD_REQUEST, [{ 'error': 'Invalid Datetime, effective date must be a minimum of 2 minutes ahead.', @@ -1743,6 +1728,7 @@ def test_validate_amalgamation_effective_date( 'email': 'no_one@never.get', 'filingId': 1 } + filing['filing']['business'] = {'identifier': 'T1234567'} filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) filing['filing']['amalgamationApplication']['nameRequest']['legalType'] = Business.LegalTypes.BCOMP.value filing['filing']['amalgamationApplication']['type'] = amalgamation_type @@ -1795,6 +1781,7 @@ def test_amalgamation_permission_and_completing_party_flag(mocker, app, session, 'email': 'test@email.com', 'filingId': 1 } + filing['filing']['business'] = {'identifier': 'T1234567'} filing['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' @@ -1838,4 +1825,509 @@ def test_amalgamation_permission_and_completing_party_flag(mocker, app, session, assert expected_msg in str(err.msg[0].get('message', err.msg[0].get('error', ''))) else: assert err is None - \ No newline at end of file + + +# ---- COLIN amalgamating businesses (corps not loaded in LEAR) ---- + +COLIN_IDENTIFIER = 'BC0870226' + + +def _mock_colin_snapshot_response(status_code=HTTPStatus.OK, **overrides): + """Return a mocked colin-api snapshot (json, status code), business fields overridable.""" + business = { + 'identifier': COLIN_IDENTIFIER, + 'legalName': 'COLIN TEST COMPANY LTD.', + 'legalType': Business.LegalTypes.COMP.value, + 'state': Business.State.ACTIVE.name, + 'goodStanding': True, + 'adminFreeze': False, + 'foundingDate': '2000-01-01T08:00:00+00:00', + 'taxId': '791861078BC0001', + 'hasFutureEffectiveFiling': False + } + business.update(overrides) + return {'business': business}, status_code + + +def _setup_colin_amalgamation_mocks(mocker, affiliated=True): + """Wire the standard mocks: one LEAR TING plus one identifier not in LEAR.""" + def mock_find_by_identifier(identifier): + if identifier == COLIN_IDENTIFIER: + return None + return Business(identifier=identifier, + founding_date=datetime.now(timezone.utc), + state=Business.State.ACTIVE, + legal_type=Business.LegalTypes.BCOMP.value) + + mocker.patch('legal_api.services.filings.validations.amalgamation_application.validate_name_request', + return_value=[]) + mocker.patch('legal_api.services.filings.validations.amalgamation_application._has_pending_filing', + return_value=False) + mocker.patch('business_model.models.business.Business.find_by_identifier', side_effect=mock_find_by_identifier) + mocker.patch('legal_api.services.filings.validations.amalgamation_application._is_business_affliated', + side_effect=lambda identifier, account_id: affiliated or identifier != COLIN_IDENTIFIER) + + +@pytest.mark.parametrize( + 'test_status, expected_msg', + [ + ('SUCCESS', None), + ('HISTORICAL', f'Cannot amalgamate with {COLIN_IDENTIFIER} which is in historical state.'), + ('FUTURE_EFFECTIVE', f'{COLIN_IDENTIFIER} has a draft, pending or future effective filing.'), + ('FROZEN', f'{COLIN_IDENTIFIER} is frozen.'), + ('NOT_AFFILIATED', f'{COLIN_IDENTIFIER} is not affiliated with the currently ' + 'selected BC Registries account.'), + ('NOT_GOOD_STANDING', f'{COLIN_IDENTIFIER} is not in good standing.'), + ('GOOD_STANDING_UNKNOWN', f'{COLIN_IDENTIFIER} is not in good standing.'), + ('OUT_OF_SCOPE_TYPE', f'A business with identifier:{COLIN_IDENTIFIER} not found.'), + ('COLIN_404', f'A business with identifier:{COLIN_IDENTIFIER} not found.'), + ('COLIN_500', f'Unable to verify {COLIN_IDENTIFIER} - COLIN is unavailable, try again.'), + ('COLIN_DOWN', f'Unable to verify {COLIN_IDENTIFIER} - COLIN is unavailable, try again.'), + ] +) +def test_colin_amalgamating_business(mocker, app, session, jwt, test_status, expected_msg): + """Assert a COLIN BC/ULC/CC corp not loaded in LEAR is validated from its snapshot.""" + account_id = '123456' + filing = _get_amalg_template() + filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' + filing['filing']['amalgamationApplication']['amalgamatingBusinesses'] = [ + {'role': AmalgamatingBusiness.Role.amalgamating.name, 'identifier': 'BC1234567'}, + {'role': AmalgamatingBusiness.Role.amalgamating.name, 'identifier': COLIN_IDENTIFIER} + ] + + _setup_colin_amalgamation_mocks(mocker, affiliated=test_status != 'NOT_AFFILIATED') + + snapshot_responses = { + 'HISTORICAL': _mock_colin_snapshot_response(state=Business.State.HISTORICAL.name), + 'FUTURE_EFFECTIVE': _mock_colin_snapshot_response(hasFutureEffectiveFiling=True), + 'FROZEN': _mock_colin_snapshot_response(adminFreeze=True), + 'NOT_GOOD_STANDING': _mock_colin_snapshot_response(goodStanding=False), + 'GOOD_STANDING_UNKNOWN': _mock_colin_snapshot_response(goodStanding=None), + 'OUT_OF_SCOPE_TYPE': _mock_colin_snapshot_response(legalType='CP'), + 'COLIN_404': ({'message': 'not found'}, HTTPStatus.NOT_FOUND), + 'COLIN_500': ({}, HTTPStatus.INTERNAL_SERVER_ERROR), + 'COLIN_DOWN': (None, None), + } + get_snapshot = mocker.patch( + 'legal_api.services.filings.validations.amalgamation_application.colin.get_snapshot', + return_value=snapshot_responses.get(test_status, _mock_colin_snapshot_response())) + + with jwt_request_context(app, jwt, [BASIC_USER], 'basic-user', account_id): + err = validate(None, filing, account_id) + + get_snapshot.assert_called_once_with(COLIN_IDENTIFIER) + + if expected_msg is None: + assert not err + else: + assert err.code == HTTPStatus.BAD_REQUEST + assert expected_msg == err.msg[0]['error'] + + +def test_colin_amalgamating_business_staff_skips_account_checks(mocker, app, session, jwt): + """Assert staff can amalgamate an unaffiliated, not-in-good-standing COLIN corp.""" + account_id = '123456' + filing = _get_amalg_template() + filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' + filing['filing']['amalgamationApplication']['amalgamatingBusinesses'] = [ + {'role': AmalgamatingBusiness.Role.amalgamating.name, 'identifier': 'BC1234567'}, + {'role': AmalgamatingBusiness.Role.amalgamating.name, 'identifier': COLIN_IDENTIFIER} + ] + + _setup_colin_amalgamation_mocks(mocker, affiliated=False) + mocker.patch('legal_api.services.filings.validations.amalgamation_application.colin.get_snapshot', + return_value=_mock_colin_snapshot_response(goodStanding=False)) + + with jwt_request_context(app, jwt, [STAFF_ROLE], 'staff-user', account_id): + err = validate(None, filing, account_id) + + assert not err + + +def test_colin_amalgamating_business_multiple_tings(mocker, app, session, jwt): + """Assert each COLIN TING is fetched once and validated from its own snapshot.""" + account_id = '123456' + filing = _get_amalg_template() + filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' + filing['filing']['amalgamationApplication']['amalgamatingBusinesses'] = [ + {'role': AmalgamatingBusiness.Role.amalgamating.name, 'identifier': COLIN_IDENTIFIER}, + {'role': AmalgamatingBusiness.Role.amalgamating.name, 'identifier': 'BC0870227'} + ] + + def mock_find_by_identifier(identifier): + return None + + mocker.patch('legal_api.services.filings.validations.amalgamation_application.validate_name_request', + return_value=[]) + mocker.patch('business_model.models.business.Business.find_by_identifier', side_effect=mock_find_by_identifier) + mocker.patch('legal_api.services.filings.validations.amalgamation_application._is_business_affliated', + return_value=True) + get_snapshot = mocker.patch( + 'legal_api.services.filings.validations.amalgamation_application.colin.get_snapshot', + side_effect=[_mock_colin_snapshot_response(), + _mock_colin_snapshot_response(identifier='BC0870227')]) + + with jwt_request_context(app, jwt, [BASIC_USER], 'basic-user', account_id): + err = validate(None, filing, account_id) + + assert not err + assert [call.args[0] for call in get_snapshot.call_args_list] == [COLIN_IDENTIFIER, 'BC0870227'] + + +def test_colin_ccc_ting_satisfies_ccc_rule(mocker, app, session, jwt): + """Assert a COLIN CC TING counts for the resulting-CC rule.""" + account_id = '123456' + filing = _get_amalg_template() + filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' + filing['filing']['amalgamationApplication']['nameRequest']['legalType'] = Business.LegalTypes.BC_CCC.value + filing['filing']['amalgamationApplication']['amalgamatingBusinesses'] = [ + {'role': AmalgamatingBusiness.Role.amalgamating.name, 'identifier': 'BC1234567'}, + {'role': AmalgamatingBusiness.Role.amalgamating.name, 'identifier': COLIN_IDENTIFIER} + ] + + _setup_colin_amalgamation_mocks(mocker) + mocker.patch('legal_api.services.filings.validations.amalgamation_application.colin.get_snapshot', + return_value=_mock_colin_snapshot_response(legalType=Business.LegalTypes.BC_CCC.value)) + + with jwt_request_context(app, jwt, [BASIC_USER], 'basic-user', account_id): + err = validate(None, filing, account_id) + + assert not err + + +def test_colin_ting_name_is_adoptable(mocker, app, session, jwt): + """Assert a regular amalgamation can adopt the name of a COLIN TING of the same type.""" + account_id = '123456' + filing = _get_amalg_template() + filing['filing']['amalgamationApplication']['nameRequest'] = { + 'legalType': Business.LegalTypes.COMP.value, + 'legalName': 'COLIN TEST COMPANY LTD.' + } + filing['filing']['amalgamationApplication']['amalgamatingBusinesses'] = [ + {'role': AmalgamatingBusiness.Role.amalgamating.name, 'identifier': 'BC1234567'}, + {'role': AmalgamatingBusiness.Role.amalgamating.name, 'identifier': COLIN_IDENTIFIER} + ] + + _setup_colin_amalgamation_mocks(mocker) + mocker.patch('legal_api.services.filings.validations.amalgamation_application.colin.get_snapshot', + return_value=_mock_colin_snapshot_response()) + + with jwt_request_context(app, jwt, [BASIC_USER], 'basic-user', account_id): + err = validate(None, filing, account_id) + + assert not err + + +@pytest.mark.parametrize( + 'resulting_legal_type, expect_error', + [ + (Business.LegalTypes.COMP.value, False), + (Business.LegalTypes.BCOMP.value, True) + ] +) +def test_colin_holding_business_legal_type_match(mocker, app, session, jwt, resulting_legal_type, expect_error): + """Assert the short-form legal-type match runs against a COLIN holding business.""" + account_id = '123456' + filing = _get_amalg_template() + filing['filing']['amalgamationApplication']['type'] = Amalgamation.AmalgamationTypes.vertical.name + filing['filing']['amalgamationApplication']['nameRequest'] = {'legalType': resulting_legal_type} + filing['filing']['amalgamationApplication']['amalgamatingBusinesses'] = [ + {'role': AmalgamatingBusiness.Role.holding.name, 'identifier': COLIN_IDENTIFIER}, + {'role': AmalgamatingBusiness.Role.amalgamating.name, 'identifier': 'BC1234567'} + ] + + _setup_colin_amalgamation_mocks(mocker) + mocker.patch('legal_api.services.filings.validations.amalgamation_application.colin.get_snapshot', + return_value=_mock_colin_snapshot_response()) + mocker.patch('legal_api.services.filings.validations.amalgamation_application.validate_primary_or_holding_match', + return_value=[]) + + with jwt_request_context(app, jwt, [BASIC_USER], 'basic-user', account_id): + err = validate(None, filing, account_id) + + type_error = 'Legal type should be same as the legal type in primary or holding business.' + if expect_error: + assert type_error in [x['error'] for x in err.msg] + else: + assert not err + + +REFRESH_HINT = "Refresh the draft with the business's current data." +SOURCE_UPDATE_HINT = ('This data comes from the holding business - its information must be ' + 'corrected outside of this filing before this amalgamation can be filed.') + + +def _factory_short_form_source_business(identifier='BC7654321', share_class_name='Class A Shares'): + """Create a holding/primary business with offices, a director, shares and a resolution.""" + business = factory_business(identifier, entity_type=Business.LegalTypes.COMP.value) + for office_type in ['registeredOffice', 'recordsOffice']: + office = Office(office_type=office_type) + for address_type in [Address.MAILING, Address.DELIVERY]: + office.addresses.append(Address(city='Victoria', street=f'{identifier} {office_type} {address_type} St', + postal_code='V1V 1V1', country='CA', region='BC', + address_type=address_type)) + business.offices.append(office) + + share_class = ShareClass(name=share_class_name, priority=1, max_share_flag=True, max_shares=1000, + par_value_flag=True, par_value=0.85, currency='CAD', special_rights_flag=True, + business_id=business.id) + share_class.series.append(ShareSeries(name='Series A1 Shares', priority=1, max_share_flag=True, + max_shares=500, special_rights_flag=False)) + share_class.save() + + business.resolutions.append(Resolution(resolution_date=date(2020, 5, 13), + resolution_type=Resolution.ResolutionType.SPECIAL.value)) + + party = Party(first_name='DIRECTOR', last_name='ONE', party_type=Party.PartyTypes.PERSON.value) + party.delivery_address = factory_address(f'{identifier} director street', 'delivery') + party.save() + business.party_roles.append(PartyRole(role=PartyRole.RoleTypes.DIRECTOR.value, + appointment_date=datetime(2020, 1, 1), + party_id=party.id)) + business.save() + return business + + +def _without_nones(value): + """Drop None-valued keys - model json emits them where the filing schema wants the key absent.""" + if isinstance(value, dict): + return {k: _without_nones(v) for k, v in value.items() if v is not None} + if isinstance(value, list): + return [_without_nones(v) for v in value] + return value + + +def _short_form_filing_from_lear(business): + """Return a vertical amalgamation filing populated with the holding business's current data.""" + filing = _get_amalg_template() + aml = filing['filing']['amalgamationApplication'] + aml['type'] = Amalgamation.AmalgamationTypes.vertical.name + aml['amalgamatingBusinesses'] = [ + {'role': AmalgamatingBusiness.Role.holding.name, 'identifier': business.identifier}, + {'role': AmalgamatingBusiness.Role.amalgamating.name, 'identifier': 'BC1111111'} + ] + aml['nameRequest'] = {'legalType': business.legal_type, 'legalName': business.legal_name} + aml['offices'] = _without_nones({ + office.office_type: {f'{address.address_type}Address': address.json for address in office.addresses} + for office in business.offices.all() + }) + # no scrub here - the share class schema requires parValue/currency present even when null + aml['shareStructure'] = { + 'shareClasses': [share_class.json for share_class in business.share_classes.all()], + 'resolutionDates': [resolution.resolution_date.isoformat() for resolution in business.resolutions] + } + + completing_party = copy.deepcopy(AMALGAMATION_APPLICATION['parties'][0]) + completing_party['roles'] = [role for role in completing_party['roles'] + if role['roleType'] == 'Completing Party'] + directors = [] + for party_role in PartyRole.get_active_directors(business.id, date.today()): + director = party_role.json + director['roles'] = [{'roleType': 'Director', 'appointmentDate': director.pop('appointmentDate')}] + director.pop('role', None) + directors.append(director) + aml['parties'] = _without_nones([completing_party] + directors) + return filing + + +@pytest.mark.parametrize( + 'test_status, expected_msg', + [ + ('SUCCESS', None), + ('STALE_LEGAL_NAME', f'Legal name does not match the holding business. {REFRESH_HINT}'), + ('STALE_OFFICE', f"Offices do not match the holding business's current offices. {REFRESH_HINT}"), + ('STALE_SHARES', + f"Share structure does not match the holding business's current share structure. {REFRESH_HINT}"), + ('STALE_DIRECTORS', f"Directors do not match the holding business's current directors. {REFRESH_HINT}"), + ('STALE_RESOLUTIONS', + f"Resolution dates do not match the holding business's current resolution dates. {REFRESH_HINT}"), + ] +) +def test_short_form_match_lear(app, session, jwt, test_status, expected_msg): + """Assert a short-form filing must carry the LEAR holding business's current data.""" + account_id = '123456' + holding = _factory_short_form_source_business() + factory_business('BC1111111', entity_type=Business.LegalTypes.COMP.value) + filing = _short_form_filing_from_lear(holding) + aml = filing['filing']['amalgamationApplication'] + + if test_status == 'STALE_LEGAL_NAME': + aml['nameRequest']['legalName'] = 'A NAME THE BUSINESS NO LONGER HAS LTD.' + elif test_status == 'STALE_OFFICE': + aml['offices']['registeredOffice']['deliveryAddress']['streetAddress'] = 'a street they moved away from' + elif test_status == 'STALE_SHARES': + aml['shareStructure']['shareClasses'][0]['parValue'] = 99.99 + elif test_status == 'STALE_DIRECTORS': + aml['parties'] = [party for party in aml['parties'] + if not any(role['roleType'] == 'Director' for role in party['roles'])] + elif test_status == 'STALE_RESOLUTIONS': + aml['shareStructure']['resolutionDates'] = [] + + with jwt_request_context(app, jwt, [STAFF_ROLE], 'staff-user', account_id): + err = validate(None, filing, account_id) + + if test_status == 'SUCCESS': + assert not err, err.msg + else: + assert expected_msg in [x['error'] for x in err.msg] + + +def test_short_form_match_lear_ignores_db_artifacts(app, session, jwt): + """Assert ids, priorities and numeric formatting differences do not fail the match.""" + account_id = '123456' + holding = _factory_short_form_source_business() + factory_business('BC1111111', entity_type=Business.LegalTypes.COMP.value) + filing = _short_form_filing_from_lear(holding) + aml = filing['filing']['amalgamationApplication'] + + # the UI round-trips values the comparator must not be sensitive to + aml['offices']['registeredOffice']['deliveryAddress'].pop('id', None) + aml['shareStructure']['shareClasses'][0].pop('id', None) + aml['shareStructure']['shareClasses'][0]['priority'] = 42 + aml['shareStructure']['shareClasses'][0]['parValue'] = 0.850 + + with jwt_request_context(app, jwt, [STAFF_ROLE], 'staff-user', account_id): + err = validate(None, filing, account_id) + + assert not err + + +def test_short_form_missing_sections(app, session, jwt): + """Assert a short-form filing without the adopted data sections is rejected by the schema.""" + account_id = '123456' + holding = _factory_short_form_source_business() + factory_business('BC1111111', entity_type=Business.LegalTypes.COMP.value) + filing = _short_form_filing_from_lear(holding) + aml = filing['filing']['amalgamationApplication'] + del aml['offices'] + del aml['shareStructure'] + + with jwt_request_context(app, jwt, [STAFF_ROLE], 'staff-user', account_id): + err = validate(None, filing, account_id) + + assert err.code == HTTPStatus.UNPROCESSABLE_ENTITY + + +def test_short_form_missing_legal_name(app, session, jwt): + """Assert the match validation requires the adopted legal name (the schema does not).""" + account_id = '123456' + holding = _factory_short_form_source_business() + factory_business('BC1111111', entity_type=Business.LegalTypes.COMP.value) + filing = _short_form_filing_from_lear(holding) + del filing['filing']['amalgamationApplication']['nameRequest']['legalName'] + + with jwt_request_context(app, jwt, [STAFF_ROLE], 'staff-user', account_id): + err = validate(None, filing, account_id) + + errors = [x['error'] for x in err.msg] + assert 'Legal name of the holding business is required for a short-form amalgamation.' in errors + + +def test_short_form_structural_error_names_the_source(app, session, jwt): + """Assert an inherited structural problem points the user at the source business.""" + account_id = '123456' + # a legacy share class name that predates the ' Shares' suffix rule + holding = _factory_short_form_source_business(share_class_name='Class A Common') + factory_business('BC1111111', entity_type=Business.LegalTypes.COMP.value) + filing = _short_form_filing_from_lear(holding) + + with jwt_request_context(app, jwt, [STAFF_ROLE], 'staff-user', account_id): + err = validate(None, filing, account_id) + + structural_error = next(x['error'] for x in err.msg if "must end with ' Shares'" in x['error']) + # the data matches the holding business, so the fix lives there - not in this filing + assert SOURCE_UPDATE_HINT in structural_error + assert not any('does not match' in x['error'] for x in err.msg) + + +def _full_colin_snapshot(): + """Return a complete colin snapshot - business plus the sections the match validation reads.""" + address = {'id': 1, 'streetAddress': '123 Colin St', 'streetAddressAdditional': None, + 'addressCity': 'Victoria', 'addressRegion': 'BC', 'addressCountry': 'CA', + 'postalCode': 'V1V 1V1', 'deliveryInstructions': None} + return { + **_mock_colin_snapshot_response()[0], + 'parties': [{ + 'officer': {'firstName': 'COLIN', 'middleInitial': '', 'lastName': 'DIRECTOR', + 'organizationName': ''}, + 'deliveryAddress': dict(address), + 'mailingAddress': dict(address), + 'roles': [{'roleType': 'Director'}] + }], + 'offices': { + 'registeredOffice': {'deliveryAddress': dict(address), 'mailingAddress': dict(address)}, + 'recordsOffice': {'deliveryAddress': dict(address), 'mailingAddress': dict(address)} + }, + 'shareClasses': [{ + 'id': 100, 'name': 'Class A Shares', 'priority': 100, 'hasMaximumShares': True, + 'maxNumberOfShares': 10000, 'hasParValue': False, 'parValue': None, 'currency': None, + 'currencyAdditional': None, 'hasRightsOrRestrictions': False, 'series': [] + }], + 'resolutions': [{'date': '2010-10-10'}] + } + + +@pytest.mark.parametrize('test_status', ['SUCCESS', 'STALE_LEGAL_NAME']) +def test_short_form_match_colin(mocker, app, session, jwt, test_status): + """Assert the short-form match runs against a COLIN holding business's snapshot.""" + account_id = '123456' + snapshot = _full_colin_snapshot() + + filing = _get_amalg_template() + aml = filing['filing']['amalgamationApplication'] + aml['type'] = Amalgamation.AmalgamationTypes.vertical.name + aml['amalgamatingBusinesses'] = [ + {'role': AmalgamatingBusiness.Role.holding.name, 'identifier': COLIN_IDENTIFIER}, + {'role': AmalgamatingBusiness.Role.amalgamating.name, 'identifier': 'BC1234567'} + ] + aml['nameRequest'] = {'legalType': Business.LegalTypes.COMP.value, + 'legalName': snapshot['business']['legalName']} + aml['offices'] = _without_nones(snapshot['offices']) + # no scrub here - the share class schema requires parValue/currency present even when null + aml['shareStructure'] = {'shareClasses': copy.deepcopy(snapshot['shareClasses']), + 'resolutionDates': ['2010-10-10']} + completing_party = copy.deepcopy(AMALGAMATION_APPLICATION['parties'][0]) + completing_party['roles'] = [role for role in completing_party['roles'] + if role['roleType'] == 'Completing Party'] + directors = [] + for director in copy.deepcopy(snapshot['parties']): + # the UI normalizes snapshot parties into filing shape - drop empties, stamp type and role + director['officer'] = {k: v for k, v in director['officer'].items() if v not in (None, '')} + director['officer']['partyType'] = 'person' + director['roles'] = [{'roleType': 'Director', 'appointmentDate': '2010-10-10'}] + directors.append(director) + aml['parties'] = _without_nones([completing_party] + directors) + + if test_status == 'STALE_LEGAL_NAME': + aml['nameRequest']['legalName'] = 'A NAME COLIN DOES NOT HAVE LTD.' + + _setup_colin_amalgamation_mocks(mocker) + mocker.patch('legal_api.services.filings.validations.amalgamation_application.colin.get_snapshot', + return_value=(snapshot, HTTPStatus.OK)) + + with jwt_request_context(app, jwt, [BASIC_USER], 'basic-user', account_id): + err = validate(None, filing, account_id) + + if test_status == 'SUCCESS': + assert not err, err.msg + else: + assert f'Legal name does not match the holding business. {REFRESH_HINT}' in [x['error'] for x in err.msg] + + +def test_regular_amalgamation_skips_short_form_match(mocker, app, session): + """Assert the primary/holding match validation never runs for a regular amalgamation.""" + filing = _get_amalg_template() + filing['filing']['amalgamationApplication']['nameRequest']['nrNumber'] = 'NR 1234567' + + mocker.patch('legal_api.services.filings.validations.amalgamation_application.validate_name_request', + return_value=[]) + mocker.patch('legal_api.services.filings.validations.amalgamation_application.validate_amalgamating_businesses', + return_value=[]) + match_mock = mocker.patch( + 'legal_api.services.filings.validations.amalgamation_application.validate_primary_or_holding_match', + return_value=[]) + + err = validate(None, filing) + + assert not err + match_mock.assert_not_called() diff --git a/legal-api/tests/unit/services/filings/validations/test_change_of_address.py b/legal-api/tests/unit/services/filings/validations/test_change_of_address.py index 3b50ec840c..7945ecdb34 100644 --- a/legal-api/tests/unit/services/filings/validations/test_change_of_address.py +++ b/legal-api/tests/unit/services/filings/validations/test_change_of_address.py @@ -17,10 +17,8 @@ import pytest from http import HTTPStatus -from registry_schemas.example_data import CHANGE_OF_ADDRESS, FILING_HEADER +from registry_schemas.example_data import CHANGE_OF_ADDRESS, get_filing_template -from legal_api.errors import Error -from business_model.models import Business from legal_api.services.filings.validations.validation import validate from tests.unit.models import factory_business @@ -29,7 +27,7 @@ def test_valid_address_change(session): """Test that a valid address change passes validation.""" business = factory_business('CP1234567') - filing_json = copy.deepcopy(FILING_HEADER) + filing_json = get_filing_template('changeOfAddress', 'CP1234567') filing_json['filing']['changeOfAddress'] = copy.deepcopy(CHANGE_OF_ADDRESS) error = validate(business, filing_json) assert error is None @@ -43,7 +41,7 @@ def test_validate_max_length_street_addresses(session, test_data): """Test that validation fails when street addresses exceed max length.""" field, value, msg = test_data business = factory_business('CP1234567') - filing_json = copy.deepcopy(FILING_HEADER) + filing_json = get_filing_template('changeOfAddress', 'CP1234567') filing_json['filing']['changeOfAddress'] = copy.deepcopy(CHANGE_OF_ADDRESS) filing_json['filing']['changeOfAddress']['offices']['registeredOffice']['deliveryAddress'][field] = value error = validate(business, filing_json) @@ -58,7 +56,7 @@ def test_exceed_max_length_street_addresses(session, test_data): """Test that validation fails when street addresses exceed max length.""" field, value, msg = test_data business = factory_business('CP1234567') - filing_json = copy.deepcopy(FILING_HEADER) + filing_json = get_filing_template('changeOfAddress', 'CP1234567') filing_json['filing']['changeOfAddress'] = copy.deepcopy(CHANGE_OF_ADDRESS) filing_json['filing']['changeOfAddress']['offices']['registeredOffice']['deliveryAddress'][field] = value error = validate(business, filing_json) @@ -68,7 +66,7 @@ def test_exceed_max_length_street_addresses(session, test_data): def test_invalid_region(session): """Test that validation fails when region is not BC.""" business = factory_business('CP1234567') - filing_json = copy.deepcopy(FILING_HEADER) + filing_json = get_filing_template('changeOfAddress', 'CP1234567') filing_json['filing']['changeOfAddress'] = copy.deepcopy(CHANGE_OF_ADDRESS) filing_json['filing']['changeOfAddress']['offices']['registeredOffice']['deliveryAddress']['addressRegion'] = 'AB' @@ -82,7 +80,7 @@ def test_invalid_region(session): def test_invalid_country(session): """Test that validation fails when country is not CA.""" business = factory_business('CP1234567') - filing_json = copy.deepcopy(FILING_HEADER) + filing_json = get_filing_template('changeOfAddress', 'CP1234567') filing_json['filing']['changeOfAddress'] = copy.deepcopy(CHANGE_OF_ADDRESS) filing_json['filing']['changeOfAddress']['offices']['registeredOffice']['deliveryAddress']['addressCountry'] = 'US' diff --git a/legal-api/tests/unit/services/filings/validations/test_change_of_registration.py b/legal-api/tests/unit/services/filings/validations/test_change_of_registration.py index bd85ef1ffa..483c953021 100644 --- a/legal-api/tests/unit/services/filings/validations/test_change_of_registration.py +++ b/legal-api/tests/unit/services/filings/validations/test_change_of_registration.py @@ -22,11 +22,13 @@ import pytest from registry_schemas.example_data import CHANGE_OF_REGISTRATION_TEMPLATE, REGISTRATION -from business_model.models import Business +from business_model.models import Business, User from legal_api.services import NaicsService, NameXService, flags +from legal_api.services.authz import BASIC_USER, STAFF_ROLE from legal_api.services.filings.validations.change_of_registration import validate from tests.unit.services.filings.validations import create_party, create_party_address +from tests.unit.services.utils import jwt_request_context now = datetime.now().strftime('%Y-%m-%d') @@ -270,7 +272,7 @@ def test_invalid_business_address(session, test_name, filing): @pytest.mark.parametrize( 'test_status, file_number, effect_of_order, expected_code, expected_msg', [ - ('FAIL', None, 'planOfArrangement', HTTPStatus.BAD_REQUEST, 'Court order file number is required.'), + ('FAIL', None, 'planOfArrangement', HTTPStatus.UNPROCESSABLE_ENTITY, "'fileNumber' is a required property"), ('FAIL', '12345678901234567890', 'invalid', HTTPStatus.BAD_REQUEST, 'Invalid effectOfOrder.'), ('SUCCESS', '12345678901234567890', 'planOfArrangement', None, None) ] @@ -288,9 +290,11 @@ def test_change_of_registration_court_orders(session, test_status, file_number, business = Business(identifier=filing['filing']['business']['identifier'], legal_type=filing['filing']['business']['legalType']) + from legal_api.services.filings import validate as _validate + with patch.object(NameXService, 'query_nr_number', return_value=MockResponse(nr_response)): with patch.object(NaicsService, 'find_by_code', return_value=naics_response): - err = validate(business, filing) + err = _validate(business, filing) # validate outcomes if test_status == 'FAIL': @@ -325,4 +329,38 @@ def test_change_of_registration_permission_checks(mock_is_officer_replace, mock_ assert err assert err.code == HTTPStatus.FORBIDDEN assert 'DBA' in err.msg[0]['message'] - + + +@pytest.mark.parametrize( + 'test_name, roles, user_name, expected_error', [ + ('client_name_matches', [BASIC_USER], ('Joe', 'P', 'Swanson'), None), + ('client_name_mismatch', [BASIC_USER], ('Different', None, 'Person'), + 'Completing party name must match the name of the logged in user.'), + ('staff_skipped', [STAFF_ROLE], ('Different', None, 'Person'), None), + ] +) +def test_change_of_registration_completing_party_name(app, session, jwt, test_name, roles, user_name, expected_error): + """Assert the completing party name is validated against the user record for client filings.""" + filing = copy.deepcopy(GP_CHANGE_OF_REGISTRATION) + business = Business(identifier=filing['filing']['business']['identifier'], + legal_type=filing['filing']['business']['legalType']) + + firstname, middlename, lastname = user_name + login_source = 'IDIR' if STAFF_ROLE in roles else 'BCSC' + user = User(username='cp-test-user', firstname=firstname, middlename=middlename, lastname=lastname, + sub='sub', iss='iss', idp_userid='123', login_source=login_source) + user.save() + + with patch.object(NameXService, 'query_nr_number', return_value=MockResponse(nr_response)): + with patch.object(NaicsService, 'find_by_code', return_value=naics_response): + with jwt_request_context(app, jwt, roles, login_source=login_source): + err = validate(business, filing) + + if expected_error: + assert err is not None + assert any(msg['error'] == expected_error for msg in err.msg) + else: + assert err is None or all( + msg['error'] != 'Completing party name must match the name of the logged in user.' + for msg in err.msg + ) diff --git a/legal-api/tests/unit/services/filings/validations/test_common_validations.py b/legal-api/tests/unit/services/filings/validations/test_common_validations.py index 23ff08d3a9..191498794e 100644 --- a/legal-api/tests/unit/services/filings/validations/test_common_validations.py +++ b/legal-api/tests/unit/services/filings/validations/test_common_validations.py @@ -19,7 +19,7 @@ from legal_api.core.filing import Filing as CoreFiling from legal_api.errors import Error -from business_model.models import Business, Party, PartyRole, ShareClass +from business_model.models import Business, Filing, Party, PartyRole, ShareClass from business_model.models.share_series import ShareSeries from legal_api.services import flags from legal_api.services.permissions import PermissionService @@ -44,12 +44,14 @@ RESTORATION, ) -from tests.unit.models import factory_business, factory_party_role +from tests.unit.models import factory_business, factory_party_role, factory_pending_filing from legal_api.services.filings.validations.common_validations import ( EXCLUDED_WORDS_FOR_CLASS, EXCLUDED_WORDS_FOR_SERIES, find_updated_keys_for_firms, + _get_file_data, + _nr_in_pending_filing, is_officer_proprietor_replace_valid, validate_authorization_received, validate_certify_name, @@ -603,6 +605,38 @@ def test_validate_certified_by_corps(session, legal_type, input_value, expected_ assert errors[0]['error'] == 'Certified by field is required.' assert errors[0]['path'] == '/filing/header/certifiedBy' +@pytest.mark.parametrize('test_name, roles, login_source, certified_by, expected_error', [ + ('api_user_missing', [], 'API_GW', '', 'Certified by field is required.'), + ('api_user_whitespace', [], 'API_GW', ' John Doe ', 'Certified by field cannot start or end with whitespace.'), + ('api_user_valid', [], 'API_GW', 'John Doe', None), + ('staff_missing', ['staff'], 'IDIR', '', 'Certified by field is required.'), + ('staff_valid', ['staff'], 'IDIR', 'John Doe', None), + ('public_user_not_required', [], 'BCSC', '', None), +]) +def test_validate_certified_by_corps_completing_party(app, session, test_name, roles, + login_source, certified_by, expected_error): + """Corps IA requires certifiedBy for staff and API users (API users identified by loginSource).""" + filing = copy.deepcopy(FILING_HEADER) + filing['filing']['header']['certifiedBy'] = certified_by + filing['filing']['incorporationApplication'] = INCORPORATION + + with ( + patch('legal_api.services.filings.validations.common_validations.jwt.validate_roles', + side_effect=lambda user, required: bool(set(required) & set(roles))), + app.test_request_context() + ): + from flask.globals import request_ctx + request_ctx.current_user = {'sub': 'test-user', 'loginSource': login_source} + errors = validate_certified_by(filing, 'incorporationApplication', 'BEN') + + if expected_error: + assert errors + assert errors[0]['error'] == expected_error + assert errors[0]['path'] == '/filing/header/certifiedBy' + else: + assert errors == [] + + @pytest.mark.parametrize( 'legal_type', [ @@ -1025,8 +1059,8 @@ def test_validate_party_role_firms(mock_permission_service, mock_business, mock_ else: mock_business.find_by_identifier.return_value = None - mock_response = type('Response', (), {'status_code': HTTPStatus.OK if business_in_colin else HTTPStatus.NOT_FOUND})() - mock_colin.query_business.return_value = mock_response + mock_colin.query_business.return_value = \ + ({'business': {}}, HTTPStatus.OK) if business_in_colin else (None, HTTPStatus.NOT_FOUND) if has_permission: mock_permission_service.check_user_permission.return_value = None @@ -1113,11 +1147,10 @@ def test_is_officer_proprietor_replace_valid(session, test_name, legal_type, exi ('test@example.co.uk', True), ('user@[192.168.1.1]', True), ('no_one@never.get', True), - ('"quoted"@example.com', True), ('user_name@domain.org', True), ('test123@test123.com', True), ('john.o\'smith@gov.bc.ca', True), - # Invalid email formats + # Invalid email formats. ('no_one@never.', False), ('invalid', False), ('@invalid.com', False), @@ -1127,6 +1160,7 @@ def test_is_officer_proprietor_replace_valid(session, test_name, legal_type, exi ('test @example.com', False), ('test@ example.com', False), ('test@@example.com', False), + ('"quoted"@example.com', False), ]) def test_contact_point_email_format_via_schema(session, email, is_valid): """The contactPoint email format is enforced by the schema (the same EMAIL_PATTERN the API used). @@ -1178,7 +1212,7 @@ def test_validate_court_order_with_flag_on(session, has_permission, expected_err result = validate_court_order('/filing/alteration/courtOrder', court_order) if has_permission: - assert result is None + assert result == [] else: assert isinstance(result, list) assert len(result) == 1 @@ -2286,3 +2320,116 @@ def test_validate_foreign_jurisdiction(session, test_name, foreign_jurisdiction, is_region_for_us_required=is_region_for_us_required ) assert errors == expected_errors + +@pytest.mark.parametrize( + 'file_key, expected_class, expected_id', + [ + ('CORP-DS0000101951', 'CORP', 'DS0000101951'), + ('COOP-DS0000101952', 'COOP', 'DS0000101952'), + ('FIRM-DS0000101953', 'FIRM', 'DS0000101953'), + ] +) +def test_get_file_data_from_drs(session, monkeypatch, file_key, expected_class, expected_id): + """Test that DRS document keys retrieve content from DRS.""" + + monkeypatch.setattr( + 'legal_api.services.filings.validations.common_validations.doc_service.get_document', + lambda drs_id, doc_class, doc_binary=True: + type( + 'Response', + (), + { + 'ok': True, + 'content': b'test pdf content' + } + )() + ) + + data, size = _get_file_data(file_key) + + assert data == b'test pdf content' + assert size == len(b'test pdf content') + + +def test_get_file_data_drs_failure(session, monkeypatch): + """Test that DRS retrieval errors raise an exception.""" + + monkeypatch.setattr( + 'legal_api.services.flags.value', + lambda flag, default=None: + ["drs-upload"] + if flag == "enable-new-feature" else default + ) + + monkeypatch.setattr( + 'legal_api.services.filings.validations.common_validations.doc_service.get_document', + lambda *args, **kwargs: + type( + 'Response', + (), + { + 'ok': False, + 'status_code': 404 + } + )() + ) + + with pytest.raises(ValueError, match="DRS get_document failed"): + _get_file_data("CORP-DS0000101951") + + +@pytest.mark.parametrize( + "test_name, filing_type, status, has_nr, expected", + [ + # All valid NR filing types in PENDING state are blocked + ("ia_pending", "incorporationApplication", "PENDING", True, True), + ("reg_pending", "registration", "PENDING", True, True), + ("amalg_pending", "amalgamationApplication", "PENDING", True, True), + ("cont_in_pending", "continuationIn", "PENDING", True, True), + ("alt_pending", "alteration", "PENDING", True, True), + ("con_name_pending", "changeOfName", "PENDING", True, True), + ("cor_reg_pending", "changeOfRegistration", "PENDING", True, True), + ("conv_pending", "conversion", "PENDING", True, True), + # PAID filing blocks + ("ia_paid", "incorporationApplication", "PAID", True, True), + # DRAFT filing does not block + ("ia_draft", "incorporationApplication", "DRAFT", True, False), + # Valid NR filing type but no NR information present — does not block + ("ia_no_nr", "incorporationApplication", "PENDING", False, False), + # Invalid NR filing type (dissolution is not in FILING_TYPES_WITH_NR) — does not block + ("dissolution_pending", "dissolution", "PENDING", True, False), + ] +) +def test_nr_in_pending_filing(session, test_name, filing_type, status, has_nr, expected): + """Assert _nr_in_pending_filing blocks based on filing type, status, and NR presence.""" + import random + nr_number = f"NR {random.randint(1000000, 9999999)}" + + # Some filing types require a sub-type field to satisfy the Filing model + required_subtype_fields = { + "amalgamationApplication": {"type": "regular"}, + "dissolution": {"dissolutionType": "voluntary"}, + } + base_data = dict(required_subtype_fields.get(filing_type, {})) + if has_nr: + base_data["nameRequest"] = {"nrNumber": nr_number} + filing_data = {filing_type: base_data} + filing_json = { + "filing": { + "header": {"name": filing_type}, + **filing_data, + } + } + + if status == "PAID": + f = factory_pending_filing(None, filing_json) + f.payment_completion_date = f.filing_date + f.save() + elif status == "DRAFT": + f = Filing() + f.filing_json = filing_json + f.save() + else: # PENDING + factory_pending_filing(None, filing_json) + + assert _nr_in_pending_filing(nr_number) is expected diff --git a/legal-api/tests/unit/services/filings/validations/test_continuation_in.py b/legal-api/tests/unit/services/filings/validations/test_continuation_in.py index 0fa8782598..fc92c93be0 100644 --- a/legal-api/tests/unit/services/filings/validations/test_continuation_in.py +++ b/legal-api/tests/unit/services/filings/validations/test_continuation_in.py @@ -25,7 +25,7 @@ from business_model.models import Business from legal_api.services import NameXService from legal_api.services.filings.validations.validation import validate -from legal_api.services.filings.validations.continuation_in import validate_business_in_colin, _validate_foreign_jurisdiction +from legal_api.services.filings.validations.continuation_in import validate_continuation_in_foreign_jurisdiction, validate_continuation_in_expro_business_in_colin from registry_schemas.example_data import CONTINUATION_IN from tests.unit.services.filings.validations import create_party, create_party_address, lists_are_equal @@ -56,17 +56,24 @@ def _mock_nr_response(legal_type): }) +def _get_continuation_in_template(): + """Return the Amalgamation Application filing template.""" + filing = {'filing': {}} + filing['filing']['header'] = {'name': 'continuationIn', 'date': '2019-04-08', + 'certifiedBy': 'full name', 'authorizationReceived': True, + 'email': 'no_one@never.get', 'filingId': 1} + filing['filing']['business'] = {'identifier': 'T1234567'} + filing['filing']['continuationIn'] = copy.deepcopy(CONTINUATION_IN) + return filing + + def test_invalid_nr_continuation_in(mocker, app, session, monkeypatch): """Assert that nr is invalid.""" monkeypatch.setattr( 'legal_api.services.flags.value', lambda flag, default=None: "C CBEN CCC CUL" if flag == 'supported-continuation-in-entities' else default ) - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'continuationIn', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['continuationIn'] = copy.deepcopy(CONTINUATION_IN) + filing = _get_continuation_in_template() filing['filing']['continuationIn']['nameRequest']['nrNumber'] = 'NR 1234567' invalid_nr_response = { @@ -78,8 +85,8 @@ def test_invalid_nr_continuation_in(mocker, app, session, monkeypatch): 'consumptionDate': '' }] } - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=None) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_business_in_colin', + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=[]) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_continuation_in_expro_business_in_colin', return_value=[]) with patch.object(NameXService, 'query_nr_number', return_value=MockResponse(invalid_nr_response)): err = validate(None, filing) @@ -110,21 +117,17 @@ def test_continuation_in_parties_missing_role(mocker, app, session, legal_type, Business.LegalTypes.ULC_CONTINUE_IN.value: 1, Business.LegalTypes.CCC_CONTINUE_IN.value: 3 } - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'continuationIn', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['continuationIn'] = copy.deepcopy(CONTINUATION_IN) + filing = _get_continuation_in_template() filing['filing']['continuationIn']['isApproved'] = True filing['filing']['continuationIn']['nameRequest']['legalType'] = legal_type filing['filing']['continuationIn']['nameRequest']['nrNumber'] = 'NR 1234567' filing['filing']['continuationIn']['parties'] = [] - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=None) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=[]) mocker.patch('legal_api.services.filings.validations.continuation_in.validate_name_request', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_business_in_colin', + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_continuation_in_expro_business_in_colin', return_value=[]) err = validate(None, filing) @@ -150,16 +153,7 @@ def test_continuation_in_parties_missing_role(mocker, app, session, legal_type, ) def test_continuation_in_parties_invalid_role(mocker, app, session, parties, expected_msg): """Assert that continuation in party roles can be validated for invalid roles.""" - filing = {'filing': {}} - filing['filing']['header'] = { - 'name': 'continuationIn', - 'date': '2019-04-08', - 'certifiedBy': 'full name', - 'authorizationReceived': True, - 'email': 'no_one@never.get', - 'filingId': 1 - } - filing['filing']['continuationIn'] = copy.deepcopy(CONTINUATION_IN) + filing = _get_continuation_in_template() filing['filing']['continuationIn']['isApproved'] = True filing['filing']['continuationIn']['nameRequest']['legalType'] = Business.LegalTypes.CONTINUE_IN.value filing['filing']['continuationIn']['nameRequest']['nrNumber'] = 'NR 1234567' @@ -174,9 +168,9 @@ def test_continuation_in_parties_invalid_role(mocker, app, session, parties, exp p = create_party(party['roles'], index + 1, mailing_addr, delivery_addr) filing['filing']['continuationIn']['parties'].append(p) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=None) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=[]) mocker.patch('legal_api.services.filings.validations.continuation_in.validate_name_request', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_business_in_colin', return_value=[]) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_continuation_in_expro_business_in_colin', return_value=[]) err = validate(None, filing) @@ -359,11 +353,7 @@ def test_validate_continuation_in_office(session, mocker, test_name, legal_type, lambda flag, default=None: "C CBEN CCC CUL" if flag == 'supported-continuation-in-entities' else default ) - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'continuationIn', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['continuationIn'] = copy.deepcopy(CONTINUATION_IN) + filing = _get_continuation_in_template() filing['filing']['continuationIn']['isApproved'] = True filing['filing']['continuationIn']['nameRequest'] = {} @@ -385,10 +375,10 @@ def test_validate_continuation_in_office(session, mocker, test_name, legal_type, recoffice['mailingAddress']['addressCountry'] = mailing_country mocker.patch('legal_api.services.filings.validations.continuation_in.validate_roles', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=None) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=[]) mocker.patch('legal_api.services.filings.validations.continuation_in.validate_name_request', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_business_in_colin', + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_continuation_in_expro_business_in_colin', return_value=[]) err = validate(None, filing) @@ -652,11 +642,7 @@ def test_validate_continuation_in_share_classes(session, mocker, test_name, lega 'legal_api.services.flags.value', lambda flag, default=None: "C CBEN CCC CUL" if flag == 'supported-continuation-in-entities' else default ) - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'continuationIn', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['continuationIn'] = copy.deepcopy(CONTINUATION_IN) + filing = _get_continuation_in_template() filing['filing']['continuationIn']['isApproved'] = True filing['filing']['continuationIn']['nameRequest'] = {} @@ -695,10 +681,10 @@ def test_validate_continuation_in_share_classes(session, mocker, test_name, lega share_structure['shareClasses'][0]['series'][1]['name'] = series_name_2 mocker.patch('legal_api.services.filings.validations.continuation_in.validate_roles', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=None) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=[]) mocker.patch('legal_api.services.filings.validations.continuation_in.validate_name_request', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_business_in_colin', + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_continuation_in_expro_business_in_colin', return_value=[]) # perform test @@ -726,11 +712,7 @@ def test_continuation_in_court_orders(mocker, app, session, 'legal_api.services.flags.value', lambda flag, default=None: "C CBEN CCC CUL" if flag == 'supported-continuation-in-entities' else default ) - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'continuationIn', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['continuationIn'] = copy.deepcopy(CONTINUATION_IN) + filing = _get_continuation_in_template() filing['filing']['continuationIn']['isApproved'] = True filing['filing']['continuationIn']['nameRequest']['nrNumber'] = 'NR 1234567' @@ -741,10 +723,10 @@ def test_continuation_in_court_orders(mocker, app, session, filing['filing']['continuationIn']['courtOrder'] = court_order mocker.patch('legal_api.services.filings.validations.continuation_in.validate_roles', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=None) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=[]) mocker.patch('legal_api.services.filings.validations.continuation_in.validate_name_request', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_business_in_colin', + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_continuation_in_expro_business_in_colin', return_value=[]) err = validate(None, filing) @@ -772,21 +754,17 @@ def test_continuation_in_foreign_jurisdiction(mocker, app, session, legal_type, 'legal_api.services.flags.value', lambda flag, default=None: "C CBEN CCC CUL" if flag == 'supported-continuation-in-entities' else default ) - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'continuationIn', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['continuationIn'] = copy.deepcopy(CONTINUATION_IN) + filing = _get_continuation_in_template() filing['filing']['continuationIn']['nameRequest']['nrNumber'] = 'NR 1234567' filing['filing']['continuationIn']['nameRequest']['legalType'] = legal_type del filing['filing']['continuationIn']['foreignJurisdiction']['affidavitFileKey'] mocker.patch('legal_api.services.filings.validations.continuation_in.validate_roles', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=None) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=[]) mocker.patch('legal_api.services.filings.validations.continuation_in.validate_name_request', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_business_in_colin', + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_continuation_in_expro_business_in_colin', return_value=[]) err = validate(None, filing) @@ -798,38 +776,30 @@ def test_continuation_in_foreign_jurisdiction(mocker, app, session, legal_type, assert not err -def test_validate_business_in_colin(mocker, app, session, monkeypatch): +def test_validate_continuation_in_expro_business_in_colin(mocker, app, session, monkeypatch): """Assert valid continuation EXPRO business""" monkeypatch.setattr( 'legal_api.services.flags.value', lambda flag, default=None: "C CBEN CCC CUL" if flag == 'supported-continuation-in-entities' else default ) - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'continuationIn', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['continuationIn'] = copy.deepcopy(CONTINUATION_IN) + filing = _get_continuation_in_template() filing['filing']['continuationIn']['nameRequest']['legalType'] = 'C' filing['filing']['continuationIn']['nameRequest']['nrNumber'] = 'NR 1234567' - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=None) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=[]) mocker.patch('legal_api.services.filings.validations.continuation_in.validate_name_request', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_business_in_colin', + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_continuation_in_expro_business_in_colin', return_value=(404, {})) err = validate(None, filing) assert err.code == HTTPStatus.BAD_REQUEST -def test_validate_business_in_colin_founding_date_mismatch(mocker, app, session): +def test_validate_continuation_in_expro_business_in_colin_founding_date_mismatch(mocker, app, session): """Assert continuation EXPRO business with founding date mismatch.""" - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'continuationIn', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['continuationIn'] = copy.deepcopy(CONTINUATION_IN) + filing = _get_continuation_in_template() # Add the EXPRO business data to simulate a mismatch in founding date filing['filing']['continuationIn']['business'] = { @@ -838,30 +808,28 @@ def test_validate_business_in_colin_founding_date_mismatch(mocker, app, session) 'foundingDate': '2009-07-23T07:00:00.000+00:00' } - mocker.patch('legal_api.services.filings.validations.continuation_in.colin.query_business', return_value=mocker.Mock( - status_code=HTTPStatus.OK, - json=lambda: { + mocker.patch('legal_api.services.filings.validations.continuation_in.colin.query_business', return_value=( + { 'business': { 'identifier': 'A0077779', 'legalName': 'Test Company Inc.', # Different founding date to trigger validation error 'foundingDate': '2010-01-01T18:21:13-00:00' } - } + }, + HTTPStatus.OK )) - err = validate_business_in_colin(filing, 'continuationIn') + err = validate_continuation_in_expro_business_in_colin( + filing["filing"]["continuationIn"].get("business"), + f"/filing/continuationIn/business") assert err[0]['error'] == 'Founding date does not match with founding date from Colin.' assert err[0]['path'] == '/filing/continuationIn/business/foundingDate' -def test_validate_business_in_colin_founding_date_match(mocker, app, session): +def test_validate_continuation_in_expro_business_in_colin_founding_date_match(mocker, app, session): """Assert continuation EXPRO business with matching founding date.""" - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'continuationIn', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['continuationIn'] = copy.deepcopy(CONTINUATION_IN) + filing = _get_continuation_in_template() # Add the EXPRO business data with a matching founding date filing['filing']['continuationIn']['business'] = { @@ -870,18 +838,20 @@ def test_validate_business_in_colin_founding_date_match(mocker, app, session): 'foundingDate': '2009-07-23T18:31:24-00:00' } - mocker.patch('legal_api.services.filings.validations.continuation_in.colin.query_business', return_value=mocker.Mock( - status_code=HTTPStatus.OK, - json=lambda: { + mocker.patch('legal_api.services.filings.validations.continuation_in.colin.query_business', return_value=( + { 'business': { 'identifier': 'A0077779', 'legalName': 'Test Company Inc.', 'foundingDate': '2009-07-23T18:31:24-00:00' } - } + }, + HTTPStatus.OK )) - err = validate_business_in_colin(filing, 'continuationIn') + err = validate_continuation_in_expro_business_in_colin( + filing["filing"]["continuationIn"].get("business"), + f"/filing/continuationIn/business") assert len(err) == 0 @@ -912,10 +882,11 @@ def test_validate_foreign_jurisdiction_incorporation_date(mocker, app, session): } mocker.patch('legal_api.services.filings.validations.continuation_in.validate_foreign_jurisdiction', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=None) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=[]) # Run the validation function - err = _validate_foreign_jurisdiction(filing, 'continuationIn', 'CCC') + foreign_jurisdiction = filing['filing']['continuationIn']['foreignJurisdiction'] + err = validate_continuation_in_foreign_jurisdiction('CCC', foreign_jurisdiction, '/filing/continuationIn/foreignJurisdiction') # Assert that the error list contains the appropriate error for future incorporation date assert len(err) == 1 @@ -936,20 +907,16 @@ def test_validate_before_and_after_approval(mocker, app, session, test_status, i 'legal_api.services.flags.value', lambda flag, default=None: "C CBEN CCC CUL" if flag == 'supported-continuation-in-entities' else default ) - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'continuationIn', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['continuationIn'] = copy.deepcopy(CONTINUATION_IN) + filing = _get_continuation_in_template() filing['filing']['continuationIn']['isApproved'] = is_approved del filing['filing']['continuationIn']['offices'] del filing['filing']['continuationIn']['parties'] del filing['filing']['continuationIn']['shareStructure'] - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=None) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=[]) mocker.patch('legal_api.services.filings.validations.continuation_in.validate_name_request', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_business_in_colin', + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_continuation_in_expro_business_in_colin', return_value=[]) err = validate(None, filing) @@ -991,11 +958,7 @@ def test_continuation_in_share_class_series_validation(mocker, app, session, leg 'legal_api.services.flags.value', lambda flag, default=None: "C CBEN CCC CUL" if flag == 'supported-continuation-in-entities' else default ) - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'continuationIn', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['continuationIn'] = copy.deepcopy(CONTINUATION_IN) + filing = _get_continuation_in_template() filing['filing']['continuationIn']['isApproved'] = True filing['filing']['continuationIn']['nameRequest'] = {} @@ -1010,10 +973,10 @@ def test_continuation_in_share_class_series_validation(mocker, app, session, leg share_class.pop('series', None) mocker.patch('legal_api.services.filings.validations.continuation_in.validate_roles', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=None) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=[]) mocker.patch('legal_api.services.filings.validations.continuation_in.validate_name_request', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_business_in_colin', + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_continuation_in_expro_business_in_colin', return_value=[]) err = validate(None, filing) @@ -1040,11 +1003,7 @@ def test_continuation_in_parties_delivery_address_validation(mocker, app, sessio 'legal_api.services.flags.value', lambda flag, default=None: "C CBEN CCC CUL" if flag == 'supported-continuation-in-entities' else default ) - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'continuationIn', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} - filing['filing']['continuationIn'] = copy.deepcopy(CONTINUATION_IN) + filing = _get_continuation_in_template() filing['filing']['continuationIn']['isApproved'] = True filing['filing']['continuationIn']['nameRequest'] = {} @@ -1057,9 +1016,9 @@ def test_continuation_in_parties_delivery_address_validation(mocker, app, sessio del filing['filing']['continuationIn']['parties'][0]['deliveryAddress'] mocker.patch('legal_api.services.filings.validations.continuation_in.validate_roles', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=None) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=[]) mocker.patch('legal_api.services.filings.validations.continuation_in.validate_name_request', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_business_in_colin', return_value=[]) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_continuation_in_expro_business_in_colin', return_value=[]) err = validate(None, filing) @@ -1078,11 +1037,18 @@ def test_continuation_in_parties_delivery_address_validation(mocker, app, sessio ('SUCCESS', '2020-09-18T00:00:00+00:00', None, None), ('SUCCESS', None, None, None), ('FAIL_INVALID_DATE_TIME_FORMAT', '2020-09-44T00:00:00z', - HTTPStatus.UNPROCESSABLE_CONTENT, [{ - 'path': 'filing/header/effectiveDate', - 'error': "'2020-09-44T00:00:00z' is not a 'date-time'", - 'context': [] - }]), + HTTPStatus.UNPROCESSABLE_CONTENT, [ + { + 'path': 'filing/header/effectiveDate', + 'error': "'2020-09-44T00:00:00z' is not a 'date-time'", + 'context': [] + }, + { + 'path': 'filing/header/effectiveDate', + 'error': "'2020-09-44T00:00:00z' is not a 'date-time'", + 'context': [] + } + ]), ('FAIL_INVALID_DATE_TIME_MINIMUM', '2020-09-17T00:01:00+00:00', HTTPStatus.BAD_REQUEST, [{ 'error': 'Invalid Datetime, effective date must be a minimum of 2 minutes ahead.', @@ -1101,15 +1067,11 @@ def test_validate_continuation_in_effective_date(mocker, app, session, jwt, test 'legal_api.services.flags.value', lambda flag, default=None: "C CBEN CCC CUL" if flag == 'supported-continuation-in-entities' else default ) - filing = {'filing': {}} - filing['filing']['header'] = {'name': 'continuationIn', 'date': '2019-04-08', - 'certifiedBy': 'full name', 'authorizationReceived': True, - 'email': 'no_one@never.get', 'filingId': 1} + filing = _get_continuation_in_template() if effective_date is not None: filing['filing']['header']['effectiveDate'] = effective_date - filing['filing']['continuationIn'] = copy.deepcopy(CONTINUATION_IN) filing['filing']['continuationIn']['isApproved'] = True filing['filing']['continuationIn']['nameRequest'] = {} @@ -1117,9 +1079,9 @@ def test_validate_continuation_in_effective_date(mocker, app, session, jwt, test filing['filing']['continuationIn']['nameRequest']['legalType'] = 'CBEN' mocker.patch('legal_api.services.filings.validations.continuation_in.validate_roles', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=None) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=[]) mocker.patch('legal_api.services.filings.validations.continuation_in.validate_name_request', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_business_in_colin', return_value=[]) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_continuation_in_expro_business_in_colin', return_value=[]) # perform test with freeze_time(now): @@ -1219,17 +1181,18 @@ def test_validate_foreign_jurisdiction_field_lengths(mocker, app, session, } } + foreign_jurisdiction = filing['filing']['continuationIn']['foreignJurisdiction'] if identifier is not None: - filing['filing']['continuationIn']['foreignJurisdiction']['identifier'] = identifier + foreign_jurisdiction['identifier'] = identifier if legal_name is not None: - filing['filing']['continuationIn']['foreignJurisdiction']['legalName'] = legal_name + foreign_jurisdiction['legalName'] = legal_name mocker.patch('legal_api.services.filings.validations.continuation_in.validate_foreign_jurisdiction', return_value=[]) - mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=None) + mocker.patch('legal_api.services.filings.validations.continuation_in.validate_pdf', return_value=[]) - err = _validate_foreign_jurisdiction(filing, 'continuationIn', 'C') + err = validate_continuation_in_foreign_jurisdiction('C', foreign_jurisdiction, '/filing/continuationIn/foreignJurisdiction') assert err == expected_errors diff --git a/legal-api/tests/unit/services/filings/validations/test_continuation_out.py b/legal-api/tests/unit/services/filings/validations/test_continuation_out.py index 138cb12cb4..def920bb1b 100644 --- a/legal-api/tests/unit/services/filings/validations/test_continuation_out.py +++ b/legal-api/tests/unit/services/filings/validations/test_continuation_out.py @@ -29,7 +29,6 @@ date_format = '%Y-%m-%d' legal_name = 'Test name request' -validate_active_cco_path = 'legal_api.services.filings.validations.continuation_out.validate_active_cco' def _create_consent_continuation_out(business, foreign_jurisdiction, effective_date=datetime.utcnow()): @@ -56,7 +55,7 @@ def _create_consent_continuation_out(business, foreign_jurisdiction, effective_d 'test_name, expected_code, message', [ ('FAIL_IN_FUTURE', HTTPStatus.BAD_REQUEST, 'Continuation out date must be today or past.'), - ('FAIL_NO_CCO', HTTPStatus.BAD_REQUEST, 'No active consent continuation out for this date and/or jurisdiction.'), + ('SUCCESS_NO_CCO', None, None), ('SUCCESS', None, None) ] ) @@ -77,7 +76,7 @@ def test_validate_continuation_out_date(session, test_name, expected_code, messa if test_name == 'FAIL_IN_FUTURE': filing['filing']['continuationOut']['continuationOutDate'] = \ (LegislationDatetime.now() + datedelta.datedelta(days=1)).strftime(date_format) - elif test_name == 'FAIL_NO_CCO': + elif test_name == 'SUCCESS_NO_CCO': effective_date -= datedelta.datedelta(months=6, days=1) _create_consent_continuation_out(business, @@ -86,7 +85,7 @@ def test_validate_continuation_out_date(session, test_name, expected_code, messa err = validate(business, filing) # validate outcomes - if test_name != 'SUCCESS': + if test_name == 'FAIL_IN_FUTURE': assert expected_code == err.code assert message == err.msg[0]['error'] else: @@ -127,7 +126,6 @@ def test_validate_foreign_jurisdiction(session, mocker, test_name, expected_code filing['filing']['continuationOut']['foreignJurisdiction']['country'] = 'US' filing['filing']['continuationOut']['foreignJurisdiction']['region'] = 'NONE' - mocker.patch(validate_active_cco_path, return_value=[]) err = validate(business, filing) # validate outcomes @@ -148,7 +146,6 @@ def test_valid_foreign_jurisdiction(session, mocker, monkeypatch): business = factory_business(identifier='BC1234567', entity_type='BC', founding_date=datetime.utcnow()) filing = copy.deepcopy(FILING_HEADER) filing['filing']['header']['name'] = 'continuationOut' - mocker.patch(validate_active_cco_path, return_value=[]) for country in pycountry.countries: filing['filing']['continuationOut'] = copy.deepcopy(CONTINUATION_OUT) @@ -192,7 +189,6 @@ def test_continuation_out_court_order(session, mocker, test_status, file_number, else: del filing['filing']['continuationOut']['courtOrder']['fileNumber'] - mocker.patch(validate_active_cco_path, return_value=[]) err = validate(business, filing) # validate outcomes diff --git a/legal-api/tests/unit/services/filings/validations/test_correction.py b/legal-api/tests/unit/services/filings/validations/test_correction.py index bc5b29523b..f671709f03 100644 --- a/legal-api/tests/unit/services/filings/validations/test_correction.py +++ b/legal-api/tests/unit/services/filings/validations/test_correction.py @@ -59,7 +59,7 @@ def test_valid_correction(session, app, jwt, test_name, legal_type, identifier, f['filing']['correction']['correctedFilingId'] = corrected_filing.id f['filing']['correction']['type'] = "CLIENT" if test_name == 'COD': - CORRECTION_COD['filing']['correction']['relationships'].append({ + f['filing']['correction']['relationships'].append({ 'entity': { 'givenName': 'Phillip Tandy', 'familyName': 'Miller', @@ -152,3 +152,92 @@ def test_correction__corrected_filing_is_not_complete(session, app, jwt): # check that validation failed as expected assert HTTPStatus.BAD_REQUEST == err.code assert 'Corrected filing is not a valid filing.' == err.msg[0]['error'] + + +def test_correction__invalid_director_dates(session, app, jwt): + """Check that a correction fails if the director appointment date is after cessation date.""" + # setup + identifier = 'BC1234567' + business = factory_business(identifier, entity_type='BC') + factory_business_mailing_address(business) + filing_template = copy.deepcopy(FILING_TEMPLATE) + filing_template['filing']['header']['name'] = 'changeOfDirectors' + filing_template['filing']['changeOfDirectors'] = CHANGE_OF_DIRECTORS + corrected_filing = factory_completed_filing(business, filing_template) + + f = copy.deepcopy(CORRECTION_COD) + f['filing']['header']['identifier'] = identifier + f['filing']['correction']['correctedFilingId'] = corrected_filing.id + f['filing']['correction']['type'] = "CLIENT" + + # Set appointment date after cessation date + f['filing']['correction']['relationships'][0]['roles'][0]['appointmentDate'] = '2025-02-01' + f['filing']['correction']['relationships'][0]['roles'][0]['cessationDate'] = '2025-01-01' + + with jwt_request_context(app, jwt, [STAFF_ROLE]): + err = validate(business, f) + + # check that validation failed as expected + assert err is not None + assert err.code == HTTPStatus.BAD_REQUEST + assert err.msg[0]['error'] == 'Appointment date cannot be after cessation date.' + + +@pytest.mark.parametrize("date_label", ["Appointment", "Cessation"]) +@pytest.mark.parametrize( + "test_name, offset_days, earliest_allowed_offset, expected_error", + [ + ("SUCCESS - Valid date", -5, -10, None), + ("SUCCESS - Date is today", 0, -10, None), + ("SUCCESS - Date is exactly earliest_allowed", -10, -10, None), + ("FAIL - Future date", 1, -10, "{date_label} date cannot be in the future."), + ("FAIL - Date before earliest allowed", -15, -10, "{date_label} date cannot be before the business founding date."), + ("SUCCESS - None date", None, -10, None) + ] +) +def test_validate_relationship_date(session, app, jwt, date_label, test_name, offset_days, earliest_allowed_offset, expected_error): + """Test validation of relationship dates.""" + from business_common.utils import LegislationDatetime + from datetime import timedelta + + today = LegislationDatetime.datenow() + # If offset_days is None, we pass None. Otherwise compute date + date_value = today + timedelta(days=offset_days) if offset_days is not None else None + earliest_allowed_date = today + timedelta(days=earliest_allowed_offset) + + # setup + identifier = 'BC1234567' + business = factory_business(identifier, entity_type='BC', founding_date=earliest_allowed_date) + factory_business_mailing_address(business) + filing_template = copy.deepcopy(FILING_TEMPLATE) + filing_template['filing']['header']['name'] = 'changeOfDirectors' + filing_template['filing']['changeOfDirectors'] = CHANGE_OF_DIRECTORS + corrected_filing = factory_completed_filing(business, filing_template) + + f = copy.deepcopy(CORRECTION_COD) + f['filing']['header']['identifier'] = identifier + f['filing']['correction']['correctedFilingId'] = corrected_filing.id + f['filing']['correction']['type'] = "CLIENT" + + + # Set appointment date after cessation date + if date_value: + if date_label == "Appointment": + f['filing']['correction']['relationships'][0]['roles'][0]['appointmentDate'] = date_value.isoformat() + else: + f['filing']['correction']['relationships'][0]['roles'][0]['appointmentDate'] = earliest_allowed_date.isoformat() + f['filing']['correction']['relationships'][0]['roles'][0]['cessationDate'] = date_value.isoformat() + + with jwt_request_context(app, jwt, [STAFF_ROLE]): + err = validate(business, f) + + # check that validation failed as expected + if expected_error: + assert HTTPStatus.BAD_REQUEST == err.code + if date_label == "Cessation" and earliest_allowed_date > date_value: + assert len(err.msg) == 2 + else: + assert len(err.msg) == 1 + assert err.msg[0]["error"] == expected_error.format(date_label=date_label) + else: + assert not err diff --git a/legal-api/tests/unit/services/filings/validations/test_correction_firms.py b/legal-api/tests/unit/services/filings/validations/test_correction_firms.py index 4c7a758bc9..39bc8b12dc 100644 --- a/legal-api/tests/unit/services/filings/validations/test_correction_firms.py +++ b/legal-api/tests/unit/services/filings/validations/test_correction_firms.py @@ -90,6 +90,7 @@ def test_valid_firms_correction(app, session, jwt, test_name, filing): f['filing']['header']['identifier'] = identifier f['filing']['correction']['correctedFilingId'] = corrected_filing.id + f['filing']['correction']['correctedFilingType'] = 'changeOfRegistration' nr_res = copy.deepcopy(nr_response) nr_res['legalType'] = legal_type @@ -121,6 +122,7 @@ def test_firms_correction_invalid_parties(app, session, jwt, test_name, filing, f['filing']['header']['identifier'] = identifier f['filing']['correction']['correctedFilingId'] = corrected_filing.id + f['filing']['correction']['correctedFilingType'] = 'changeOfRegistration' del f['filing']['correction']['parties'][0]['roles'][0] nr_res = copy.deepcopy(nr_response) @@ -189,6 +191,7 @@ def test_firms_correction_naics(app, session, jwt, test_name, filing, existing_n f['filing']['header']['identifier'] = identifier f['filing']['correction']['correctedFilingId'] = corrected_filing.id + f['filing']['correction']['correctedFilingType'] = 'changeOfRegistration' if correction_naics_code: f['filing']['correction']['business']['naics']['naicsCode'] = correction_naics_code else: @@ -252,6 +255,7 @@ def test_firms_correction_start_date(app, session, jwt, test_name, filing, usern f['filing']['header']['identifier'] = identifier f['filing']['correction']['correctedFilingId'] = corrected_filing.id + f['filing']['correction']['correctedFilingType'] = 'changeOfRegistration' f['filing']['correction']['startDate'] = start_date.strftime('%Y-%m-%d') nr_res = copy.deepcopy(nr_response) diff --git a/legal-api/tests/unit/services/filings/validations/test_correction_ia.py b/legal-api/tests/unit/services/filings/validations/test_correction_ia.py index 6cf2ff2807..84bf94279d 100644 --- a/legal-api/tests/unit/services/filings/validations/test_correction_ia.py +++ b/legal-api/tests/unit/services/filings/validations/test_correction_ia.py @@ -15,6 +15,7 @@ import copy import datedelta +import pycountry from datetime import datetime, timezone from freezegun import freeze_time from http import HTTPStatus @@ -22,10 +23,22 @@ import pytest +from business_model.models import AmalgamatingBusiness, Amalgamation, Business, CourtOrder, Resolution +from business_common.utils.legislation_datetime import LegislationDatetime +from business_common.utils.datetime import datetime as dt, timedelta from legal_api.services import NameXService from legal_api.services.authz import BASIC_USER, STAFF_ROLE from legal_api.services.filings import validate -from registry_schemas.example_data import CORRECTION_INCORPORATION, INCORPORATION_FILING_TEMPLATE +from registry_schemas.example_data import ( + AMALGAMATION_APPLICATION, + AMALGAMATION_OUT, + CORRECTION_INCORPORATION, + CONTINUATION_IN_FILING_TEMPLATE, + CONTINUATION_OUT, + COURT_ORDER_FILING_TEMPLATE, + FILING_HEADER, + INCORPORATION_FILING_TEMPLATE +) from tests.unit import MockResponse from tests.unit.models import factory_business, factory_completed_filing @@ -33,6 +46,7 @@ from tests.unit.services.utils import jwt_request_context +date_format = '%Y-%m-%d' INCORPORATION_APPLICATION = copy.deepcopy(INCORPORATION_FILING_TEMPLATE) CORRECTION = copy.deepcopy(CORRECTION_INCORPORATION) @@ -334,6 +348,86 @@ def test_correction_share_class_series_validation(session, app, jwt, legal_type, ]), ] ) +def test_correction_resolution_date_old(session, app, jwt, test_name, has_rights_or_restrictions, + has_series, resolution_dates, expected_code, expected_msg): + """Test share class/series resolution date validation in correction filings.""" + identifier = 'BC1234567' + business = factory_business(identifier, entity_type='BC') + business.founding_date = FOUNDING_DATE + + corrected_filing = factory_completed_filing(business, INCORPORATION_APPLICATION) + + filing = copy.deepcopy(CORRECTION) + filing['filing']['header']['identifier'] = identifier + filing['filing']['correction']['correctedFilingId'] = corrected_filing.id + del filing['filing']['correction']['commentOnly'] + + # Share structure setup + filing['filing']['correction']['shareStructure'] = copy.deepcopy( + INCORPORATION_FILING_TEMPLATE['filing']['incorporationApplication'].get('shareStructure', {}) + ) + share_class = filing['filing']['correction']['shareStructure']['shareClasses'][0] + share_class['hasRightsOrRestrictions'] = has_rights_or_restrictions + + # Series handling + if has_series: + share_class['series'] = share_class.get('series', [{}]) + share_class['series'][0]['hasRightsOrRestrictions'] = True + else: + share_class.pop('series', None) + + filing['filing']['correction']['shareStructure']['resolutionDates'] = resolution_dates + + # Remove the second share class if it exists + share_classes = filing['filing']['correction']['shareStructure']['shareClasses'] + if len(share_classes) > 1: + share_classes.pop(1) + + with freeze_time(NOW): + with jwt_request_context(app, jwt, [BASIC_USER]): + err = validate(business, filing) + + if expected_code: + assert err + assert any(expected_msg[0]['error'] in e['error'] for e in err.msg) + else: + assert err is None + + +@pytest.mark.parametrize( + 'test_name, has_rights_or_restrictions, has_series, resolution_dates, expected_code, expected_msg', + [ + ('SUCCESS_class_has_rights', True, False, [{'date':'2024-01-01'}], None, None), + ('SUCCESS_class_no_rights', False, False, [], None, None), + ('SUCCESS_series_has_rights', True, True, [{'date':'2024-01-01'}], None, None), + ('SUCCESS_series_no_rights', False, False, [], None, None), + ('SUCCESS_existing_resolution', True, False, [{'date':'2024-01-01'}], None, None), + + ('FAILURE_class_missing_date', True, False, [], HTTPStatus.BAD_REQUEST, [ + {'error': 'Resolution date is required when hasRightsOrRestrictions is true.', + 'path': '/filing/correction/shareStructure/resolutionDates'} + ]), + ('FAILURE_series_missing_date', False, True, [], HTTPStatus.BAD_REQUEST, [ + {'error': 'Resolution date is required when hasRightsOrRestrictions is true.', + 'path': '/filing/correction/shareStructure/resolutionDates'} + ]), + + ('FAILURE_future_date', True, False, [{'date': (NOW + datedelta.DAY).date().isoformat()}], HTTPStatus.BAD_REQUEST, [ + {'error': 'Resolution date cannot be in the future.', + 'path': '/filing/correction/shareStructure/resolutionDates'} + ]), + + ('FAILURE_before_founding', True, False, [{'date': (FOUNDING_DATE - datedelta.DAY).date().isoformat()}], HTTPStatus.BAD_REQUEST, [ + {'error': 'Resolution date cannot be before the business founding date.', + 'path': '/filing/correction/shareStructure/resolutionDates'} + ]), + + ('FAILURE_invalid_resolution_id', True, False, [{'id': 9999999, 'date': '2024-01-01'}], HTTPStatus.BAD_REQUEST, [ + {'error': 'Not a valid Resolution Id for this business.', + 'path': '/filing/correction/shareStructure/resolutionDates/0'} + ]), + ] +) def test_correction_resolution_date(session, app, jwt, test_name, has_rights_or_restrictions, has_series, resolution_dates, expected_code, expected_msg): """Test share class/series resolution date validation in correction filings.""" @@ -362,6 +456,14 @@ def test_correction_resolution_date(session, app, jwt, test_name, has_rights_or_ else: share_class.pop('series', None) + if test_name == "SUCCESS_existing_resolution": + res_obj = Resolution() + res_obj.resolution_date=datetime.fromisoformat(resolution_dates[0]["date"]) + res_obj.business_id = business.id + res_obj.resolution_type = Resolution.ResolutionType.SPECIAL.value + res_obj.save() + resolution_dates[0]["id"] = res_obj.id + filing['filing']['correction']['shareStructure']['resolutionDates'] = resolution_dates # Remove the second share class if it exists @@ -378,3 +480,796 @@ def test_correction_resolution_date(session, app, jwt, test_name, has_rights_or_ assert any(expected_msg[0]['error'] in e['error'] for e in err.msg) else: assert err is None + + +@pytest.mark.parametrize( + 'use_nr, new_name, legal_type, new_legal_type, nr_type, mock_directors, should_pass, num_errors', [ + (False, '', 'BEN', 'BEN', '', True, False, 1), + (False, '', 'BEN', 'BC', '', True, True, 0), + (False, '', 'BEN', 'ULC', '', True, False, 1), + (False, '', 'BEN', 'CC', '', True, True, 1), + (False, '', 'BEN', 'CP', '', True, False, 1), + (False, '', 'BEN', 'C', '', True, False, 1), + (False, '', 'BEN', 'CBEN', '', True, False, 1), + (False, '', 'BEN', 'CUL', '', True, False, 1), + (False, '', 'BEN', 'CCC', '', True, False, 1), + + (False, '', 'BC', 'BC', '', True, False, 1), + (False, '', 'BC', 'BEN', '', True, True, 0), + (False, '', 'BC', 'ULC', '', True, True, 0), + (False, '', 'BC', 'CC', '', True, True, 0), + (False, '', 'BC', 'CP', '', True, False, 1), + (False, '', 'BC', 'C', '', True, False, 1), + (False, '', 'BC', 'CBEN', '', True, False, 1), + (False, '', 'BC', 'CUL', '', True, False, 1), + (False, '', 'BC', 'CCC', '', True, False, 1), + + (False, '', 'ULC', 'ULC', '', True, False, 1), + (False, '', 'ULC', 'BC', '', True, True, 0), + (False, '', 'ULC', 'BEN', '', True, True, 0), + (False, '', 'ULC', 'CC', '', True, False, 1), + (False, '', 'ULC', 'CP', '', True, False, 1), + (False, '', 'ULC', 'C', '', True, False, 1), + (False, '', 'ULC', 'CBEN', '', True, False, 1), + (False, '', 'ULC', 'CUL', '', True, False, 1), + (False, '', 'ULC', 'CCC', '', True, False, 1), + + (False, '', 'CC', 'CC', '', True, False, 1), + (False, '', 'CC', 'BEN', '', True, False, 1), + (False, '', 'CC', 'BC', '', True, False, 1), + (False, '', 'CC', 'ULC', '', True, False, 1), + (False, '', 'CC', 'CP', '', True, False, 1), + (False, '', 'CC', 'C', '', True, False, 1), + (False, '', 'CC', 'CBEN', '', True, False, 1), + (False, '', 'CC', 'CUL', '', True, False, 1), + (False, '', 'CC', 'CCC', '', True, False, 1), + + (False, '', 'CBEN', 'CBEN', '', True, False, 1), + (False, '', 'CBEN', 'C', '', True, True, 0), + (False, '', 'CBEN', 'CUL', '', True, False, 1), + (False, '', 'CBEN', 'CCC', '', True, True, 1), + (False, '', 'CBEN', 'BEN', '', True, False, 1), + (False, '', 'CBEN', 'BC', '', True, False, 1), + (False, '', 'CBEN', 'ULC', '', True, False, 1), + (False, '', 'CBEN', 'CC', '', True, False, 1), + (False, '', 'CBEN', 'CP', '', True, False, 1), + + (False, '', 'C', 'C', '', True, False, 1), + (False, '', 'C', 'CBEN', '', True, True, 0), + (False, '', 'C', 'CUL', '', True, True, 0), + (False, '', 'C', 'CCC', '', True, True, 0), + (False, '', 'C', 'BC', '', True, False, 1), + (False, '', 'C', 'BEN', '', True, False, 1), + (False, '', 'C', 'ULC', '', True, False, 1), + (False, '', 'C', 'CC', '', True, False, 1), + (False, '', 'C', 'CP', '', True, False, 1), + + (False, '', 'CUL', 'CUL', '', True, False, 1), + (False, '', 'CUL', 'C', '', True, True, 0), + (False, '', 'CUL', 'CBEN', '', True, True, 0), + (False, '', 'CUL', 'CCC', '', True, False, 1), + (False, '', 'CUL', 'ULC', '', True, False, 1), + (False, '', 'CUL', 'BC', '', True, False, 1), + (False, '', 'CUL', 'BEN', '', True, False, 1), + (False, '', 'CUL', 'CC', '', True, False, 1), + (False, '', 'CUL', 'CP', '', True, False, 1), + + (False, '', 'CCC', 'CCC', '', True, False, 1), + (False, '', 'CCC', 'CBEN', '', True, False, 1), + (False, '', 'CCC', 'C', '', True, False, 1), + (False, '', 'CCC', 'CUL', '', True, False, 1), + (False, '', 'CCC', 'CC', '', True, False, 1), + (False, '', 'CCC', 'BEN', '', True, False, 1), + (False, '', 'CCC', 'BC', '', True, False, 1), + (False, '', 'CCC', 'ULC', '', True, False, 1), + (False, '', 'CCC', 'CP', '', True, False, 1), + + # check minimum directors validation + (False, '', 'BC', 'CC', '', False, False, 1), + (False, '', 'CBEN', 'CCC', '', False, False, 1) +]) +@patch('business_model.models.PartyRole.get_parties_by_role') +def test_new_legal_type(mock_get_parties, session, app, jwt, use_nr, new_name, legal_type, new_legal_type, nr_type, mock_directors, should_pass, num_errors): + """Test that a valid Alteration without NR correction passes validation.""" + # setup + identifier = 'BC1234567' + business = factory_business(identifier, entity_type=legal_type) + + corrected_filing = factory_completed_filing(business, INCORPORATION_APPLICATION) + + filing = copy.deepcopy(CORRECTION) + filing['filing']['header']['identifier'] = identifier + filing['filing']['correction']['correctedFilingId'] = corrected_filing.id + del filing['filing']['correction']['commentOnly'] + + filing['filing']['correction']['newLegalType'] = new_legal_type + + class MockDirector: + def __init__(self, cessation_date=None): + self.cessation_date = cessation_date + + def create_mock_directors(count=3): + """Return a list of mock directors for tests.""" + return [MockDirector() for _ in range(count)] + + # mock directors + if mock_directors: + mock_get_parties.return_value = create_mock_directors(3) + else: + mock_get_parties.return_value = [] + + if use_nr: + filing['filing']['business']['identifier'] = identifier + filing['filing']['business']['legalName'] = 'legal_name-BC1234567' + filing['filing']['business']['legalType'] = legal_type + + filing['filing']['correction']['nameRequest']['nrNumber'] = identifier + filing['filing']['correction']['nameRequest']['legalName'] = new_name + filing['filing']['correction']['nameRequest']['legalType'] = new_legal_type + + nr_json = { + "state": "APPROVED", + "expirationDate": "", + "requestTypeCd": nr_type, + "names": [{ + "name": new_name, + "state": "APPROVED", + "consumptionDate": "" + }], + "legalType": new_legal_type + } + + nr_response = MockResponse(nr_json) + with patch.object(NameXService, 'query_nr_number', return_value=nr_response): + with jwt_request_context(app, jwt, [BASIC_USER]): + err = validate(business, filing) + else: + del filing['filing']['correction']['nameRequest'] + with jwt_request_context(app, jwt, [BASIC_USER]): + err = validate(business, filing) + + if err: + print(err.msg) + + if should_pass: + # check that validation passed + assert None is err + else: + # check that validation failed + assert err + assert HTTPStatus.BAD_REQUEST == err.code + assert len(err.msg) == num_errors + + # check for minimum directors error message + if (new_legal_type in [Business.LegalTypes.BC_CCC.value, Business.LegalTypes.CCC_CONTINUE_IN.value, + Business.LegalTypes.COOP.value] and not mock_directors): + assert 'Must have a minimum of three directors. File a change of director filing first.' in [e['error'] for e in err.msg] + + +def test_validate_correction_continuation_in_incorporation_date(mocker, app, session, jwt): + """Assert that an error is raised if the correction continuation_in incorporation date is set to a future date.""" + identifier = 'BC1234567' + business = factory_business(identifier, entity_type='C') + corrected_filing = factory_completed_filing(business, CONTINUATION_IN_FILING_TEMPLATE) + + filing = copy.deepcopy(CORRECTION) + filing['filing']['header']['identifier'] = identifier + filing['filing']['correction']['correctedFilingId'] = corrected_filing.id + + filing['filing']['correction']['correctedFilingType'] = 'continuationIn' + future_date = (dt.now() + timedelta(days=1)).date().isoformat() # Set date to tomorrow + filing['filing']['correction']['continuationIn'] = { + 'country': 'CA', + 'region': 'AB', + 'legalName': 'HAULER SERVICES', + 'identifier': 'AB1234567', + 'incorporationDate': future_date + } + filing['filing']['business']['legalType'] = 'C' + del filing['filing']['correction']['commentOnly'] + + + with jwt_request_context(app, jwt, [BASIC_USER]): + err = validate(business, filing) + + # Assert that the error list contains the appropriate error for future incorporation date + assert len(err.msg) == 1 + assert err.msg[0]['error'] == 'Incorporation date cannot be in the future.' + assert err.msg[0]['path'] == '/filing/correction/continuationIn/incorporationDate' + + +@pytest.mark.parametrize( + 'test_name, identifier, legal_name, expected_errors', + [ + ( + 'SUCCESS', + 'C1234567', + 'Test', + [] + ), + ( + 'SUCCESS_IDENTIFIER_AT_MAX_LENGTH', + 'A' * 50, + 'Test', + [] + ), + ( + 'SUCCESS_LEGAL_NAME_AT_MAX_LENGTH', + 'C1234567', + 'A' * 1000, + [] + ), + ( + 'FAIL_IDENTIFIER_EXCEEDS_MAX_LENGTH', + 'A' * 51, + 'Test', + [ + { + 'error': 'Identifier must not exceed 50 characters.', + 'path': '/filing/correction/continuationIn/identifier' + } + ] + ), + ( + 'FAIL_LEGAL_NAME_EXCEEDS_MAX_LENGTH', + 'C1234567', + 'A' * 1001, + [ + { + 'error': 'Legal name must not exceed 1000 characters.', + 'path': '/filing/correction/continuationIn/legalName' + } + ] + ), + ( + 'FAIL_BOTH_EXCEED_MAX_LENGTH', + 'A' * 51, + 'A' * 1001, + [ + { + 'error': 'Identifier must not exceed 50 characters.', + 'path': '/filing/correction/continuationIn/identifier' + }, + { + 'error': 'Legal name must not exceed 1000 characters.', + 'path': '/filing/correction/continuationIn/legalName' + } + ] + ), + ] +) +def test_validate_continuation_in_field_lengths(mocker, app, session, jwt, + test_name, identifier, legal_name, expected_errors): + """Assert that identifier and legalName enforce max length for continuation in filings. + + The required (non-empty / non-whitespace) rule for these fields now lives in the schema; + see test_continuation_in_foreign_jurisdiction_name_rejected_by_schema. + """ + _identifier = 'BC1234567' + business = factory_business(_identifier, entity_type='C') + corrected_filing = factory_completed_filing(business, CONTINUATION_IN_FILING_TEMPLATE) + + filing = copy.deepcopy(CORRECTION) + filing['filing']['header']['identifier'] = _identifier + filing['filing']['correction']['correctedFilingId'] = corrected_filing.id + filing['filing']['correction']['correctedFilingType'] = 'continuationIn' + continuation_in = { + 'country': 'US', + 'region': 'WA', + 'legalName': 'HAULER SERVICES', + 'incorporationDate': dt.now().date().isoformat() + } + filing['filing']['business']['legalType'] = 'C' + del filing['filing']['correction']['commentOnly'] + + if identifier is not None: + continuation_in['identifier'] = identifier + + if legal_name is not None: + continuation_in['legalName'] = legal_name + + filing['filing']['correction']['continuationIn'] = continuation_in + with jwt_request_context(app, jwt, [BASIC_USER]): + err = validate(business, filing) + + if expected_errors: + assert err.msg == expected_errors + else: + assert err is None + + +def test_validate_continuation_in_expro_founding_date_match(mocker, app, session, jwt): + """Assert continuation EXPRO business with matching founding date.""" + identifier = 'BC1234567' + business = factory_business(identifier, entity_type='C') + corrected_filing = factory_completed_filing(business, CONTINUATION_IN_FILING_TEMPLATE) + + filing = copy.deepcopy(CORRECTION) + filing['filing']['header']['identifier'] = identifier + filing['filing']['correction']['correctedFilingId'] = corrected_filing.id + filing['filing']['correction']['correctedFilingType'] = 'continuationIn' + filing['filing']['correction']['continuationIn'] = { + 'country': 'CA', + 'region': 'AB', + 'legalName': 'HAULER SERVICES', + 'identifier': 'AB1234567', + 'incorporationDate': dt.now().date().isoformat(), + 'expro': { + 'identifier': 'A0077779', + 'legalName': 'Test Company Inc.' + } + } + filing['filing']['business']['legalType'] = 'C' + del filing['filing']['correction']['commentOnly'] + + mocker.patch('legal_api.services.filings.validations.continuation_in.colin.query_business', return_value=( + { + 'business': { + 'identifier': 'A0077779', + 'legalName': 'Test Company Inc.' + } + }, + HTTPStatus.OK + )) + + with jwt_request_context(app, jwt, [BASIC_USER]): + err = validate(business, filing) + assert not err + + +@pytest.mark.parametrize('filing_type', ['continuationOut', 'amalgamationOut']) +@pytest.mark.parametrize( + 'test_name, expected_code, message', + [ + ('FAIL_IN_FUTURE', HTTPStatus.BAD_REQUEST, '{0} out date must be today or past.'), + ('SUCCESS_NO_CCO', None, None), + ('SUCCESS', None, None) + ] +) +def test_validate_continuation_out_date(session, app, jwt, filing_type, test_name, expected_code, message): + """Assert validate continuation_out_date.""" + identifier = 'BC1234567' + business = factory_business(identifier, entity_type='BC') + continuation_out_filing = copy.deepcopy(FILING_HEADER) + continuation_out_filing['filing'][filing_type] = copy.deepcopy(CONTINUATION_OUT if filing_type == 'continuationOut' else AMALGAMATION_OUT) + continuation_out_filing['filing']['header']['name'] = filing_type + + corrected_filing = factory_completed_filing(business, continuation_out_filing) + + + filing = copy.deepcopy(CORRECTION) + filing['filing']['header']['identifier'] = identifier + filing['filing']['correction']['correctedFilingId'] = corrected_filing.id + filing['filing']['correction']['correctedFilingType'] = filing_type + filing['filing']['correction'][filing_type] = { + 'country': 'CA', + 'region': 'AB', + 'legalName': 'HAULER SERVICES', + 'date': '2023-06-19' + } + del filing['filing']['correction']['commentOnly'] + + if test_name == 'FAIL_IN_FUTURE': + filing['filing']['correction'][filing_type]['date'] = \ + (LegislationDatetime.now() + datedelta.datedelta(days=1)).strftime(date_format) + + with jwt_request_context(app, jwt, [BASIC_USER]): + err = validate(business, filing) + + # validate outcomes + if test_name == 'FAIL_IN_FUTURE': + assert expected_code == err.code + assert message.format(filing_type.replace('Out', '').capitalize()) == err.msg[0]['error'] + else: + assert not err + + +@pytest.mark.parametrize('filing_type', ['continuationOut', 'amalgamationOut']) +@pytest.mark.parametrize( + 'test_name, expected_code, message', + [ + ('FAIL_NO_COUNTRY', HTTPStatus.UNPROCESSABLE_ENTITY, None), + ('FAIL_INVALID_COUNTRY', HTTPStatus.BAD_REQUEST, 'Invalid country.'), + ('FAIL_REGION_BC', HTTPStatus.BAD_REQUEST, 'Region should not be BC.'), + ('FAIL_INVALID_REGION', HTTPStatus.BAD_REQUEST, 'Invalid region.'), + ('FAIL_INVALID_US_REGION', HTTPStatus.BAD_REQUEST, 'Invalid region.'), + ('SUCCESS', None, None) + ] +) +def test_validate_continuation_out_foreign_jurisdiction(session, app, jwt, filing_type, test_name, expected_code, message): + """Assert validate continuation_out foreign jurisdiction.""" + identifier = 'BC1234567' + business = factory_business(identifier, entity_type='BC') + continuation_out_filing = copy.deepcopy(FILING_HEADER) + continuation_out_filing['filing'][filing_type] = copy.deepcopy(CONTINUATION_OUT if filing_type == 'continuationOut' else AMALGAMATION_OUT) + continuation_out_filing['filing']['header']['name'] = filing_type + + corrected_filing = factory_completed_filing(business, continuation_out_filing) + + filing = copy.deepcopy(CORRECTION) + filing['filing']['header']['identifier'] = identifier + filing['filing']['correction']['correctedFilingId'] = corrected_filing.id + filing['filing']['correction']['correctedFilingType'] = filing_type + filing['filing']['correction'][filing_type] = { + 'country': 'CA', + 'region': 'AB', + 'legalName': 'HAULER SERVICES', + 'date': '2023-06-19' + } + del filing['filing']['correction']['commentOnly'] + + + if test_name == 'FAIL_NO_COUNTRY': + del filing['filing']['correction'][filing_type]['country'] + elif test_name == 'FAIL_INVALID_COUNTRY': + filing['filing']['correction'][filing_type]['country'] = 'NONE' + elif test_name == 'FAIL_REGION_BC': + filing['filing']['correction'][filing_type]['region'] = 'BC' + elif test_name == 'FAIL_INVALID_REGION': + filing['filing']['correction'][filing_type]['region'] = 'NONE' + elif test_name == 'FAIL_INVALID_US_REGION': + filing['filing']['correction'][filing_type]['country'] = 'US' + filing['filing']['correction'][filing_type]['region'] = 'NONE' + + with jwt_request_context(app, jwt, [BASIC_USER]): + err = validate(business, filing) + + # validate outcomes + if test_name != 'SUCCESS': + assert expected_code == err.code + if message: + assert message == err.msg[0]['error'] + else: + assert not err + + +def test_validate_correction_amalgamation_ting_not_found(mocker, app, session, jwt): + """Assert that an error is raised if the correction amalgamation filing has an id that doesn't exist""" + identifier = 'BC1234567' + business = factory_business(identifier, entity_type='BC') + filing_type = 'amalgamationApplication' + data, corrected_filing = _create_amalgation_business(business) + del data['amalgamatingBusinesses'][0] + data['amalgamatingBusinesses'][0]['id'] = 986546516 + + filing = copy.deepcopy(CORRECTION) + filing['filing']['header']['identifier'] = identifier + filing['filing']['correction']['correctedFilingId'] = corrected_filing.id + + filing['filing']['correction']['correctedFilingType'] = filing_type + filing['filing']['correction']['amalgamation'] = data + filing['filing']['business']['legalType'] = 'BC' + del filing['filing']['correction']['commentOnly'] + + + with jwt_request_context(app, jwt, [BASIC_USER]): + err = validate(business, filing) + + assert len(err.msg) == 1 + assert err.msg[0]['error'] == 'Amalgamating business not found.' + assert err.msg[0]['path'] == '/filing/correction/amalgamation/amalgamatingBusinesses/0' + + +@pytest.mark.parametrize( + 'test_name, expected_code, message', + [ + ('FAIL_NO_COUNTRY', HTTPStatus.UNPROCESSABLE_ENTITY, None), + ('FAIL_INVALID_COUNTRY', HTTPStatus.BAD_REQUEST, 'Invalid country.'), + ('FAIL_INVALID_REGION', HTTPStatus.BAD_REQUEST, 'Invalid region.'), + ('SUCCESS', None, None) + ] +) +def test_validate_correction_amalgamation_foreign_jurisdiction(mocker, app, session, jwt, test_name, expected_code, message): + """Assert that an error is raised if the correction amalgamation filing has an invalid business""" + identifier = 'BC1234567' + business = factory_business(identifier, entity_type='BC') + filing_type = 'amalgamationApplication' + data, corrected_filing = _create_amalgation_business(business) + del data['amalgamatingBusinesses'][0] + + if test_name == 'FAIL_NO_COUNTRY': + del data['amalgamatingBusinesses'][0]['foreignJurisdiction']['country'] + elif test_name == 'FAIL_INVALID_COUNTRY': + data['amalgamatingBusinesses'][0]['foreignJurisdiction']['country'] = 'NONE' + elif test_name == 'FAIL_INVALID_REGION': + data['amalgamatingBusinesses'][0]['foreignJurisdiction']['region'] = 'NONE' + elif test_name == 'FAIL_INVALID_US_REGION': + data['amalgamatingBusinesses'][0]['foreignJurisdiction']['country'] = 'US' + data['amalgamatingBusinesses'][0]['foreignJurisdiction']['region'] = 'NONE' + + filing = copy.deepcopy(CORRECTION) + filing['filing']['header']['identifier'] = identifier + filing['filing']['correction']['correctedFilingId'] = corrected_filing.id + + filing['filing']['correction']['correctedFilingType'] = filing_type + filing['filing']['correction']['amalgamation'] = data + filing['filing']['business']['legalType'] = 'BC' + del filing['filing']['correction']['commentOnly'] + + with jwt_request_context(app, jwt, [BASIC_USER]): + err = validate(business, filing) + + # validate outcomes + if test_name != 'SUCCESS': + assert expected_code == err.code + if message: + assert message == err.msg[0]['error'] + else: + assert not err + + +def test_validate_correction_amalgamation_ting_invalid(mocker, app, session, jwt): + """Assert that an error is raised if the correction amalgamation filing has an invalid business""" + identifier = 'BC1234567' + business = factory_business(identifier, entity_type='BC') + filing_type = 'amalgamationApplication' + data, corrected_filing = _create_amalgation_business(business) + data['amalgamatingBusinesses'][0]['foreignJurisdiction'] = { + 'country': 'CA', + 'region': 'AB', + } + data['amalgamatingBusinesses'][0]['legalName'] = 'HAULER SERVICES' + filing = copy.deepcopy(CORRECTION) + filing['filing']['header']['identifier'] = identifier + filing['filing']['correction']['correctedFilingId'] = corrected_filing.id + + filing['filing']['correction']['correctedFilingType'] = filing_type + filing['filing']['correction']['amalgamation'] = data + filing['filing']['business']['legalType'] = 'BC' + del filing['filing']['correction']['commentOnly'] + + + with jwt_request_context(app, jwt, [BASIC_USER]): + err = validate(business, filing) + + assert len(err.msg) == 1 + assert err.msg[0]['error'] == 'Can only correct foreign businesses.' + assert err.msg[0]['path'] == '/filing/correction/amalgamation/amalgamatingBusinesses/0' + + +def _create_amalgation_business(business): + amalgamating_identifier = 'BC1234567' + amalgamating_business = factory_business(amalgamating_identifier, entity_type='BC') + filing_type = "amalgamationApplication" + filing_json = copy.deepcopy(FILING_HEADER) + filing_json['filing'][filing_type] = copy.deepcopy(AMALGAMATION_APPLICATION) + filing_json['filing']['header']['name'] = filing_type + + filing = factory_completed_filing(business, filing_json) + amalgamating_business_json = [ + { + 'role': 'amalgamating', + 'identifier': amalgamating_identifier + }, + { + 'role': 'amalgamating', + 'foreignJurisdiction': { + 'country': 'CA', + 'region': 'AB', + }, + 'legalName': 'HAULER SERVICES', + 'identifier': 'AB1234567' + } + ] + + data = { + 'courtApproval': False, + 'amalgamatingBusinesses':amalgamating_business_json + } + amalgamation = Amalgamation( + amalgamation_type=Amalgamation.AmalgamationTypes.regular, + amalgamation_date=LegislationDatetime.now(), + court_approval=False, + filing_id=filing.id, + business_id=business.id, + ) + amalgamation.amalgamating_businesses.append(AmalgamatingBusiness( + role=AmalgamatingBusiness.Role.amalgamating, + business_id=amalgamating_business.id + )) + amalgamation.amalgamating_businesses.append(AmalgamatingBusiness( + role=AmalgamatingBusiness.Role.amalgamating, + foreign_jurisdiction=amalgamating_business_json[1]['foreignJurisdiction']['country'], + foreign_jurisdiction_region=amalgamating_business_json[1]['foreignJurisdiction']['region'], + foreign_name=amalgamating_business_json[1]['legalName'], + foreign_identifier=amalgamating_business_json[1]['identifier'] + )) + business.amalgamation.append(amalgamation) + business.save() + for ting in amalgamation.amalgamating_businesses.all(): + if ting.business_id: + amalgamating_business_json[0]['id'] = ting.id + else: + amalgamating_business_json[1]['id'] = ting.id + return data, filing + + +@pytest.mark.parametrize('invalid_court_order, error_msg', [ + ({ + 'fileNumber': '123456789012345678901', # long fileNumber + 'orderDate': '2021-01-30T09:56:01+01:00', + 'effectOfOrder': 'planOfArrangement' + }, "'123456789012345678901' is too long"), + ({ + 'orderDate': '2021-01-30T09:56:01+01:00', + 'effectOfOrder': 'planOfArrangement' + }, "'fileNumber' is a required property"), + ({ + 'fileNumber': 'Valid file number', + 'orderDate': 'a2021-01-30T09:56:01', # Invalid date + 'effectOfOrder': 'planOfArrangement' + }, "'a2021-01-30T09:56:01' is not a 'date-time'"), + ({ + 'fileNumber': 'Valid file number', + 'orderDate': (datetime.now(timezone.utc) + timedelta(days=1)).isoformat(), + 'effectOfOrder': 'planOfArrangement' + }, "Court order date cannot be in the future."), + ({ + 'fileNumber': 'Valid File Number', + 'orderDate': '2021-01-30T09:56:01+01:00', + 'effectOfOrder': 'invalid' # Invalid effectOfOrder + }, 'Invalid effectOfOrder.') +]) +def test_validate_invalid_court_order(app, jwt, session, invalid_court_order, error_msg): + """Assert not valid court order.""" + identifier = 'BC1234567' + business = factory_business(identifier, entity_type='BC') + + corrected_filing = factory_completed_filing(business, INCORPORATION_APPLICATION) + + filing = copy.deepcopy(CORRECTION) + filing['filing']['header']['identifier'] = identifier + filing['filing']['correction']['correctedFilingId'] = corrected_filing.id + del filing['filing']['correction']['commentOnly'] + filing['filing']['correction']['courtOrder'] = invalid_court_order + + with jwt_request_context(app, jwt, [BASIC_USER]): + err = validate(business, filing) + assert err + assert err.msg[0]['error'] == error_msg + + +@pytest.mark.parametrize('invalid_court_order, error_msg', [ + ({ + 'fileNumber': '123456789012345678901', # long fileNumber + 'orderDate': '2021-01-30T09:56:01+01:00', + 'effectOfOrder': 'planOfArrangement' + }, "'123456789012345678901' is too long"), + ({ + 'orderDate': '2021-01-30T09:56:01+01:00', + 'effectOfOrder': 'planOfArrangement' + }, "'fileNumber' is a required property"), + ({ + 'fileNumber': 'Valid file number', + 'orderDate': 'a2021-01-30T09:56:01', # Invalid date + 'effectOfOrder': 'planOfArrangement' + }, "'a2021-01-30T09:56:01' is not a 'date-time'"), + ({ + 'fileNumber': 'Valid file number', + 'orderDate': (datetime.now(timezone.utc) + timedelta(days=1)).isoformat(), + 'effectOfOrder': 'planOfArrangement' + }, "Court order date cannot be in the future."), + ({ + 'fileNumber': 'Valid File Number', + 'orderDate': '2021-01-30T09:56:01+01:00', + 'effectOfOrder': 'invalid' # Invalid effectOfOrder + }, 'Invalid effectOfOrder.') +]) +def test_validate_invalid_court_orders_new(app, jwt, session, invalid_court_order, error_msg): + """Assert not valid court order.""" + identifier = 'BC1234567' + business = factory_business(identifier, entity_type='BC') + + ia_filing = factory_completed_filing(business, INCORPORATION_APPLICATION) + + filing = copy.deepcopy(CORRECTION) + filing['filing']['header']['identifier'] = identifier + filing['filing']['correction']['correctedFilingId'] = ia_filing.id + del filing['filing']['correction']['commentOnly'] + invalid_court_order["filingId"] = ia_filing.id + + filing['filing']['correction']['courtOrders'] = [invalid_court_order] + + with jwt_request_context(app, jwt, [BASIC_USER]): + err = validate(business, filing) + assert err + assert err.msg[0]['error'] == error_msg + + +@pytest.mark.parametrize('invalid_court_order, error_msg', [ + ({ + 'fileNumber': '123456789012345678901', # long fileNumber + 'orderDate': '2021-01-30T09:56:01+01:00', + 'effectOfOrder': 'planOfArrangement' + }, "'123456789012345678901' is too long"), + ({ + 'orderDate': '2021-01-30T09:56:01+01:00', + 'effectOfOrder': 'planOfArrangement' + }, "'fileNumber' is a required property"), + ({ + 'fileNumber': 'Valid File Number', + 'orderDate': '2021-01-30T09:56:01+01:00', + 'effectOfOrder': 'invalid' # Invalid effectOfOrder + }, 'Invalid effectOfOrder.') +]) +def test_validate_invalid_court_orders_old(app, jwt, session, invalid_court_order, error_msg): + """Assert not valid court order.""" + identifier = 'BC1234567' + business = factory_business(identifier, entity_type='BC') + + ia_filing = factory_completed_filing(business, INCORPORATION_APPLICATION) + court_order_filing = factory_completed_filing(business, COURT_ORDER_FILING_TEMPLATE) + court_order = CourtOrder( + filing_id=court_order_filing.id, + business_id=business.id, + effect_of_order='planOfArrangement', + order_details='Court order details' + ) + court_order.save() + + filing = copy.deepcopy(CORRECTION) + filing['filing']['header']['identifier'] = identifier + filing['filing']['correction']['correctedFilingId'] = ia_filing.id + del filing['filing']['correction']['commentOnly'] + invalid_court_order["filingId"] = ia_filing.id + invalid_court_order["id"] = court_order.id + + filing['filing']['correction']['courtOrders'] = [invalid_court_order] + + with jwt_request_context(app, jwt, [BASIC_USER]): + err = validate(business, filing) + assert err + assert err.msg[0]['error'] == error_msg + + +@pytest.mark.parametrize('test_name, error_msg', [ + ("court_order_not_found", "Court order not found."), + ("filing_id_does_not_match", "Filing Id does not match corrected filing Id."), + ("multiple_court_orders", "Only one court order can be added."), + ("multiple_court_orders_existing", "Only one court order can be added per filing."), +]) +def test_validate_invalid_court_orders(app, jwt, session, test_name, error_msg): + """Assert not valid court order.""" + identifier = 'BC1234567' + business = factory_business(identifier, entity_type='BC') + + invalid_court_order = { + 'fileNumber': '123456789', + 'effectOfOrder': 'planOfArrangement', + 'orderDetails': 'Court order details', + "filingId": 5645646 + } + ia_filing = factory_completed_filing(business, INCORPORATION_APPLICATION) + + filing = copy.deepcopy(CORRECTION) + filing['filing']['header']['identifier'] = identifier + filing['filing']['correction']['correctedFilingId'] = ia_filing.id + del filing['filing']['correction']['commentOnly'] + if test_name == "court_order_not_found": + invalid_court_order["filingId"] = ia_filing.id + invalid_court_order["id"] = 99999999 + filing['filing']['correction']['courtOrders'] = [invalid_court_order] + elif test_name == "filing_id_does_not_match": + invalid_court_order["filingId"] = 6546456 + filing['filing']['correction']['courtOrders'] = [invalid_court_order] + elif test_name == "multiple_court_orders": + invalid_court_order["filingId"] = ia_filing.id + filing['filing']['correction']['courtOrders'] = [invalid_court_order, invalid_court_order] + elif test_name == "multiple_court_orders_existing": + court_order = CourtOrder( + filing_id=ia_filing.id, + business_id=business.id, + effect_of_order='planOfArrangement', + order_details='Court order details' + ) + court_order.save() + + invalid_court_order["filingId"] = ia_filing.id + invalid_court_order2 = copy.deepcopy(invalid_court_order) + invalid_court_order2["id"] = court_order.id + filing['filing']['correction']['courtOrders'] = [invalid_court_order, invalid_court_order2] + + with jwt_request_context(app, jwt, [BASIC_USER]): + err = validate(business, filing) + assert err + assert err.msg[0]['error'] == error_msg diff --git a/legal-api/tests/unit/services/filings/validations/test_correction_special_resolution.py b/legal-api/tests/unit/services/filings/validations/test_correction_special_resolution.py index 0c048b2851..8b8a71f30f 100644 --- a/legal-api/tests/unit/services/filings/validations/test_correction_special_resolution.py +++ b/legal-api/tests/unit/services/filings/validations/test_correction_special_resolution.py @@ -45,6 +45,7 @@ def test_valid_special_resolution_correction(session, app, jwt): f = copy.deepcopy(correction_data) f['filing']['header']['identifier'] = identifier f['filing']['correction']['correctedFilingId'] = corrected_filing.id + del f['filing']['correction']['rulesFileKey'] with jwt_request_context(app, jwt, [STAFF_ROLE]): if err := validate(business, f): @@ -86,7 +87,8 @@ def test_parties_special_resolution_correction(session, app, jwt, test_name, leg correction_data = copy.deepcopy(FILING_HEADER) correction_data['filing']['correction'] = copy.deepcopy(CORRECTION_CP_SPECIAL_RESOLUTION) correction_data['filing']['header']['name'] = 'correction' - + del correction_data['filing']['correction']['rulesFileKey'] + f = copy.deepcopy(correction_data) f['filing']['header']['identifier'] = identifier f['filing']['correction']['correctedFilingId'] = corrected_filing.id diff --git a/legal-api/tests/unit/services/filings/validations/test_court_order.py b/legal-api/tests/unit/services/filings/validations/test_court_order.py index 433207a5a3..17a0bbb461 100644 --- a/legal-api/tests/unit/services/filings/validations/test_court_order.py +++ b/legal-api/tests/unit/services/filings/validations/test_court_order.py @@ -22,7 +22,7 @@ from legal_api.services.filings.validations.court_order import validate from tests.unit.models import factory_business -from tests.unit.services.filings.test_utils import _upload_file +from tests.unit.services.filings.test_utils import _upload_file, mock_drs_get_document from tests.unit.services.filings.validations import lists_are_equal @@ -64,8 +64,9 @@ def test_court_orders(session, test_status, expected_code, expected_msg): 'error': 'Document must be set to fit onto 8.5” x 11” letter-size paper.', 'path': file_key_path}]) ]) -def test_court_order_file(session, minio_server, test_name, expected_code, expected_msg): +def test_court_order_file(session, monkeypatch, test_name, expected_code, expected_msg): """Assert valid court order.""" + mock_drs_get_document(monkeypatch) business = factory_business('BC1234567') filing = copy.deepcopy(COURT_ORDER_FILING_TEMPLATE) diff --git a/legal-api/tests/unit/services/filings/validations/test_dissolution.py b/legal-api/tests/unit/services/filings/validations/test_dissolution.py index a141d70dca..e858326c22 100644 --- a/legal-api/tests/unit/services/filings/validations/test_dissolution.py +++ b/legal-api/tests/unit/services/filings/validations/test_dissolution.py @@ -28,10 +28,10 @@ from reportlab.lib.pagesizes import letter from business_model.models import Business -from legal_api.services import MinioService, flags +from legal_api.services import flags from legal_api.services.filings.validations import dissolution from legal_api.services.filings.validations.dissolution import validate -from tests.unit.services.filings.test_utils import _upload_file +from tests.unit.services.filings.test_utils import _upload_file, mock_drs_get_document from tests.unit.services.filings.validations import create_party, create_party_address, lists_are_equal @@ -76,7 +76,7 @@ def test_dissolution_type(session, test_status, legal_type, dissolution_type, del filing['filing']['dissolution']['affidavitFileKey'] del filing['filing']['dissolution']['parties'] - with patch.object(dissolution, 'validate_affidavit', return_value=None), \ + with patch.object(dissolution, 'validate_affidavit', return_value=[]), \ patch.object(dissolution, 'validate_dissolution_parties_roles', return_value=None), \ patch('legal_api.services.filings.validations.dissolution.check_good_standing_permission', return_value=None): err = validate(business, filing) @@ -117,7 +117,7 @@ def test_dissolution_statement_type(session, test_status, legal_type, dissolutio del filing['filing']['dissolution']['dissolutionStatementType'] # perform test - with patch.object(dissolution, 'validate_affidavit', return_value=None), \ + with patch.object(dissolution, 'validate_affidavit', return_value=[]), \ patch.object(dissolution, 'validate_dissolution_parties_roles', return_value=None), \ patch('legal_api.services.filings.validations.dissolution.check_good_standing_permission', return_value=None): err = validate(business, filing) @@ -181,7 +181,7 @@ def test_dissolution_party_roles(session, legal_type, dissolution_type, roles, e p = create_party([role], i + 1, mailing_addr, delivery_addr) filing['filing']['dissolution']['parties'].append(p) - with patch.object(dissolution, 'validate_affidavit', return_value=None), \ + with patch.object(dissolution, 'validate_affidavit', return_value=[]), \ patch('legal_api.services.filings.validations.dissolution.check_good_standing_permission', return_value=None): err = validate(business, filing) @@ -244,7 +244,7 @@ def test_dissolution_address(session, test_status, legal_type, address_validatio if address_validation == 'liquidator_only_non_ca_address': filing['filing']['dissolution']['parties'][1]['mailingAddress']['addressCountry'] = 'US' - with patch.object(dissolution, 'validate_affidavit', return_value=None), \ + with patch.object(dissolution, 'validate_affidavit', return_value=[]), \ patch.object(dissolution, 'validate_dissolution_parties_roles', return_value=None), \ patch('legal_api.services.filings.validations.dissolution.check_good_standing_permission', return_value=None): err = validate(business, filing) @@ -318,7 +318,7 @@ def test_dissolution_special_resolution(session, test_name, legal_type, dissolut resolution_date_time = datetime.strptime(resolution_date_str, '%Y-%m-%d') business.founding_date = resolution_date_time - timedelta(days=1000) - with patch.object(dissolution, 'validate_affidavit', return_value=None), \ + with patch.object(dissolution, 'validate_affidavit', return_value=[]), \ patch.object(dissolution, 'validate_dissolution_parties_roles', return_value=None), \ patch('legal_api.services.filings.validations.dissolution.check_good_standing_permission', return_value=None): err = validate(business, filing) @@ -351,9 +351,10 @@ def test_dissolution_special_resolution(session, test_name, legal_type, dissolut }]), ] ) -def test_dissolution_affidavit(session, minio_server, test_name, legal_type, dissolution_type, key, scenario, +def test_dissolution_affidavit(session, monkeypatch, test_name, legal_type, dissolution_type, key, scenario, identifier, expected_code, expected_msg): # pylint: disable=too-many-arguments """Assert that an affidavit can be validated.""" + mock_drs_get_document(monkeypatch) # setup business = Business(identifier=identifier, legal_type=legal_type) @@ -391,15 +392,12 @@ def test_dissolution_affidavit(session, minio_server, test_name, legal_type, dis else: assert err is None - # Cleanup - if file_key := filing['filing']['dissolution'].get('affidavitFileKey', None): - MinioService.delete_file(file_key) @pytest.mark.parametrize( 'test_status, file_number, effect_of_order, expected_code, expected_msg', [ - ('FAIL', None, 'planOfArrangement', HTTPStatus.BAD_REQUEST, 'Court order file number is required.'), + ('FAIL', None, 'planOfArrangement', HTTPStatus.UNPROCESSABLE_ENTITY, "'fileNumber' is a required property"), ('FAIL', '12345678901234567890', 'invalid', HTTPStatus.BAD_REQUEST, 'Invalid effectOfOrder.'), ('SUCCESS', '12345678901234567890', 'planOfArrangement', None, None) ] @@ -424,10 +422,12 @@ def test_dissolution_court_orders(session, test_status, file_number, effect_of_o filing['filing']['dissolution']['courtOrder'] = court_order - with patch.object(dissolution, 'validate_affidavit', return_value=None), \ + from legal_api.services.filings import validate as _validate + + with patch.object(dissolution, 'validate_affidavit', return_value=[]), \ patch.object(dissolution, 'validate_dissolution_parties_roles', return_value=None), \ patch('legal_api.services.filings.validations.dissolution.check_good_standing_permission', return_value=None): - err = validate(business, filing) + err = _validate(business, filing) # validate outcomes if test_status == 'FAIL': @@ -499,7 +499,7 @@ def test_dissolution_custodian_email(session, test_status, legal_type, dissoluti filing['filing']['dissolution']['details'] = "Some Details" del filing['filing']['dissolution']['affidavitFileKey'] - with patch.object(dissolution, 'validate_affidavit', return_value=None), \ + with patch.object(dissolution, 'validate_affidavit', return_value=[]), \ patch.object(dissolution, 'validate_dissolution_parties_roles', return_value=None), \ patch('legal_api.services.filings.validations.dissolution.check_good_standing_permission', return_value=None): err = validate(business, filing) @@ -574,7 +574,7 @@ def test_dissolution_custodian_name(session, test_status, legal_type, dissolutio if first_name is not None: officer['firstName'] = first_name - with patch.object(dissolution, 'validate_affidavit', return_value=None), \ + with patch.object(dissolution, 'validate_affidavit', return_value=[]), \ patch.object(dissolution, 'validate_dissolution_parties_roles', return_value=None), \ patch('legal_api.services.filings.validations.dissolution.check_good_standing_permission', return_value=None): err = validate(business, filing) @@ -686,7 +686,7 @@ def test_dissolution_good_standing_permission(session, test_name, good_standing, patch.object(Business, 'good_standing', new_callable=lambda: good_standing), patch.object(PermissionService, 'check_user_permission', return_value=permission_error), patch.object(dissolution, 'validate_dissolution_parties_roles', return_value=None), - patch.object(dissolution, 'validate_affidavit', return_value=None), + patch.object(dissolution, 'validate_affidavit', return_value=[]), patch.object(dissolution, 'check_good_standing_permission', return_value=permission_error if not good_standing and not has_permission and flag_enabled else None), patch.object(dissolution, 'validate_permission_and_completing_party', return_value=None), patch.object(dissolution, '_check_dissolution_permission', return_value=None), diff --git a/legal-api/tests/unit/services/filings/validations/test_incorporation_application.py b/legal-api/tests/unit/services/filings/validations/test_incorporation_application.py index 087245320e..00969d8cab 100644 --- a/legal-api/tests/unit/services/filings/validations/test_incorporation_application.py +++ b/legal-api/tests/unit/services/filings/validations/test_incorporation_application.py @@ -29,7 +29,7 @@ from business_model.models import Business from legal_api.services.filings import validate -from tests.unit.services.filings.test_utils import _upload_file +from tests.unit.services.filings.test_utils import _upload_file, mock_drs_get_document from . import create_party, create_party_address, lists_are_equal, create_officer from tests import not_github_ci @@ -655,10 +655,11 @@ def test_validate_name_request(session, mocker, test_name, legal_type, expected_ ) ]) @pytest.mark.parametrize('cp_flag_enabled', [True, False]) -def test_validate_incorporation_role(session, minio_server, mocker, test_name, +def test_validate_incorporation_role(session, monkeypatch, mocker, test_name, legal_type, parties, expected_code, expected_msg, cp_flag_enabled): """Assert that incorporation parties roles can be validated.""" + mock_drs_get_document(monkeypatch) mocker.patch.object(flags, 'value', return_value=["incorporationApplication-completingParty"] if cp_flag_enabled else []) filing_json = copy.deepcopy(INCORPORATION_FILING_TEMPLATE) @@ -1551,11 +1552,17 @@ def test_validate_incorporation_share_classes(session, mocker, test_name, legal_ ('SUCCESS', '2020-09-18T00:00:00+00:00', None, None), ('SUCCESS', None, None, None), ('FAIL_INVALID_DATE_TIME_FORMAT', '2020-09-44T00:00:00Z', - HTTPStatus.UNPROCESSABLE_CONTENT, [{ - 'error': "'2020-09-44T00:00:00Z' is not a 'date-time'", - 'path': 'filing/header/effectiveDate', - 'context': [] - }]), + HTTPStatus.UNPROCESSABLE_CONTENT, [ + { + 'error': "'2020-09-44T00:00:00Z' is not a 'date-time'", + 'path': 'filing/header/effectiveDate', + 'context': [] + }, + { + 'error': "'2020-09-44T00:00:00Z' is not a 'date-time'", + 'path': 'filing/header/effectiveDate', + 'context': [] + }]), ('FAIL_INVALID_DATE_TIME_MINIMUM', '2020-09-17T00:01:00+00:00', HTTPStatus.BAD_REQUEST, [{ 'error': 'Invalid Datetime, effective date must be a minimum of 2 minutes ahead.', @@ -1571,7 +1578,6 @@ def test_validate_incorporation_share_classes(session, mocker, test_name, legal_ def test_validate_incorporation_effective_date(session, mocker, test_name, effective_date, expected_code, expected_msg): """Assert that validator validates effective date correctly.""" filing_json = copy.deepcopy(FILING_HEADER) - filing_json['filing'].pop('business') filing_json['filing']['header'] = {'name': incorporation_application_name, 'date': '2019-04-08', 'certifiedBy': 'full name', 'authorizationReceived': True, 'email': 'no_one@never.get', 'filingId': 1} @@ -1626,9 +1632,10 @@ def test_validate_incorporation_effective_date(session, mocker, test_name, effec 'path': memorandum_file_key_path }]), ]) -def test_validate_cooperative_documents(session, mocker, minio_server, test_name, key, scenario, expected_code, +def test_validate_cooperative_documents(session, monkeypatch, mocker, test_name, key, scenario, expected_code, expected_msg): """Assert that validator validates cooperative documents correctly.""" + mock_drs_get_document(monkeypatch) filing_json = copy.deepcopy(INCORPORATION_FILING_TEMPLATE) filing_json['filing']['header'] = {'name': incorporation_application_name, 'date': '2019-04-08', 'certifiedBy': 'full name', 'authorizationReceived': True, @@ -1736,7 +1743,6 @@ def test_incorporation_application_share_class_series_validation(mocker, app, se has_rights_or_restrictions, has_series, should_pass): """Test share class/series validation in incorporation application.""" filing_json = copy.deepcopy(FILING_HEADER) - filing_json['filing'].pop('business') filing_json['filing']['header'] = {'name': incorporation_application_name, 'date': '2019-04-08', 'certifiedBy': 'full name', 'authorizationReceived': True, 'email': 'no_one@never.get', 'filingId': 1} @@ -1770,7 +1776,6 @@ def test_validate_incorporation_application_parties_delivery_address(mocker, app has_delivery_address, expected_code, expected_msg): """Test parties delivery address validation in incorporation application.""" filing_json = copy.deepcopy(FILING_HEADER) - filing_json['filing'].pop('business') filing_json['filing']['header'] = {'name': incorporation_application_name, 'date': '2019-04-08', 'certifiedBy': 'full name', 'authorizationReceived': True, 'email': 'no_one@never.get', 'filingId': 1} @@ -1849,7 +1854,6 @@ def test_ia_phone_number_validation(session, should_pass, phone_number, extensio (True, 'test@example.co.uk'), (True, 'user@[192.168.1.1]'), (True, 'no_one@never.get'), - (True, '"quoted"@example.com'), # Invalid email (False, 'no_one@never.'), (False, '@invalid.com'), @@ -1858,6 +1862,7 @@ def test_ia_phone_number_validation(session, should_pass, phone_number, extensio (False, 'test@domain'), (False, 'test @example.com'), (False, 'test@ example.com'), + (False, '"quoted"@example.com'), ]) def test_ia_email_validation(session, should_pass, email): """Test validate email format if provided.""" diff --git a/legal-api/tests/unit/services/filings/validations/test_registrars_notation.py b/legal-api/tests/unit/services/filings/validations/test_registrars_notation.py new file mode 100644 index 0000000000..af056bcbfc --- /dev/null +++ b/legal-api/tests/unit/services/filings/validations/test_registrars_notation.py @@ -0,0 +1,52 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test Registrars Order validations.""" +import copy +from http import HTTPStatus + +import pytest +from registry_schemas.example_data import REGISTRARS_NOTATION_FILING_TEMPLATE + +from legal_api.services.filings.validations.registrars_notation import validate + +from tests.unit.models import factory_business + + +@pytest.mark.parametrize( + 'test_status, expected_code, expected_msg', + [ + ('FAIL_INVALID_EFFECT_OF_ORDER', HTTPStatus.BAD_REQUEST, [ + {'error': 'Invalid effectOfOrder.', 'path': '/filing/registrarsNotation/effectOfOrder'}]), + ('FAIL_MISSING_FILE_NUMBER', HTTPStatus.BAD_REQUEST, [ + {'error': 'Court Order Number is required when this filing is pursuant to a Plan of Arrangement.', + 'path': '/filing/registrarsNotation/fileNumber'}]), + ('SUCCESS', None, None) + ] +) +def test_registrars_notation(session, test_status, expected_code, expected_msg): + """Assert valid registrars notation.""" + business = factory_business('BC1234567') + filing = copy.deepcopy(REGISTRARS_NOTATION_FILING_TEMPLATE) + if test_status == 'FAIL_INVALID_EFFECT_OF_ORDER': + filing['filing']['registrarsNotation']['effectOfOrder'] = 'invalid' + elif test_status == 'FAIL_MISSING_FILE_NUMBER': + filing['filing']['registrarsNotation']['fileNumber'] = None + err = validate(business, filing) + + if expected_code: + assert err.code == expected_code + assert err.msg[0]['error'] == expected_msg[0]['error'] + assert err.msg[0]['path'] == expected_msg[0]['path'] + else: + assert err is None diff --git a/legal-api/tests/unit/services/filings/validations/test_registrars_order.py b/legal-api/tests/unit/services/filings/validations/test_registrars_order.py new file mode 100644 index 0000000000..63bdbb4e4f --- /dev/null +++ b/legal-api/tests/unit/services/filings/validations/test_registrars_order.py @@ -0,0 +1,52 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Test Registrars Order validations.""" +import copy +from http import HTTPStatus + +import pytest +from registry_schemas.example_data import REGISTRARS_ORDER_FILING_TEMPLATE + +from legal_api.services.filings.validations.registrars_order import validate + +from tests.unit.models import factory_business + + +@pytest.mark.parametrize( + 'test_status, expected_code, expected_msg', + [ + ('FAIL_INVALID_EFFECT_OF_ORDER', HTTPStatus.BAD_REQUEST, [ + {'error': 'Invalid effectOfOrder.', 'path': '/filing/registrarsOrder/effectOfOrder'}]), + ('FAIL_MISSING_FILE_NUMBER', HTTPStatus.BAD_REQUEST, [ + {'error': 'Court Order Number is required when this filing is pursuant to a Plan of Arrangement.', + 'path': '/filing/registrarsOrder/fileNumber'}]), + ('SUCCESS', None, None) + ] +) +def test_registrars_order(session, test_status, expected_code, expected_msg): + """Assert valid registrars order.""" + business = factory_business('BC1234567') + filing = copy.deepcopy(REGISTRARS_ORDER_FILING_TEMPLATE) + if test_status == 'FAIL_INVALID_EFFECT_OF_ORDER': + filing['filing']['registrarsOrder']['effectOfOrder'] = 'invalid' + elif test_status == 'FAIL_MISSING_FILE_NUMBER': + filing['filing']['registrarsOrder']['fileNumber'] = None + err = validate(business, filing) + + if expected_code: + assert err.code == expected_code + assert err.msg[0]['error'] == expected_msg[0]['error'] + assert err.msg[0]['path'] == expected_msg[0]['path'] + else: + assert err is None diff --git a/legal-api/tests/unit/services/filings/validations/test_registration.py b/legal-api/tests/unit/services/filings/validations/test_registration.py index 11f432db17..0759da7100 100644 --- a/legal-api/tests/unit/services/filings/validations/test_registration.py +++ b/legal-api/tests/unit/services/filings/validations/test_registration.py @@ -21,7 +21,7 @@ from dateutil.relativedelta import relativedelta from business_common.utils.legislation_datetime import LegislationDatetime -from business_model.models import Business +from business_model.models import Business, User from legal_api.services import NaicsService, NameXService, flags from legal_api.services.filings.validations.validation import validate from legal_api.services.authz import BASIC_USER, STAFF_ROLE @@ -157,8 +157,7 @@ def test_dba_registration(mocker, app, session, jwt): with patch.object(flags, 'is_on', return_value=True): with patch('legal_api.services.filings.validations.common_validations.Business.find_by_identifier', return_value=None): with patch('legal_api.services.filings.validations.common_validations.colin.query_business') as mock_colin: - mock_response = type('Response', (), {'status_code': HTTPStatus.NOT_FOUND})() - mock_colin.return_value = mock_response + mock_colin.return_value = (None, HTTPStatus.NOT_FOUND) with patch('legal_api.services.filings.validations.common_validations.PermissionService.check_user_permission', return_value=None): with jwt_request_context(app, jwt, [BASIC_USER]): err = validate(None, DBA_REGISTRATION) @@ -335,8 +334,7 @@ def test_invalid_business_address(app, session, jwt, test_name, filing): with patch.object(NaicsService, 'find_by_code', return_value=naics_response): with patch('legal_api.services.filings.validations.common_validations.Business.find_by_identifier', return_value=None): with patch('legal_api.services.filings.validations.common_validations.colin.query_business') as mock_colin: - mock_response = type('Response', (), {'status_code': HTTPStatus.NOT_FOUND})() - mock_colin.return_value = mock_response + mock_colin.return_value = (None, HTTPStatus.NOT_FOUND) with patch('legal_api.services.filings.validations.common_validations.PermissionService.check_user_permission', return_value=None): with jwt_request_context(app, jwt, [BASIC_USER]): err = validate(None, filing) @@ -443,4 +441,57 @@ def mock_check(permissions_msg, filing_type): permission_error = msg break assert permission_error is not None, "Permission error not found in validation messages." - \ No newline at end of file + + +MISMATCH_ERROR = 'Completing party name must match the name of the logged in user.' + + +@pytest.mark.parametrize( + 'test_name, roles, login_source, user_name, token_name, expected_error', [ + ('client_name_matches', [BASIC_USER], 'BCSC', ('Joe', 'P', 'Swanson'), None, None), + ('client_long_name_matches', [BASIC_USER], 'BCSC', + ('Josephakis Vasilliadopolous Constantinople', None, 'Swanson'), None, None), + ('client_name_mismatch', [BASIC_USER], 'BCSC', ('Different', None, 'Person'), None, MISMATCH_ERROR), + ('client_first_filing_user_created_from_token', [BASIC_USER], 'BCSC', None, ('Joe P', 'Swanson'), None), + ('client_first_filing_token_name_mismatch', [BASIC_USER], 'BCSC', None, ('Different', 'Person'), + MISMATCH_ERROR), + ('client_user_record_no_name', [BASIC_USER], 'BCSC', (None, None, None), None, MISMATCH_ERROR), + ('staff_skipped', [STAFF_ROLE], 'IDIR', ('Different', None, 'Person'), None, None), + ('api_gw_skipped', [BASIC_USER], 'API_GW', ('Different', None, 'Person'), None, None), + ] +) +def test_validate_completing_party_name(app, session, jwt, test_name, roles, login_source, user_name, token_name, + expected_error): + """Assert the completing party name is validated against the user record for client filings.""" + filing = copy.deepcopy(GP_REGISTRATION) + if test_name == 'client_long_name_matches': + filing['filing']['registration']['parties'][0]['officer']['firstName'] = \ + 'Josephakis Vasilliadopolous Constantinople' + filing['filing']['registration']['parties'][0]['officer']['middleName'] = '' + + if user_name: + firstname, middlename, lastname = user_name + user = User(username='cp-test-user', firstname=firstname, middlename=middlename, lastname=lastname, + sub='sub', iss='iss', idp_userid='123', login_source=login_source) + user.save() + + extra_claims = {'firstname': token_name[0], 'lastname': token_name[1]} if token_name else None + + naics_response = { + 'code': REGISTRATION['business']['naics']['naicsCode'], + 'classTitle': REGISTRATION['business']['naics']['naicsDescription'] + } + with patch.object(NameXService, 'query_nr_number', return_value=_mock_nr_response('GP')): + with patch.object(NaicsService, 'find_by_code', return_value=naics_response): + with jwt_request_context(app, jwt, roles, login_source=login_source, extra_claims=extra_claims): + err = validate(None, filing) + + if token_name: + # a first time filer has no user record yet; one is created from the JWT during validation + assert User.query.filter_by(idp_userid='123').one_or_none() is not None + + if expected_error: + assert err is not None + assert any(msg['error'] == expected_error for msg in err.msg) + else: + assert err is None or all(msg['error'] != MISMATCH_ERROR for msg in err.msg) diff --git a/legal-api/tests/unit/services/filings/validations/test_restoration.py b/legal-api/tests/unit/services/filings/validations/test_restoration.py index bd034c6526..e0deb8f51d 100644 --- a/legal-api/tests/unit/services/filings/validations/test_restoration.py +++ b/legal-api/tests/unit/services/filings/validations/test_restoration.py @@ -23,9 +23,10 @@ from business_common.utils.legislation_datetime import LegislationDatetime from business_model.models import Business, Filing from legal_api.services.filings.validations.validation import validate -from registry_schemas.example_data import FILING_HEADER, RESTORATION +from registry_schemas.example_data import RESTORATION, get_filing_template -from tests.unit.services.filings.validations import lists_are_equal + +TEST_IDENTIFIER = 'BC1234567' date_format = '%Y-%m-%d' now = datetime.now().strftime(date_format) @@ -72,8 +73,8 @@ def execute_test_restoration_nr(mocker, filing_sub_type, legal_type, nr_number, expiry_date_str = expiry_date.strftime(date_format) mocker.patch('legal_api.services.NameXService.validate_nr', return_value=validate_nr_result) - business = Business(identifier='BC1234567', legal_type=legal_type, restoration_expiry_date=LegislationDatetime.now()) - filing = copy.deepcopy(FILING_HEADER) + business = Business(identifier=TEST_IDENTIFIER, legal_type=legal_type, restoration_expiry_date=LegislationDatetime.now()) + filing = get_filing_template('restoration', TEST_IDENTIFIER) filing['filing']['restoration'] = copy.deepcopy(RESTORATION) filing['filing']['restoration']['type'] = filing_sub_type filing['filing']['restoration']['expiry'] = expiry_date_str @@ -133,8 +134,8 @@ def json(self): ) def test_validate_party(session, test_name, party_role, expected_msg): """Assert that party is validated.""" - business = Business(identifier='BC1234567', legal_type='BC') - filing = copy.deepcopy(FILING_HEADER) + business = Business(identifier=TEST_IDENTIFIER, legal_type='BC') + filing = get_filing_template('restoration', TEST_IDENTIFIER) filing['filing']['restoration'] = copy.deepcopy(RESTORATION) filing['filing']['header']['name'] = 'restoration' filing['filing']['restoration']['relationships'] = relationships @@ -170,11 +171,11 @@ def test_validate_party(session, test_name, party_role, expected_msg): ) def test_validate_relationship(session, test_status, restoration_type, expected_code, expected_msg): """Assert that applicant's relationship is validated.""" - business = Business(identifier='BC1234567', legal_type='BC', restoration_expiry_date=LegislationDatetime.now()) + business = Business(identifier=TEST_IDENTIFIER, legal_type='BC', restoration_expiry_date=LegislationDatetime.now()) limited_restoration_filing = None if restoration_type in ('limitedRestorationExtension', 'limitedRestorationToFull'): limited_restoration_filing = factory_limited_restoration_filing() - filing = copy.deepcopy(FILING_HEADER) + filing = get_filing_template('restoration', TEST_IDENTIFIER) filing['filing']['restoration'] = copy.deepcopy(RESTORATION) filing['filing']['header']['name'] = 'restoration' filing['filing']['restoration']['type'] = restoration_type @@ -228,9 +229,9 @@ def test_validate_expiry_date(session, test_name, restoration_type, delta_date, expiry_date = restoration_expiry_date if delta_date: expiry_date = expiry_date + delta_date - business = Business(identifier='BC1234567', legal_type='BC', restoration_expiry_date=restoration_expiry_date) + business = Business(identifier=TEST_IDENTIFIER, legal_type='BC', restoration_expiry_date=restoration_expiry_date) - filing = copy.deepcopy(FILING_HEADER) + filing = get_filing_template('restoration', TEST_IDENTIFIER) filing['filing']['restoration'] = copy.deepcopy(RESTORATION) filing['filing']['header']['name'] = 'restoration' if restoration_type == 'limitedRestorationExtension': @@ -278,8 +279,8 @@ def test_approval_type(session, test_status, restoration_types, legal_types, app if restoration_type in ('limitedRestorationExtension', 'limitedRestorationToFull'): limited_restoration_filing = factory_limited_restoration_filing(limited_restoration_approval_type) for legal_type in legal_types: - business = Business(identifier='BC1234567', legal_type=legal_type, restoration_expiry_date=LegislationDatetime.now()) - filing = copy.deepcopy(FILING_HEADER) + business = Business(identifier=TEST_IDENTIFIER, legal_type=legal_type, restoration_expiry_date=LegislationDatetime.now()) + filing = get_filing_template('restoration', TEST_IDENTIFIER) filing['filing']['restoration'] = copy.deepcopy(RESTORATION) filing['filing']['header']['name'] = 'restoration' filing['filing']['restoration']['type'] = restoration_type @@ -333,8 +334,8 @@ def test_restoration_court_orders(session, test_status, restoration_types, legal if restoration_type in ('limitedRestorationExtension', 'limitedRestorationToFull'): limited_restoration_filing = factory_limited_restoration_filing(limited_restoration_approval_type) for legal_type in legal_types: - business = Business(identifier='BC1234567', legal_type=legal_type, restoration_expiry_date=LegislationDatetime.now()) - filing = copy.deepcopy(FILING_HEADER) + business = Business(identifier=TEST_IDENTIFIER, legal_type=legal_type, restoration_expiry_date=LegislationDatetime.now()) + filing = get_filing_template('restoration', TEST_IDENTIFIER) filing['filing']['restoration'] = copy.deepcopy(RESTORATION) filing['filing']['header']['name'] = 'restoration' filing['filing']['restoration']['type'] = restoration_type @@ -395,8 +396,8 @@ def test_restoration_registrar(session, test_status, restoration_types, legal_ty if restoration_type in ('limitedRestorationExtension', 'limitedRestorationToFull'): limited_restoration_filing = factory_limited_restoration_filing(limited_restoration_approval_type) for legal_type in legal_types: - business = Business(identifier='BC1234567', legal_type=legal_type, restoration_expiry_date=LegislationDatetime.now()) - filing = copy.deepcopy(FILING_HEADER) + business = Business(identifier=TEST_IDENTIFIER, legal_type=legal_type, restoration_expiry_date=LegislationDatetime.now()) + filing = get_filing_template('restoration', TEST_IDENTIFIER) filing['filing']['restoration'] = copy.deepcopy(RESTORATION) filing['filing']['header']['name'] = 'restoration' filing['filing']['restoration']['type'] = restoration_type @@ -523,12 +524,12 @@ def test_restoration_nr_type(session, mocker, test_status, filing_sub_type, lega ) def test_validate_contact_point(session, test_status, restoration_type, has_contact_point, expected_code, expected_msg): """Assert that contact point is not allowed for limited restoration filings.""" - business = Business(identifier='BC1234567', legal_type='BC', restoration_expiry_date=LegislationDatetime.now()) + business = Business(identifier=TEST_IDENTIFIER, legal_type='BC', restoration_expiry_date=LegislationDatetime.now()) limited_restoration_filing = None if restoration_type == 'limitedRestorationExtension': limited_restoration_filing = factory_limited_restoration_filing() - filing = copy.deepcopy(FILING_HEADER) + filing = get_filing_template('restoration', TEST_IDENTIFIER) filing['filing']['restoration'] = copy.deepcopy(RESTORATION) filing['filing']['header']['name'] = 'restoration' filing['filing']['restoration']['type'] = restoration_type diff --git a/legal-api/tests/unit/services/filings/validations/test_special_resolution.py b/legal-api/tests/unit/services/filings/validations/test_special_resolution.py index 2e3edaa540..88378f5980 100644 --- a/legal-api/tests/unit/services/filings/validations/test_special_resolution.py +++ b/legal-api/tests/unit/services/filings/validations/test_special_resolution.py @@ -13,7 +13,7 @@ # limitations under the License. """Test suite to ensure Special Resolution is validated correctly.""" import copy -from datetime import datetime +from datetime import datetime, timezone from http import HTTPStatus import pytest @@ -27,7 +27,7 @@ @pytest.mark.parametrize( 'test_name, resolution, identifier, resolution_date, signing_date, signatory_given_name, signatory_family_name, business_founding_date, expected_code, expected_msg', [ - ('SUCCESS', 'some resolution', 'CP1234567', '2021-01-10', '2021-01-10', 'jane', 'doe', '2010-01-10', None, None), + ('SUCCESS - resolution date before UTC founding date', 'some resolution', 'CP1234567', '2010-01-09', '2010-01-09', 'jane', 'doe', '2010-01-10', None, None), ('MISSING - resolution', None, 'CP1234567', '2021-01-10', '2021-01-10', 'jane', 'doe', '2010-01-10', HTTPStatus.BAD_REQUEST, 'Resolution must be provided.'), ('MISSING - resolution date', 'some resolution', 'CP1234567', None, '2021-01-10', 'jane', 'doe', '2010-01-10', @@ -61,7 +61,10 @@ def test_validate(session, test_name, resolution, identifier, resolution_date, s """Assert that a SR can be validated.""" # setup business = Business(identifier=identifier) - business.founding_date = datetime.strptime(business_founding_date, '%Y-%m-%d') + business.founding_date = datetime.strptime( + business_founding_date, + '%Y-%m-%d' + ).replace(tzinfo=timezone.utc) filing = copy.deepcopy(FILING_HEADER) filing['filing']['specialResolution'] = copy.deepcopy(SPECIAL_RESOLUTION) diff --git a/legal-api/tests/unit/services/filings/validations/test_validation.py b/legal-api/tests/unit/services/filings/validations/test_validation.py index d1f4928e54..1f4b5531e1 100644 --- a/legal-api/tests/unit/services/filings/validations/test_validation.py +++ b/legal-api/tests/unit/services/filings/validations/test_validation.py @@ -100,7 +100,7 @@ def test_validate_coa_basic(session, test_name, now, delivery_region, delivery_c f = copy.deepcopy(FILING_HEADER) f['filing']['header']['date'] = now.isoformat() - f['filing']['header']['name'] = 'changeOfDirectors' + f['filing']['header']['name'] = 'changeOfAddress' f['filing']['business']['identifier'] = identifier f['filing']['changeOfAddress'] = CHANGE_OF_ADDRESS office = f['filing']['changeOfAddress']['offices']['registeredOffice'] @@ -113,6 +113,9 @@ def test_validate_coa_basic(session, test_name, now, delivery_region, delivery_c err = validate(business, f) # validate outcomes + if err: + print(err.code) + print(err.msg) if expected_code: assert err.code == expected_code assert lists_are_equal(err.msg, expected_msg) diff --git a/legal-api/tests/unit/services/test_authorization.py b/legal-api/tests/unit/services/test_authorization.py index 4f1c7ca918..889df6006f 100644 --- a/legal-api/tests/unit/services/test_authorization.py +++ b/legal-api/tests/unit/services/test_authorization.py @@ -545,6 +545,21 @@ def test_authorized_missing_args(): assert not rv +def test_call_auth_api_forwards_account_linking_key(app, jwt, requests_mock): + """Assert that _call_auth_api forwards the Account-Linking-Key header when present on the request.""" + identifier = 'CP1234567' + with jwt_request_context(app, jwt, roles=[BASIC_USER], username='test-user', + **{'Account-Linking-Key': 'test-linking-key'}): + auth_mock = requests_mock.get( + f"{app.config['AUTH_SVC_URL']}/entities/{identifier}/authorizations", + json={'roles': ['view']}, + status_code=HTTPStatus.OK + ) + authorized(identifier, jwt, ['view']) + + assert auth_mock.last_request.headers.get('Account-Linking-Key') == 'test-linking-key' + + def test_authorized_bad_url(monkeypatch, app, jwt): """Assert that an invalid auth service URL returns False.""" identifier = 'CP1234567' @@ -642,10 +657,10 @@ def test_authorized_invalid_roles(monkeypatch, app, jwt): ('staff_historical_cp', Business.State.HISTORICAL, ['CP'], 'staff', [STAFF_ROLE], ['courtOrder', 'putBackOn', 'registrarsNotation', 'registrarsOrder']), ('staff_historical_corps', Business.State.HISTORICAL, ['BC', 'BEN', 'CC', 'ULC'], 'staff', [STAFF_ROLE], - ['courtOrder', 'putBackOn', 'registrarsNotation', 'registrarsOrder', + ['correction', 'courtOrder', 'putBackOn', 'registrarsNotation', 'registrarsOrder', {'restoration': ['fullRestoration', 'limitedRestoration']}]), ('staff_historical_continue_in_corps', Business.State.HISTORICAL, ['C', 'CBEN', 'CUL', 'CCC'], 'staff', [STAFF_ROLE], - ['courtOrder', 'putBackOn', 'registrarsNotation', 'registrarsOrder', + ['correction', 'courtOrder', 'putBackOn', 'registrarsNotation', 'registrarsOrder', {'restoration': ['fullRestoration', 'limitedRestoration']}]), ('staff_historical_llc', Business.State.HISTORICAL, ['LLC'], 'staff', [STAFF_ROLE], []), ('staff_historical_firms', Business.State.HISTORICAL, ['SP', 'GP'], 'staff', [STAFF_ROLE], @@ -878,7 +893,10 @@ def test_get_allowed(monkeypatch, app, jwt, test_name, state, legal_types, usern ['CP', 'BC', 'BEN', 'CC', 'ULC', 'LLC', 'C', 'CBEN', 'CUL', 'CCC'], 'staff', [STAFF_ROLE], False), ('staff_historical', Business.State.HISTORICAL, 'correction', None, - ['CP', 'BC', 'BEN', 'CC', 'ULC', 'LLC', 'C', 'CBEN', 'CUL', 'CCC'], 'staff', [STAFF_ROLE], False), + ['CP', 'LLC', 'SP', 'GP'], 'staff', [STAFF_ROLE], False), + + ('staff_historical_allowed', Business.State.HISTORICAL, 'correction', None, + ['BC', 'BEN', 'CC', 'ULC', 'C', 'CBEN', 'CUL', 'CCC'], 'staff', [STAFF_ROLE], True), ('staff_historical_allowed', Business.State.HISTORICAL, 'courtOrder', None, ['SP', 'GP', 'CP', 'BC', 'BEN', 'CC', 'ULC', 'C', 'CBEN', 'CUL', 'CCC'], 'staff', [STAFF_ROLE], True), @@ -1140,7 +1158,8 @@ def test_is_allowed(monkeypatch, app, session, jwt, test_name, state, filing_typ FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER])), ('staff_historical_corps', True, Business.State.HISTORICAL, ['BC', 'BEN', 'CC', 'ULC'], 'staff', [STAFF_ROLE], - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER, @@ -1148,7 +1167,8 @@ def test_is_allowed(monkeypatch, app, session, jwt, test_name, state, filing_typ FilingKey.RESTRN_LTD_CORPS])), ('staff_historical_continue_in_corps', True, Business.State.HISTORICAL, ['C', 'CBEN', 'CCC', 'CUL'], 'staff', [STAFF_ROLE], - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER, @@ -1475,7 +1495,8 @@ def test_get_allowed_actions(monkeypatch, app, session, jwt, requests_mock, FilingKey.REGISTRARS_ORDER])), ('staff_historical_corps', True, Business.State.HISTORICAL, ['BC', 'BEN', 'CC', 'ULC', 'C', 'CBEN', 'CCC', 'CUL'], 'staff', [STAFF_ROLE], - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER, @@ -1637,13 +1658,15 @@ def test_get_allowed_filings_blocker_admin_freeze(monkeypatch, app, session, jwt FilingKey.REGISTRARS_ORDER]), ), ('staff_historical_corps', True, Business.State.HISTORICAL, ['BC', 'BEN', 'CC', 'ULC'], 'staff', [STAFF_ROLE], - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER])), ('staff_historical_continue_in_corps', True, Business.State.HISTORICAL, ['C', 'CBEN', 'CCC', 'CUL'], 'staff', [STAFF_ROLE], - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER])), @@ -1800,7 +1823,8 @@ def test_get_allowed_filings_blocker_for_amalgamating_business(monkeypatch, app, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER])), ('staff_historical_corps', True, Business.State.HISTORICAL, ['BC', 'BEN', 'CC', 'ULC'], 'staff', [STAFF_ROLE], - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER, @@ -1808,7 +1832,8 @@ def test_get_allowed_filings_blocker_for_amalgamating_business(monkeypatch, app, FilingKey.RESTRN_LTD_CORPS])), ('staff_historical_continue_in_corps', True, Business.State.HISTORICAL, ['C', 'CBEN', 'CCC', 'CUL'], 'staff', [STAFF_ROLE], - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER, @@ -2314,7 +2339,8 @@ def test_allowed_filings_blocker_filing_amalgamations(monkeypatch, app, session, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER])), ('staff_historical_corps', Business.State.HISTORICAL, ['BC', 'BEN', 'CC', 'ULC'], 'staff', [STAFF_ROLE], - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER, @@ -2322,7 +2348,8 @@ def test_allowed_filings_blocker_filing_amalgamations(monkeypatch, app, session, FilingKey.RESTRN_LTD_CORPS])), ('staff_historical_continue_in_corps', Business.State.HISTORICAL, ['C', 'CBEN', 'CCC', 'CUL'], 'staff', [STAFF_ROLE], - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER, @@ -2638,7 +2665,8 @@ def test_allowed_filings_warnings(monkeypatch, app, session, jwt, test_name, sta FilingKey.REGISTRARS_ORDER])), ('staff_historical_corps_unaffected', Business.State.HISTORICAL, ['BC', 'BEN', 'CC', 'ULC'], 'staff', [STAFF_ROLE], ['dissolution', None], [None, None], - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER, @@ -2646,7 +2674,8 @@ def test_allowed_filings_warnings(monkeypatch, app, session, jwt, test_name, sta FilingKey.RESTRN_LTD_CORPS])), ('staff_historical_continue_in_corps_unaffected', Business.State.HISTORICAL, ['C', 'CBEN', 'CCC', 'CUL'], 'staff', [STAFF_ROLE], ['dissolution', None], [None, None], - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER, @@ -2654,13 +2683,15 @@ def test_allowed_filings_warnings(monkeypatch, app, session, jwt, test_name, sta FilingKey.RESTRN_LTD_CORPS])), ('staff_historical_corps_invalid_state_filing_fail', Business.State.HISTORICAL, ['BC', 'BEN', 'CC', 'ULC'], 'staff', [STAFF_ROLE], ['amalgamationOut', 'continuationOut'], [None, None], - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER])), ('staff_historical_continue_in_corps_invalid_state_filing_fail', Business.State.HISTORICAL, ['C', 'CBEN', 'CCC', 'CUL'], 'staff', [STAFF_ROLE], ['amalgamationOut', 'continuationOut'], [None, None], - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER])), @@ -3091,7 +3122,8 @@ def test_allowed_filings_completed_filing_check(monkeypatch, app, session, jwt, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER])), ('staff_historical_corps', True, Business.State.HISTORICAL, ['BC', 'BEN', 'CC', 'ULC'], 'staff', [STAFF_ROLE], 0, - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER, @@ -3287,7 +3319,8 @@ def test_get_allowed_filings_blocker_in_dissolution(monkeypatch, app, session, j # historical business - staff user ('staff_historical_corps', Business.State.HISTORICAL, ['BC', 'BEN', 'CC', 'ULC'], 'staff', [STAFF_ROLE], None, - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER, @@ -3295,7 +3328,8 @@ def test_get_allowed_filings_blocker_in_dissolution(monkeypatch, app, session, j FilingKey.RESTRN_LTD_CORPS])), ('staff_historical_continue_in_corps', Business.State.HISTORICAL, ['C', 'CBEN', 'CCC', 'CUL'], 'staff', [STAFF_ROLE], None, - expected_lookup([FilingKey.COURT_ORDER, + expected_lookup([FilingKey.CORRCTN, + FilingKey.COURT_ORDER, FilingKey.PUT_BACK_ON, FilingKey.REGISTRARS_NOTATION, FilingKey.REGISTRARS_ORDER, diff --git a/legal-api/tests/unit/services/test_colin.py b/legal-api/tests/unit/services/test_colin.py new file mode 100644 index 0000000000..c177f64ac4 --- /dev/null +++ b/legal-api/tests/unit/services/test_colin.py @@ -0,0 +1,184 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests to assure the colin service.""" +from http import HTTPStatus + +import pytest +from requests import exceptions + +from legal_api.services.cache import cache +from legal_api.services.colin import ColinService + + +PUBLIC_PATH = 'businesses/BC0870226/public' + + +@pytest.fixture(autouse=True) +def _clear_colin_cache(app): + """Keep cached colin responses from leaking between tests.""" + cache.clear() + + +@pytest.fixture +def system_token(mocker): + """Return the mocked system service token fetch.""" + return mocker.patch('legal_api.services.colin.AccountService.get_bearer_token', return_value='system-token') + + +def test_call_colin_api(app, requests_mock, system_token): + """Assert the colin api is called with the configured url and the system token.""" + colin_mock = requests_mock.get(f'{app.config["COLIN_URL"]}/{PUBLIC_PATH}', json={'business': {}}) + + json_data, status_code = ColinService.call_colin_api(PUBLIC_PATH) + + assert status_code == HTTPStatus.OK + assert json_data == {'business': {}} + assert colin_mock.last_request.headers['Authorization'] == 'Bearer system-token' + + +def test_call_colin_api_with_given_token(app, requests_mock, system_token): + """Assert a given token is used as-is and the system token is not fetched.""" + colin_mock = requests_mock.get(f'{app.config["COLIN_URL"]}/{PUBLIC_PATH}', json={'business': {}}) + + _, status_code = ColinService.call_colin_api(PUBLIC_PATH, token='user-token') + + assert status_code == HTTPStatus.OK + assert colin_mock.last_request.headers['Authorization'] == 'Bearer user-token' + system_token.assert_not_called() + + +def test_call_colin_api_without_token(app, requests_mock, mocker): + """Assert no call is made when a token could not be fetched.""" + colin_mock = requests_mock.get(f'{app.config["COLIN_URL"]}/{PUBLIC_PATH}', json={'business': {}}) + mocker.patch('legal_api.services.colin.AccountService.get_bearer_token', return_value=None) + + assert ColinService.call_colin_api(PUBLIC_PATH) == (None, None) + assert not colin_mock.called + + +def test_call_colin_api_without_colin_url(app, requests_mock, system_token, mocker): + """Assert an unconfigured COLIN_URL returns (None, None) instead of raising.""" + colin_mock = requests_mock.get(f'{app.config["COLIN_URL"]}/{PUBLIC_PATH}', json={'business': {}}) + mocker.patch.dict(app.config, {'COLIN_URL': ''}) + + assert ColinService.call_colin_api(PUBLIC_PATH) == (None, None) + assert not colin_mock.called + + +def test_call_colin_api_non_json_body(app, requests_mock, system_token): + """Assert a non-JSON body returns (None, status) and is never cached.""" + colin_mock = requests_mock.get(f'{app.config["COLIN_URL"]}/{PUBLIC_PATH}', text='bad gateway') + + assert ColinService.call_colin_api(PUBLIC_PATH) == (None, HTTPStatus.OK) + assert ColinService.call_colin_api(PUBLIC_PATH) == (None, HTTPStatus.OK) + + assert colin_mock.call_count == 2 + + +def test_call_colin_api_token_failure(app, mocker): + """Assert a token-service outage returns (None, None) instead of raising.""" + mocker.patch('legal_api.services.colin.AccountService.get_bearer_token', + side_effect=exceptions.ConnectionError('keycloak down')) + + assert ColinService.call_colin_api(PUBLIC_PATH) == (None, None) + + +def test_call_colin_api_connection_failure(app, requests_mock, system_token): + """Assert a colin connection failure returns (None, None) instead of raising.""" + requests_mock.get(f'{app.config["COLIN_URL"]}/{PUBLIC_PATH}', exc=exceptions.ConnectTimeout) + + assert ColinService.call_colin_api(PUBLIC_PATH) == (None, None) + + +def test_call_colin_api_caches_definitive_responses(app, requests_mock, system_token): + """Assert responses are cached per path - repeat calls don't hit colin again.""" + public_mock = requests_mock.get(f'{app.config["COLIN_URL"]}/{PUBLIC_PATH}', json={'business': {}}) + missing_mock = requests_mock.get(f'{app.config["COLIN_URL"]}/businesses/BC0000000/public', + status_code=HTTPStatus.NOT_FOUND, json={'message': 'not found'}) + + assert ColinService.call_colin_api(PUBLIC_PATH) == ({'business': {}}, HTTPStatus.OK) + assert ColinService.call_colin_api(PUBLIC_PATH) == ({'business': {}}, HTTPStatus.OK) + # a 404 is a definitive answer too + assert ColinService.call_colin_api('businesses/BC0000000/public')[1] == HTTPStatus.NOT_FOUND + assert ColinService.call_colin_api('businesses/BC0000000/public')[1] == HTTPStatus.NOT_FOUND + + assert public_mock.call_count == 1 + assert missing_mock.call_count == 1 + + +def test_call_colin_api_caches_per_token(app, requests_mock, system_token): + """Assert a response cached under one token is not served to a call with another.""" + colin_mock = requests_mock.get(f'{app.config["COLIN_URL"]}/{PUBLIC_PATH}', json={'business': {}}) + + ColinService.call_colin_api(PUBLIC_PATH) + ColinService.call_colin_api(PUBLIC_PATH, token='user-token') + ColinService.call_colin_api(PUBLIC_PATH, token='user-token') + + assert colin_mock.call_count == 2 + + +def test_call_colin_api_does_not_cache_failures(app, requests_mock, system_token): + """Assert an unavailable COLIN (5xx) is retried on the next call.""" + colin_mock = requests_mock.get(f'{app.config["COLIN_URL"]}/{PUBLIC_PATH}', + status_code=HTTPStatus.INTERNAL_SERVER_ERROR) + + assert ColinService.call_colin_api(PUBLIC_PATH)[1] == HTTPStatus.INTERNAL_SERVER_ERROR + assert ColinService.call_colin_api(PUBLIC_PATH)[1] == HTTPStatus.INTERNAL_SERVER_ERROR + + assert colin_mock.call_count == 2 + + +def test_call_colin_api_cache_opt_out(app, requests_mock, system_token): + """Assert use_cache=False bypasses the cache entirely - no stale reads, no writes.""" + colin_mock = requests_mock.get(f'{app.config["COLIN_URL"]}/{PUBLIC_PATH}', json={'business': {}}) + + ColinService.call_colin_api(PUBLIC_PATH) # populates the cache + ColinService.call_colin_api(PUBLIC_PATH, use_cache=False) + ColinService.call_colin_api(PUBLIC_PATH, use_cache=False) + + assert colin_mock.call_count == 3 + + +def test_query_business(app, requests_mock, system_token): + """Assert query_business goes through call_colin_api to the public endpoint.""" + colin_mock = requests_mock.get(f'{app.config["COLIN_URL"]}/{PUBLIC_PATH}', json={'business': {}}) + + json_data, status_code = ColinService.query_business('BC0870226') + + assert status_code == HTTPStatus.OK + assert json_data == {'business': {}} + assert colin_mock.last_request.headers['Authorization'] == 'Bearer system-token' + + # repeat calls are served from the cache unless the caller opts out + ColinService.query_business('BC0870226') + assert colin_mock.call_count == 1 + ColinService.query_business('BC0870226', use_cache=False) + assert colin_mock.call_count == 2 + + +def test_get_snapshot(app, requests_mock, system_token): + """Assert get_snapshot goes through call_colin_api to the snapshot endpoint, with caching.""" + snapshot_mock = requests_mock.get(f'{app.config["COLIN_URL"]}/businesses/BC0870226/snapshot', + json={'business': {'identifier': 'BC0870226'}}) + + json_data, status_code = ColinService.get_snapshot('BC0870226') + + assert status_code == HTTPStatus.OK + assert json_data['business']['identifier'] == 'BC0870226' + # repeat calls are served from the cache + assert ColinService.get_snapshot('BC0870226')[1] == HTTPStatus.OK + assert snapshot_mock.call_count == 1 + # but a caller can opt out of the cache to force a fresh fetch + assert ColinService.get_snapshot('BC0870226', use_cache=False)[1] == HTTPStatus.OK + assert snapshot_mock.call_count == 2 diff --git a/legal-api/tests/unit/services/test_doc.pdf b/legal-api/tests/unit/services/test_doc.pdf new file mode 100644 index 0000000000..25a98bd463 Binary files /dev/null and b/legal-api/tests/unit/services/test_doc.pdf differ diff --git a/legal-api/tests/unit/services/test_doc_service.py b/legal-api/tests/unit/services/test_doc_service.py new file mode 100644 index 0000000000..5290a38517 --- /dev/null +++ b/legal-api/tests/unit/services/test_doc_service.py @@ -0,0 +1,198 @@ +# Copyright © 2019 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests to verify the DRS API integration. + +Test-Suite to ensure that the client for the DRS API service is working as expected. +""" +import pytest +from flask import current_app + +from business_model.models import Document +from legal_api.services import doc_service + + +MOCK_DRS_URL = "https://test.api.connect.gov.bc.ca/mockTarget/doc/api/v1" +TEST_DATAFILE = 'tests/unit/services/test_doc.pdf' +PATCH_VALID_PAYLOAD = { + "consumerIdentifier": "BC9901019", + "consumerReferenceId": "1199642", + "consumerFilingDate": "2024-08-02T19:00:00+00:00" +} + +# testdata pattern is ({description}, {doc_class}, {drs_id}) +TEST_GET_DATA = [ + ("Valid DRS id", "CORP", "DS0100001000"), +] +# testdata pattern is ({description}, {doc_class}, {doc_type}) +TEST_POST_DATA = [ + ("Valid Request", "CORP", "CORR"), +] +# testdata pattern is ({description}, {drs_id}) +TEST_PUT_DATA = [ + ("Valid Request", "DS0100001000"), +] +# testdata pattern is ({description}, {drs_id}, {payload}) +TEST_PATCH_DATA = [ + ("Valid Request", "DS0100001003", PATCH_VALID_PAYLOAD), +] +TEST_DELETE_DATA = [ + ("Valid Request", "DS0100001000"), +] +# testdata pattern is ({filename}, {bus_identifier}, {filing_id}, {filing_date}, {result}) +TEST_CREATE_URL_DATA = [ + (None, None, None, None,""), + ("filename.pdf", "BC2000000", "2999999", "2024-08-01", + "?consumerFilename=filename.pdf&consumerIdentifier=BC2000000&consumerReferenceId=2999999&consumerFilingDate=2024-08-01"), + ("filename.pdf", "BC2000000", None, None, "?consumerFilename=filename.pdf&consumerIdentifier=BC2000000"), + ("filename.pdf", None, None, None, "?consumerFilename=filename.pdf"), + (None, "BC2000000", None, None, "?consumerIdentifier=BC2000000"), +] +# testdata pattern is ({filekey}, {drs_id}) +TEST_DRS_ID_DATA = [ + (None, None), + ("DS0100001003", "DS0100001003"), + ("CORP-DS0100001003", "DS0100001003"), + ("COOP-DS0100001003", "DS0100001003"), + ("FIRM-DS0100001003", "DS0100001003"), +] + +@pytest.mark.parametrize('desc,doc_class,doc_type', TEST_POST_DATA) +def test_create_doc_record(session, jwt, desc, doc_class, doc_type): + """Assert that doc service create document records works as expected.""" + # setup + current_app.config.update(DOCUMENT_SVC_URL=MOCK_DRS_URL) + doc_info: dict = { + "documentClass": doc_class, + "documentType": doc_type + } + raw_data = None + with open(TEST_DATAFILE, 'rb') as data_file: + raw_data = data_file.read() + data_file.close() + + response = doc_service.create_document(doc_info, raw_data) + # check + assert response + assert response.ok + assert response.content + result = doc_service.get_content(response) + assert result + assert result.get("documentClass") + assert result.get("documentType") + assert result.get("documentServiceId") + assert result.get("documentURL") + + +@pytest.mark.parametrize('desc,drs_id', TEST_PUT_DATA) +def test_add_replace_document(session, jwt, desc, drs_id): + """Assert that doc service add or replace a document works as expected.""" + # setup + current_app.config.update(DOCUMENT_SVC_URL=MOCK_DRS_URL) + doc: Document = Document(file_key=drs_id) + raw_data = None + with open(TEST_DATAFILE, 'rb') as data_file: + raw_data = data_file.read() + data_file.close() + + response = doc_service.add_replace_document(doc, raw_data) + # check + assert response + assert response.ok + assert response.content + result = doc_service.get_content(response) + assert result + assert result.get("documentClass") + assert result.get("documentType") + assert result.get("documentServiceId") + assert result.get("documentURL") + + +@pytest.mark.parametrize('desc,drs_id,payload', TEST_PATCH_DATA) +def test_update_doc_record(session, jwt, desc, drs_id, payload): + """Assert that doc service update a document record works as expected.""" + # setup + current_app.config.update(DOCUMENT_SVC_URL=MOCK_DRS_URL) + doc: Document = Document(file_key=drs_id) + + response = doc_service.update_document_record(doc, payload) + # check + assert response + assert response.ok + assert response.content + result = doc_service.get_content(response) + assert result + assert result.get("documentClass") + assert result.get("documentType") + assert result.get("documentServiceId") + + +@pytest.mark.parametrize('desc,drs_id', TEST_DELETE_DATA) +def test_delete_document(session, jwt, desc, drs_id): + """Assert that doc service permanently delete a document works as expected.""" + # setup + current_app.config.update(DOCUMENT_SVC_URL=MOCK_DRS_URL) + doc: Document = Document(file_key=drs_id) + response = doc_service.delete_document(doc) + # check + assert response + assert response.ok + + +@pytest.mark.parametrize('desc,doc_class,drs_id', TEST_GET_DATA) +def test_get_doc(session, jwt, desc, doc_class, drs_id): + """Assert that doc service DRS id search returns the expected result.""" + # setup + current_app.config.update(DOCUMENT_SVC_URL=MOCK_DRS_URL) + + response = doc_service.get_document(drs_id, doc_class, False) + # check + assert response + assert response.ok + assert response.content + result = doc_service.get_content(response) + assert result + assert result[0].get("documentClass") + assert result[0].get("documentType") + assert result[0].get("documentServiceId") + + +@pytest.mark.parametrize('filename,bus_id,filing_id,filing_date,result', TEST_CREATE_URL_DATA) +def test_create_doc_url(session, jwt, filename, bus_id, filing_id, filing_date, result): + """Assert that doc service build create record url returns the expected result.""" + # setup + current_app.config.update(DOCUMENT_SVC_URL=MOCK_DRS_URL) + url = doc_service.POST_DOCUMENT_PATH.format(url=MOCK_DRS_URL,document_class="CORP",document_type="CORR") + result + doc_info: dict = { + "documentClass": "CORP", + "documentType": "CORR" + } + if filename: + doc_info["consumerFilename"] = filename + if bus_id: + doc_info["consumerIdentifier"] = bus_id + if filing_id: + doc_info["consumerReferenceId"] = filing_id + if filing_date: + doc_info["consumerFilingDate"] = filing_date + result_url = doc_service.build_create_doc_url("CORP", "CORR", doc_info) + assert url == result_url + + +@pytest.mark.parametrize('filekey,drs_id', TEST_DRS_ID_DATA) +def test_get_drs_id(session, jwt, filekey, drs_id): + """Assert that extracting the DRS id from a business documents record filekey produces the expected result.""" + doc: Document = Document(file_key=filekey) + result = doc_service.get_drs_id(doc) + assert result == drs_id diff --git a/legal-api/tests/unit/services/test_minio.py b/legal-api/tests/unit/services/test_minio.py deleted file mode 100644 index f321537d34..0000000000 --- a/legal-api/tests/unit/services/test_minio.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright © 2021 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for the Minio service. - -Test suite to ensure that the Minio service routines are working as expected. -""" -import os - -import requests -from minio.error import S3Error - -from legal_api.services import MinioService - -from .test_pdf_service import _create_pdf_file - - -def test_create_signed_put_url(session, minio_server): # pylint:disable=unused-argument - """Assert that the a PUT url can be pre-signed.""" - file_name = 'cooperative-test.pdf' - signed_url = MinioService.create_signed_put_url(file_name) - assert signed_url - assert signed_url.get('key').endswith('.pdf') - - -def test_create_signed_get_url(session, minio_server, tmpdir): # pylint:disable=unused-argument - """Assert that a GET url can be pre-signed.""" - key = _upload_file(tmpdir) - pre_signed_get = MinioService.create_signed_get_url(key) - assert pre_signed_get - get_response = requests.get(pre_signed_get) - assert get_response - - -def test_get_file_info(session, minio_server, tmpdir): # pylint:disable=unused-argument - """Assert that we can retrieve a file info.""" - key = _upload_file(tmpdir) - file_info = MinioService.get_file_info(key) - assert file_info - - -def test_get_file(session, minio_server, tmpdir): # pylint:disable=unused-argument - """Assert that we can retrieve a file.""" - key = _upload_file(tmpdir) - get_response = MinioService.get_file(key) - assert get_response - - -def test_delete_file(session, minio_server, tmpdir): # pylint:disable=unused-argument - """Assert that a file can be deleted.""" - key = _upload_file(tmpdir) - MinioService.delete_file(key) - - try: - MinioService.get_file_info(key) - except S3Error as ex: - assert ex.code == 'NoSuchKey' - - -def _upload_file(tmpdir): - d = tmpdir.mkdir('subdir') - fh = d.join('cooperative-test.pdf') - fh.write('Test File') - filename = os.path.join(fh.dirname, fh.basename) - - test_file = open(filename, 'rb') - files = {'upload_file': test_file} - file_name = fh.basename - signed_url = MinioService.create_signed_put_url(file_name) - key = signed_url.get('key') - pre_signed_put = signed_url.get('preSignedUrl') - requests.put(pre_signed_put, files=files) - return key - - -def test_put_file(session, minio_server, tmpdir): # pylint:disable=unused-argument - """Assert that a file can be replaced.""" - key = _upload_file(tmpdir) - - pdf_file = _create_pdf_file() - # Replace previous file with this pdf file - MinioService.put_file(key, pdf_file, pdf_file.getbuffer().nbytes) - - try: - file = MinioService.get_file(key) - pdf_file.seek(0) - assert file.data == pdf_file.read() - except S3Error as ex: - assert ex.code == 'NoSuchKey' diff --git a/legal-api/tests/unit/services/test_request_context.py b/legal-api/tests/unit/services/test_request_context.py new file mode 100644 index 0000000000..bacbefce7e --- /dev/null +++ b/legal-api/tests/unit/services/test_request_context.py @@ -0,0 +1,76 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests to assure the RequestContext service. + +Test-Suite to ensure that RequestContext and its helpers work as expected. +""" +import pytest +from flask import g + +from legal_api.services.request_context import add_account_linking_key_header, build_from_flask, get_request_context +from tests.unit.services.utils import helper_create_jwt_json_token_claims + + +@pytest.fixture(autouse=True) +def isolate_request_context(monkeypatch): + """Isolate flask.g for each test in this module. + + The `app` fixture is session scoped and keeps a single app context open for the whole run, + Use monkeypatch so g is restored to its initial state after each test. + """ + monkeypatch.setattr(g, 'jwt_oidc_token_info', None, raising=False) + g.pop('request_context', None) + yield + g.pop('request_context', None) + + +@pytest.mark.parametrize('headers,expected', [ + ({'Account-Linking-Key': 'test-linking-key'}, 'test-linking-key'), + ({}, None), +]) +def test_build_from_flask_account_linking_key(app, headers, expected): + """Assert that build_from_flask reads the Account-Linking-Key header when present.""" + with app.test_request_context(headers=headers): + rc = build_from_flask() + assert rc.account_linking_key == expected + + +def test_get_request_context_backwards_compatible(app, session, monkeypatch): + """Assert get_request_context() behaves as before when no Account-Linking-Key header is present.""" + token_info = helper_create_jwt_json_token_claims(username='test-user') + with app.test_request_context(headers={'Account-Id': '1234'}): + monkeypatch.setattr(g, 'jwt_oidc_token_info', token_info, raising=False) + rc = get_request_context() + assert rc.account_id == '1234' + assert rc.user is not None + assert rc.user.username == 'test-user' + assert rc.account_linking_key is None + + +def test_add_account_linking_key_header_present(app): + """Assert that the header is forwarded onto a headers dict when present on the request.""" + with app.test_request_context(headers={'Account-Linking-Key': 'test-linking-key'}): + headers = {'Authorization': 'Bearer token'} + add_account_linking_key_header(headers) + assert headers == {'Authorization': 'Bearer token', 'Account-Linking-Key': 'test-linking-key'} + + +def test_add_account_linking_key_header_absent(app): + """Assert that no header is added onto a headers dict when absent on the request.""" + with app.test_request_context(): + headers = {'Authorization': 'Bearer token'} + add_account_linking_key_header(headers) + assert headers == {'Authorization': 'Bearer token'} + diff --git a/legal-api/tests/unit/services/utils.py b/legal-api/tests/unit/services/utils.py index 01cd8d013d..f02c5a070a 100644 --- a/legal-api/tests/unit/services/utils.py +++ b/legal-api/tests/unit/services/utils.py @@ -31,7 +31,8 @@ } -def helper_create_jwt_json_token_claims(roles: List[str] = [], username: str = 'test-user'): +def helper_create_jwt_json_token_claims(roles: List[str] = [], username: str = 'test-user', + login_source: str = 'IDIR', extra_claims: dict = None): """Create a jwt token claims""" return { 'iss': 'https://example.localdomain/auth/realms/example', @@ -42,18 +43,24 @@ def helper_create_jwt_json_token_claims(roles: List[str] = [], username: str = ' 'jti': 'flask-jwt-oidc-test-support', 'typ': 'Bearer', 'username': f'{username}', + # name matches the completing party officer in the registry_schemas example filings + 'firstname': 'Joe P', + 'lastname': 'Swanson', 'idp_userid': '123', - 'loginSource': 'IDIR', + 'loginSource': login_source, 'realm_access': { 'roles': [] + roles - } + }, + **(extra_claims or {}), } -def helper_create_jwt(jwt_manager: JwtManager, roles: List[str] = [], username: str = 'test-user'): +def helper_create_jwt(jwt_manager: JwtManager, roles: List[str] = [], username: str = 'test-user', + login_source: str = 'IDIR', extra_claims: dict = None): """Create a jwt bearer token with the correct keys, roles and username.""" token_header = jwt_json_token_header - token_claims = helper_create_jwt_json_token_claims(roles=roles, username=username) + token_claims = helper_create_jwt_json_token_claims(roles=roles, username=username, login_source=login_source, + extra_claims=extra_claims) return jwt_manager.create_jwt(token_claims, token_header) @@ -73,14 +80,18 @@ def jwt_request_context(app, jwt_manager: JwtManager, roles: List[str] = [], username: str = 'test-user', - account_id: str = '1'): + account_id: str = '1', + login_source: str = 'IDIR', + extra_claims: dict = None, + **kwargs): """Push a Flask request context with a valid JWT and populate request_ctx.current_user the way the @jwt.has_one_of_roles decorator would in production. Used by tests that call authz functions directly instead of routing through a JWT-decorated view. """ - token = helper_create_jwt(jwt_manager, roles=roles, username=username) - headers = {'Authorization': f'Bearer {token}', 'Account-Id': account_id} + token = helper_create_jwt(jwt_manager, roles=roles, username=username, login_source=login_source, + extra_claims=extra_claims) + headers = {'Authorization': f'Bearer {token}', 'Account-Id': account_id, **kwargs} with app.test_request_context(headers=headers): jwt_manager._require_auth_validation() yield diff --git a/python/common/business-registry-digital-credentials/poetry.lock b/python/common/business-registry-digital-credentials/poetry.lock index b86aad484a..81e82cc343 100644 --- a/python/common/business-registry-digital-credentials/poetry.lock +++ b/python/common/business-registry-digital-credentials/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "alembic" @@ -105,7 +105,7 @@ files = [ [[package]] name = "business-model" -version = "3.3.30" +version = "3.4.1" description = "" optional = false python-versions = ">=3.13,<3.14" @@ -125,14 +125,14 @@ pg8000 = ">=1.31.2,<2.0.0" pycountry = ">=24.6.1,<25.0.0" pydantic = ">=2.10.6,<3.0.0" pytz = ">=2025.1,<2026.0" -registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.69"} +registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.74"} sql-versioning = {git = "https://github.com/bcgov/lear.git", rev = "main", subdirectory = "python/common/sql-versioning-alt"} [package.source] type = "git" url = "https://github.com/bcgov/lear.git" reference = "main" -resolved_reference = "b23c6da0282d8ab9387e3d4d618cf000489d2359" +resolved_reference = "ab63e41cf55bbea31a66c4f12d608bd619eef05d" subdirectory = "python/common/business-registry-model" [[package]] @@ -1572,8 +1572,8 @@ strict-rfc3339 = "*" [package.source] type = "git" url = "https://github.com/bcgov/business-schemas.git" -reference = "2.18.69" -resolved_reference = "25282bd6cf4de87e9dc6bc44f619ed723b64c7f2" +reference = "2.18.74" +resolved_reference = "5abdc89a0f6eb06cbbac9ad66b580d7c76190ebc" [[package]] name = "requests" diff --git a/python/common/business-registry-digital-credentials/pyproject.toml b/python/common/business-registry-digital-credentials/pyproject.toml index 98fe00c6e4..9d14110d1c 100644 --- a/python/common/business-registry-digital-credentials/pyproject.toml +++ b/python/common/business-registry-digital-credentials/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "business-registry-digital-credentials" -version = "0.2.0" +version = "0.2.1" description = "" authors = ["Lucas O'Neil "] readme = "README.md" diff --git a/python/common/business-registry-dissolution/poetry.lock b/python/common/business-registry-dissolution/poetry.lock index c9d49a6ca1..788f23a3fa 100644 --- a/python/common/business-registry-dissolution/poetry.lock +++ b/python/common/business-registry-dissolution/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "alembic" @@ -113,7 +113,7 @@ files = [ [[package]] name = "business-model" -version = "3.3.30" +version = "3.4.1" description = "" optional = false python-versions = ">=3.13,<3.14" @@ -133,14 +133,14 @@ pg8000 = ">=1.31.2,<2.0.0" pycountry = ">=24.6.1,<25.0.0" pydantic = ">=2.10.6,<3.0.0" pytz = ">=2025.1,<2026.0" -registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.69"} +registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.74"} sql-versioning = {git = "https://github.com/bcgov/lear.git", rev = "main", subdirectory = "python/common/sql-versioning-alt"} [package.source] type = "git" url = "https://github.com/bcgov/lear.git" reference = "main" -resolved_reference = "c9c4669cbe5630e05a48179619c7c19432e6fdec" +resolved_reference = "ab63e41cf55bbea31a66c4f12d608bd619eef05d" subdirectory = "python/common/business-registry-model" [[package]] @@ -1440,7 +1440,7 @@ rpds-py = ">=0.7.0" [[package]] name = "registry_schemas" -version = "2.18.69" +version = "2.18.74" description = "A short description of the project" optional = false python-versions = ">=3.6" @@ -1458,8 +1458,8 @@ strict-rfc3339 = "*" [package.source] type = "git" url = "https://github.com/bcgov/business-schemas.git" -reference = "2.18.69" -resolved_reference = "25282bd6cf4de87e9dc6bc44f619ed723b64c7f2" +reference = "2.18.74" +resolved_reference = "5abdc89a0f6eb06cbbac9ad66b580d7c76190ebc" [[package]] name = "requests" diff --git a/python/common/business-registry-dissolution/pyproject.toml b/python/common/business-registry-dissolution/pyproject.toml index 3cd24b5b70..0e22abf609 100644 --- a/python/common/business-registry-dissolution/pyproject.toml +++ b/python/common/business-registry-dissolution/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "business-registry-dissolution" -version = "0.1.2" +version = "0.1.3" description = "" authors = [{name = "BC Business Registry"}] readme = "README.md" diff --git a/python/common/business-registry-model/poetry.lock b/python/common/business-registry-model/poetry.lock index a089fb168a..75ace1727c 100644 --- a/python/common/business-registry-model/poetry.lock +++ b/python/common/business-registry-model/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.1 and should not be changed by hand. [[package]] name = "alembic" @@ -578,11 +578,11 @@ files = [ ] [package.dependencies] -google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" -google-auth = ">=1.25.0,<3.0dev" +google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0.dev0" +google-auth = ">=1.25.0,<3.0.dev0" [package.extras] -grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] +grpc = ["grpcio (>=1.38.0,<2.0.dev0)", "grpcio-status (>=1.38.0,<2.0.dev0)"] [[package]] name = "google-cloud-datastore" @@ -895,7 +895,7 @@ fqdn = {version = "*", optional = true, markers = "extra == \"format\""} idna = {version = "*", optional = true, markers = "extra == \"format\""} isoduration = {version = "*", optional = true, markers = "extra == \"format\""} jsonpointer = {version = ">1.13", optional = true, markers = "extra == \"format\""} -jsonschema-specifications = ">=2023.03.6" +jsonschema-specifications = ">=2023.3.6" referencing = ">=0.28.4" rfc3339-validator = {version = "*", optional = true, markers = "extra == \"format\""} rfc3987 = {version = "*", optional = true, markers = "extra == \"format\""} @@ -1472,7 +1472,7 @@ rpds-py = ">=0.7.0" [[package]] name = "registry_schemas" -version = "2.18.73" +version = "2.18.81" description = "A short description of the project" optional = false python-versions = ">=3.6" @@ -1490,8 +1490,8 @@ strict-rfc3339 = "*" [package.source] type = "git" url = "https://github.com/bcgov/business-schemas.git" -reference = "2.18.73" -resolved_reference = "e6999a46b6e38c52cc79f0e5a3c9fcbd0c473434" +reference = "2.18.82" +resolved_reference = "7dd37ea586adceff8f5b0d775a9c27eabd2d5135" [[package]] name = "requests" @@ -2085,4 +2085,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.13,<3.14" -content-hash = "ec2f82cd063a03708cf9e5103b1de1002f0e75e06c995790c5cc6d53558e3a0e" +content-hash = "c35bc1cf2284b6787c85070c2ca17bcea0dc81ea2f24f41ff8cb3cdbc2a92d98" diff --git a/python/common/business-registry-model/pyproject.toml b/python/common/business-registry-model/pyproject.toml index 61a2a02288..e44a4033fe 100644 --- a/python/common/business-registry-model/pyproject.toml +++ b/python/common/business-registry-model/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "business-model" -version = "3.3.32" +version = "3.4.12" description = "" authors = [ {name = "thor",email = "1042854+thorwolpert@users.noreply.github.com"} @@ -9,7 +9,7 @@ readme = "README.md" requires-python = ">=3.13,<3.14" dependencies = [ "sql-versioning @ git+https://github.com/bcgov/lear.git@main#subdirectory=python/common/sql-versioning-alt", - "registry-schemas @ git+https://github.com/bcgov/business-schemas.git@2.18.73", + "registry-schemas @ git+https://github.com/bcgov/business-schemas.git@2.18.82", "flask-migrate (>=4.1.0,<5.0.0)", "pg8000 (>=1.31.2,<2.0.0)", "pydantic (>=2.10.6,<3.0.0)", diff --git a/python/common/business-registry-model/src/business_model/models/__init__.py b/python/common/business-registry-model/src/business_model/models/__init__.py index d65804e8a1..5c8c113521 100644 --- a/python/common/business-registry-model/src/business_model/models/__init__.py +++ b/python/common/business-registry-model/src/business_model/models/__init__.py @@ -31,6 +31,7 @@ from .configuration import Configuration from .consent_continuation_out import ConsentContinuationOut from .corp_type import CorpType +from .court_order import CourtOrder from .dc_business_user import DCBusinessUser from .dc_connection import DCConnection from .dc_credential import DCCredential @@ -76,6 +77,7 @@ 'Configuration', 'ConsentContinuationOut', 'CorpType', + 'CourtOrder', 'DCBusinessUser', 'DCConnection', 'DCCredential', diff --git a/python/common/business-registry-model/src/business_model/models/amalgamating_business.py b/python/common/business-registry-model/src/business_model/models/amalgamating_business.py index f3e0cf3295..e3be182703 100644 --- a/python/common/business-registry-model/src/business_model/models/amalgamating_business.py +++ b/python/common/business-registry-model/src/business_model/models/amalgamating_business.py @@ -46,6 +46,8 @@ class Role(BaseEnum): foreign_jurisdiction_region = db.Column('foreign_jurisdiction_region', db.String(10)) foreign_name = db.Column('foreign_name', db.String(100)) foreign_identifier = db.Column('foreign_identifier', db.String(50)) + # a business in COLIN that is not loaded in LEAR (and is not a foreign business) + colin_identifier = db.Column('colin_identifier', db.String(10)) # parent keys business_id = db.Column('business_id', db.Integer, db.ForeignKey('businesses.id'), index=True) diff --git a/python/common/business-registry-model/src/business_model/models/business.py b/python/common/business-registry-model/src/business_model/models/business.py index 2b946cee0b..2d5ff6c740 100644 --- a/python/common/business-registry-model/src/business_model/models/business.py +++ b/python/common/business-registry-model/src/business_model/models/business.py @@ -316,6 +316,7 @@ class AssociationTypes(Enum): amalgamation = db.relationship('Amalgamation', lazy='dynamic') batch_processing = db.relationship('BatchProcessing', lazy='dynamic') jurisdictions = db.relationship('Jurisdiction', lazy='dynamic') + court_orders = db.relationship('CourtOrder', lazy='dynamic') @hybrid_property def identifier(self): @@ -366,6 +367,8 @@ def business_legal_name(self): @property def next_anniversary(self): """Retrieve the next anniversary date for which an AR filing is due.""" + if not self.founding_date or self.founding_date.year <= 1: + return None _founding_date = LegislationDatetime.as_legislation_timezone(self.founding_date) next_ar_year = (self.last_ar_year if self.last_ar_year else _founding_date.year) + 1 no_of_years_to_add = next_ar_year - _founding_date.year @@ -670,6 +673,7 @@ def json(self, slim=False): (self.last_ar_year if self.last_ar_year else self.founding_date.year) + 1 ) + next_anniversary = self.next_anniversary d = { **slim_json, 'arMinDate': ar_min_date.isoformat() if ar_min_date else '', @@ -685,7 +689,7 @@ def json(self, slim=False): 'naicsKey': self.naics_key, 'naicsCode': self.naics_code, 'naicsDescription': self.naics_description, - 'nextAnnualReport': self.next_anniversary.isoformat(), + 'nextAnnualReport': next_anniversary.isoformat() if next_anniversary else '', 'noDissolution': self.no_dissolution, 'associationType': self.association_type, 'allowedActions': self.allowable_actions, @@ -724,9 +728,9 @@ def _slim_json(self): def _extend_json(self, d): """Include conditional fields to json.""" - if self.last_coa_date: + if self.last_coa_date and self.last_coa_date.year > 1: d['lastAddressChangeDate'] = LegislationDatetime.format_as_legislation_date(self.last_coa_date) - if self.last_cod_date: + if self.last_cod_date and self.last_cod_date.year > 1: d['lastDirectorChangeDate'] = LegislationDatetime.format_as_legislation_date(self.last_cod_date) if self.dissolution_date: diff --git a/python/common/business-registry-model/src/business_model/models/court_order.py b/python/common/business-registry-model/src/business_model/models/court_order.py new file mode 100644 index 0000000000..565a35e533 --- /dev/null +++ b/python/common/business-registry-model/src/business_model/models/court_order.py @@ -0,0 +1,113 @@ +# Copyright (c) 2026, Province of British Columbia +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""This module contains the database model for a court order.""" +from __future__ import annotations + +from sql_versioning import Versioned + +from .db import db +from .filing import Filing + + +class CourtOrder(db.Model, Versioned): + __tablename__ = "court_orders" + __versioned__ = {} + __mapper_args__ = { + "include_properties": [ + "id", + "business_id", + "effect_of_order", + "file_number", + "filing_id", + "order_date", + "order_details" + ] + } + + id = db.Column(db.Integer, primary_key=True) + business_id = db.Column(db.Integer, db.ForeignKey("businesses.id")) + filing_id = db.Column(db.Integer, db.ForeignKey("filings.id"), nullable=False) + file_number = db.Column("file_number", db.String(20)) + effect_of_order = db.Column("effect_of_order", db.String(20)) + order_date = db.Column("order_date", db.DateTime(timezone=True), default=None) + order_details = db.Column("order_details", db.String(2000)) + + @property + def json(self) -> dict: + data = { + "id": self.id, + "filingId": self.filing_id, + "fileNumber": self.file_number, + "effectOfOrder": self.effect_of_order, + "orderDetails": self.order_details + } + if self.order_date: + data["orderDate"] = self.order_date.isoformat() + + return data + + def save(self): + db.session.add(self) + db.session.commit() + + @classmethod + def get_by_id(cls, id) -> CourtOrder | None: + """Get a court order.""" + if not id: + return None + return cls.query.filter_by(id=id).one_or_none() + + @classmethod + def get_by_filing_id(cls, filing_id) -> CourtOrder | None: + """Get court order for a filing.""" + if not filing_id: + return None + return cls.query.filter_by(filing_id=filing_id).one_or_none() + + @classmethod + def get_by_business_id(cls, business_id, include_filing_type: bool = False) -> list[CourtOrder]: + """Get court orders for a business, optionally including filing type.""" + if not business_id: + return [] + if include_filing_type: + query = (db.session.query(CourtOrder, Filing._filing_type) + .join(Filing, CourtOrder.filing_id == Filing.id)) + else: + query = db.session.query(CourtOrder) + + return query.filter(CourtOrder.business_id == business_id).order_by(CourtOrder.id.desc()).all() + + @classmethod + def get_json_with_filing_type(cls, business_id) -> list[dict]: + """Get court orders for a business with filing type.""" + return [ + { + **order.json, + "filingType": filing_type + } + for (order, filing_type) in cls.get_by_business_id(business_id, True) + ] diff --git a/python/common/business-registry-model/src/business_model/models/document.py b/python/common/business-registry-model/src/business_model/models/document.py index ad4c03a4dd..3af5c099c8 100644 --- a/python/common/business-registry-model/src/business_model/models/document.py +++ b/python/common/business-registry-model/src/business_model/models/document.py @@ -36,6 +36,7 @@ class DocumentType(Enum): COOP_MEMORANDUM = 'coop_memorandum' COURT_ORDER = 'court_order' DIRECTOR_AFFIDAVIT = 'director_affidavit' + SUPPORTING_DOCUMENT = 'supporting_document' class Document(db.Model, Versioned): diff --git a/python/common/business-registry-model/src/business_model/models/filing.py b/python/common/business-registry-model/src/business_model/models/filing.py index f9dad8283f..b2e7d37355 100644 --- a/python/common/business-registry-model/src/business_model/models/filing.py +++ b/python/common/business-registry-model/src/business_model/models/filing.py @@ -769,14 +769,11 @@ class Source(Enum): '_status', 'business_id', 'colin_only', - 'court_order_date', - 'court_order_effect_of_order', - 'court_order_file_number', 'deletion_locked', 'hide_in_ledger', 'lear_only', 'effective_date', - 'order_details', + 'details', 'paper_only', 'parent_filing_id', 'payment_account', @@ -812,10 +809,7 @@ class Source(Enum): effective_date = db.Column('effective_date', db.DateTime(timezone=True), default=func.now()) submitter_roles = db.Column('submitter_roles', db.String(200)) tech_correction_json = db.Column('tech_correction_json', JSONB) - court_order_file_number = db.Column('court_order_file_number', db.String(20)) - court_order_date = db.Column('court_order_date', db.DateTime(timezone=True), default=None) - court_order_effect_of_order = db.Column('court_order_effect_of_order', db.String(500)) - order_details = db.Column(db.String(2000)) + details = db.Column(db.String(2000)) deletion_locked = db.Column('deletion_locked', db.Boolean, unique=False, default=False) approval_type = db.Column('approval_type', db.String(15)) application_date = db.Column('application_date', db.DateTime(timezone=True)) @@ -840,6 +834,7 @@ class Source(Enum): colin_event_ids = db.relationship('ColinEventId', lazy='select') + court_orders = db.relationship('CourtOrder', lazy='dynamic') comments = db.relationship('Comment', lazy='dynamic') documents = db.relationship('Document', lazy='dynamic') filing_party_roles = db.relationship('PartyRole', lazy='dynamic') diff --git a/python/common/business-registry-model/src/business_model_migrations/env.py b/python/common/business-registry-model/src/business_model_migrations/env.py index 14a6536f6e..1db36bbe0c 100644 --- a/python/common/business-registry-model/src/business_model_migrations/env.py +++ b/python/common/business-registry-model/src/business_model_migrations/env.py @@ -1,7 +1,8 @@ - import logging +import os from logging.config import fileConfig +import sqlalchemy as sa from alembic import context from flask import current_app @@ -12,22 +13,24 @@ # Interpret the config file for Python logging. # This line sets up loggers basically. fileConfig(config.config_file_name) -logger = logging.getLogger('alembic.env') +logger = logging.getLogger("alembic.env") -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -config.set_main_option( - 'sqlalchemy.url', - str(current_app.extensions['migrate'].db.engine.url).replace('%', '%%')) -target_metadata = current_app.extensions['migrate'].db.metadata +def get_engine(): + try: + # this works with Flask-SQLAlchemy<3 and Alchemical + return current_app.extensions["migrate"].db.get_engine() + except TypeError: + # this works with Flask-SQLAlchemy>=3 + return current_app.extensions["migrate"].db.engine -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. +def get_engine_url(): + try: + return get_engine().url.render_as_string(hide_password=False).replace("%", "%%") + except AttributeError: + return str(get_engine().url).replace("%", "%%") +config.set_main_option("sqlalchemy.url", get_engine_url()) +target_metadata = current_app.extensions["migrate"].db.metadata def run_migrations_offline(): """Run migrations in 'offline' mode. @@ -39,7 +42,6 @@ def run_migrations_offline(): Calls to context.execute() here emit the given string to the script output. - """ url = config.get_main_option("sqlalchemy.url") context.configure( @@ -55,30 +57,32 @@ def run_migrations_online(): In this scenario we need to create an Engine and associate a connection with the context. - """ - # this callback is used to prevent an auto-migration from being generated # when there are no changes to the schema # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html def process_revision_directives(context, revision, directives): - if getattr(config.cmd_opts, 'autogenerate', False): + if getattr(config.cmd_opts, "autogenerate", False): script = directives[0] if script.upgrade_ops.is_empty(): directives[:] = [] - logger.info('No changes in schema detected.') - - connectable = current_app.extensions['migrate'].db.engine + logger.info("No changes in schema detected.") + connectable = get_engine() with connectable.connect() as connection: context.configure( connection=connection, target_metadata=target_metadata, process_revision_directives=process_revision_directives, - **current_app.extensions['migrate'].configure_args + **current_app.extensions["migrate"].configure_args ) with context.begin_transaction(): + owner_role = os.getenv("DATABASE_OWNER_ROLE") + if owner_role: + safe_role = owner_role.replace('"', '""') # Escape any quotes for SQL safety + connection.execute(sa.text(f'SET ROLE "{safe_role}"')) + context.run_migrations() diff --git a/python/common/business-registry-model/src/business_model_migrations/versions/20260709_155716_1cdc6fdbf4cf_court_order.py b/python/common/business-registry-model/src/business_model_migrations/versions/20260709_155716_1cdc6fdbf4cf_court_order.py new file mode 100644 index 0000000000..942c2f2b40 --- /dev/null +++ b/python/common/business-registry-model/src/business_model_migrations/versions/20260709_155716_1cdc6fdbf4cf_court_order.py @@ -0,0 +1,110 @@ +"""court order + +Revision ID: 1cdc6fdbf4cf +Revises: bc62c64eeaab +Create Date: 2026-07-09 15:57:16.422059 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '1cdc6fdbf4cf' +down_revision = 'bc62c64eeaab' +branch_labels = None +depends_on = None + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('court_orders', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('filing_id', sa.Integer(), nullable=False), + sa.Column('business_id', sa.Integer(), nullable=True), + sa.Column('file_number', sa.String(length=20), nullable=True), + sa.Column('order_date', sa.DateTime(timezone=True), nullable=True), + sa.Column('effect_of_order', sa.String(length=20), nullable=True), + sa.Column('order_details', sa.String(length=2000), nullable=True), + sa.ForeignKeyConstraint(['business_id'], ['businesses.id'], ), + sa.ForeignKeyConstraint(['filing_id'], ['filings.id'], ), + sa.PrimaryKeyConstraint('id') + ) + + op.create_table('court_orders_version', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('filing_id', sa.Integer(), nullable=False), + sa.Column('business_id', sa.Integer(), nullable=True), + sa.Column('file_number', sa.String(length=20), nullable=True), + sa.Column('order_date', sa.DateTime(timezone=True), nullable=True), + sa.Column('effect_of_order', sa.String(length=20), nullable=True), + sa.Column('order_details', sa.String(length=2000), nullable=True), + sa.Column('transaction_id', sa.BigInteger(), nullable=False), + sa.Column('end_transaction_id', sa.BigInteger(), nullable=True), + sa.Column('operation_type', sa.SmallInteger(), nullable=False), + sa.ForeignKeyConstraint(['business_id'], ['businesses.id'], ), + sa.ForeignKeyConstraint(['filing_id'], ['filings.id'], ), + sa.PrimaryKeyConstraint('id', 'transaction_id') + ) + op.create_index(op.f('ix_court_orders_version_end_transaction_id'), 'court_orders_version', ['end_transaction_id'], unique=False) + op.create_index(op.f('ix_court_orders_version_operation_type'), 'court_orders_version', ['operation_type'], unique=False) + op.create_index(op.f('ix_court_orders_version_transaction_id'), 'court_orders_version', ['transaction_id'], unique=False) + + # Migrate data from filings to court_orders + op.execute( + """ + INSERT INTO court_orders (filing_id, business_id, file_number, order_date, effect_of_order, order_details) + SELECT id, business_id, court_order_file_number, court_order_date, court_order_effect_of_order, + case when filing_type = 'courtOrder' then order_details else null end as order_details + FROM filings + WHERE status='COMPLETED' and court_order_file_number is not null and trim(court_order_file_number) <> '' + """ + ) + op.execute( + """ + INSERT INTO court_orders_version (id, filing_id, business_id, file_number, order_date, effect_of_order, order_details, transaction_id, operation_type) + SELECT c.id, c.filing_id, c.business_id, c.file_number, c.order_date, c.effect_of_order, c.order_details, f.transaction_id, 0 + FROM court_orders c + join filings f on c.filing_id = f.id + """ + ) + op.execute( + """ + UPDATE filings + SET order_details = null + WHERE filing_type = 'courtOrder' and status='COMPLETED' + """ + ) + + with op.batch_alter_table('filings', schema=None) as batch_op: + batch_op.alter_column('order_details', new_column_name='details') + batch_op.drop_column('court_order_file_number') + batch_op.drop_column('court_order_effect_of_order') + batch_op.drop_column('court_order_date') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('filings', schema=None) as batch_op: + batch_op.add_column(sa.Column('court_order_date', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True)) + batch_op.add_column(sa.Column('court_order_effect_of_order', sa.VARCHAR(length=500), autoincrement=False, nullable=True)) + batch_op.add_column(sa.Column('court_order_file_number', sa.VARCHAR(length=20), autoincrement=False, nullable=True)) + batch_op.alter_column('details', new_column_name='order_details') + + # Migrate data back from orders to filings + op.execute( + """ + UPDATE filings + SET court_order_file_number = o.file_number, + court_order_date = o.order_date, + court_order_effect_of_order = o.effect_of_order, + order_details = o.order_details + FROM court_orders o + WHERE filings.id = o.filing_id; + """ + ) + + op.drop_table('court_orders') + op.drop_table('court_orders_version') + + # ### end Alembic commands ### diff --git a/python/common/business-registry-model/src/business_model_migrations/versions/20260727_162109_d7fc1a767d69_.py b/python/common/business-registry-model/src/business_model_migrations/versions/20260727_162109_d7fc1a767d69_.py new file mode 100644 index 0000000000..c8f88f0dd3 --- /dev/null +++ b/python/common/business-registry-model/src/business_model_migrations/versions/20260727_162109_d7fc1a767d69_.py @@ -0,0 +1,67 @@ +"""Add court order metadata + +Revision ID: d7fc1a767d69 +Revises: 1cdc6fdbf4cf +Create Date: 2026-07-27 16:21:09.583470 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = 'd7fc1a767d69' +down_revision = '1cdc6fdbf4cf' +branch_labels = None +depends_on = None + + +def upgrade(): + # Add court order data to metadata of filings + op.execute( + """ + UPDATE filings f + SET meta_data = JSONB_SET(meta_data, '{courtOrder}', + jsonb_strip_nulls(jsonb_build_object( + 'fileNumber', co.file_number, + 'orderDetails', CASE WHEN co.order_details IS NOT NULL AND co.order_details <> '' THEN co.order_details ELSE NULL END, + 'effectOfOrder', CASE WHEN co.effect_of_order IS NOT NULL AND co.effect_of_order <> '' THEN co.effect_of_order ELSE NULL END + )) + ) + FROM court_orders co + WHERE + co.filing_id = f.id + """ + ) + # Add court order filing document to metadata + op.execute( + """ + UPDATE filings f + SET meta_data = JSONB_SET(meta_data, '{courtOrder,files}', + jsonb_build_array( + jsonb_build_object( + 'fileName', concat('Court Order ', co.file_number, '.pdf'), + 'fileKey', d.file_key + ) + ) + ) + FROM court_orders co + JOIN documents d ON co.filing_id = d.filing_id + WHERE + co.filing_id = f.id AND + f.filing_type = 'courtOrder' AND + d.type = 'court_order' + """ + ) + + +def downgrade(): + # Remove court order data from metadata of filings + op.execute( + """ + UPDATE filings f + SET meta_data = JSONB_SET(meta_data, '{courtOrder}', 'jsonb_build_object()') + FROM court_orders co + WHERE + co.filing_id = f.id + """ + ) diff --git a/python/common/business-registry-model/src/business_model_migrations/versions/20260824_101500_a3c9e1f04b27_colin_amalgamating_business.py b/python/common/business-registry-model/src/business_model_migrations/versions/20260824_101500_a3c9e1f04b27_colin_amalgamating_business.py new file mode 100644 index 0000000000..d73e3084b6 --- /dev/null +++ b/python/common/business-registry-model/src/business_model_migrations/versions/20260824_101500_a3c9e1f04b27_colin_amalgamating_business.py @@ -0,0 +1,31 @@ +"""colin amalgamating business + +Revision ID: a3c9e1f04b27 +Revises: d7fc1a767d69 +Create Date: 2026-08-24 10:15:00.000000 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = 'a3c9e1f04b27' +down_revision = 'd7fc1a767d69' +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table('amalgamating_businesses', schema=None) as batch_op: + batch_op.add_column(sa.Column('colin_identifier', sa.String(length=10), nullable=True)) + + with op.batch_alter_table('amalgamating_businesses_version', schema=None) as batch_op: + batch_op.add_column(sa.Column('colin_identifier', sa.String(length=10), nullable=True)) + + +def downgrade(): + with op.batch_alter_table('amalgamating_businesses_version', schema=None) as batch_op: + batch_op.drop_column('colin_identifier') + + with op.batch_alter_table('amalgamating_businesses', schema=None) as batch_op: + batch_op.drop_column('colin_identifier') diff --git a/python/common/business-registry-model/tests/models/test_amalgamating_business.py b/python/common/business-registry-model/tests/models/test_amalgamating_business.py index f624ab7d2c..2821a2e0f8 100644 --- a/python/common/business-registry-model/tests/models/test_amalgamating_business.py +++ b/python/common/business-registry-model/tests/models/test_amalgamating_business.py @@ -70,9 +70,19 @@ def test_valid_amalgamating_business_save(session): ) amalgamating_business_2.save() + amalgamating_business_3 = AmalgamatingBusiness( + role=AmalgamatingBusiness.Role.amalgamating, + colin_identifier="BC7654321", + amalgamation_id=amalgamation.id + ) + amalgamating_business_3.save() + # verify assert amalgamating_business_1.id assert amalgamating_business_2.id + assert amalgamating_business_3.id + assert amalgamating_business_3.colin_identifier == "BC7654321" + assert amalgamating_business_3.business_id is None for type in AmalgamatingBusiness.Role: assert type in [AmalgamatingBusiness.Role.holding, AmalgamatingBusiness.Role.amalgamating, diff --git a/python/common/business-registry-model/tests/models/test_business.py b/python/common/business-registry-model/tests/models/test_business.py index 821778ff72..55157c17be 100644 --- a/python/common/business-registry-model/tests/models/test_business.py +++ b/python/common/business-registry-model/tests/models/test_business.py @@ -420,6 +420,28 @@ def test_business_json(app, session): assert business.json() == d +def test_business_json_sentinel_founding_date(app, session): + """Assert json() succeeds when founding_date is a year-1 COLIN sentinel.""" + business = Business( + legal_name='legal_name', + legal_type=Business.LegalTypes.COOP.value, + founding_date=datetime(1, 1, 1, 8, 0, 0, tzinfo=UTC), + last_coa_date=datetime(1, 1, 1, 8, 0, 0, tzinfo=UTC), + last_cod_date=datetime(1, 1, 1, 8, 0, 0, tzinfo=UTC), + last_ledger_timestamp=EPOCH_DATETIME, + identifier='CP1234567', + last_modified=EPOCH_DATETIME, + state=Business.State.ACTIVE, + ) + + assert business.next_anniversary is None + business_json = business.json() + assert business_json['nextAnnualReport'] == '' + assert business_json['foundingDate'] == '0001-01-01T08:00:00+00:00' + assert business_json['lastAddressChangeDate'] == '' + assert business_json['lastDirectorChangeDate'] == '' + + ALTERNATE_NAME_1 = "operating name 1" ALTERNATE_NAME_1_IDENTIFIER = "FM1111111" ALTERNATE_NAME_1_START_DATE = "2023-09-02" diff --git a/python/common/business-registry-model/tests/models/test_court_order.py b/python/common/business-registry-model/tests/models/test_court_order.py new file mode 100644 index 0000000000..fafcd9ba73 --- /dev/null +++ b/python/common/business-registry-model/tests/models/test_court_order.py @@ -0,0 +1,179 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests to assure the CourtOrder Model. + +Test-Suite to ensure that the CourtOrder Model is working as expected. +""" +from datetime import UTC, datetime + +import pytest + +from business_model.models import CourtOrder +from tests.models import factory_business, factory_filing + + +def test_court_order_save(session): + """Assert that the court order was saved.""" + business = factory_business('CP1234567') + filing = factory_filing(business, data_dict={'filing': {'header': {'name': 'courtOrder'}}}) + + court_order = CourtOrder( + filing_id=filing.id, + business_id=business.id, + file_number='12345', + effect_of_order='planOfArrangement', + order_details='Some details' + ) + court_order.save() + assert court_order.id + + +def test_court_order_json(session): + """Assert that court order json property works.""" + business = factory_business('CP1234567') + filing = factory_filing(business, data_dict={'filing': {'header': {'name': 'courtOrder'}}}) + order_date = datetime.now(UTC) + + court_order = CourtOrder( + filing_id=filing.id, + business_id=business.id, + file_number='12345', + order_date=order_date, + effect_of_order='planOfArrangement', + order_details='Some details' + ) + court_order.save() + + co_json = court_order.json + assert co_json['id'] == court_order.id + assert co_json['filingId'] == filing.id + assert co_json['fileNumber'] == '12345' + assert co_json['orderDate'] == order_date.isoformat() + assert co_json['effectOfOrder'] == 'planOfArrangement' + assert co_json['orderDetails'] == 'Some details' + + +def test_court_order_get_by_id(session): + """Assert that a court order can be retrieved by ID.""" + business = factory_business('CP1234567') + filing = factory_filing(business, data_dict={'filing': {'header': {'name': 'courtOrder'}}}) + + court_order = CourtOrder( + filing_id=filing.id, + business_id=business.id, + file_number='12345', + effect_of_order='planOfArrangement', + order_details='Some details' + ) + court_order.save() + + fetched = CourtOrder.get_by_id(court_order.id) + assert fetched + assert fetched.id == court_order.id + + +def test_court_order_get_by_filing_id(session): + """Assert that a court order can be retrieved by filing ID.""" + business = factory_business('CP1234567') + filing = factory_filing(business, data_dict={'filing': {'header': {'name': 'courtOrder'}}}) + + court_order = CourtOrder( + filing_id=filing.id, + business_id=business.id, + file_number='12345', + effect_of_order='planOfArrangement', + order_details='Some details' + ) + court_order.save() + + fetched = CourtOrder.get_by_filing_id(filing.id) + assert fetched + assert fetched.filing_id == filing.id + + +@pytest.mark.parametrize('include_filing_type', [True, False]) +def test_court_order_get_by_business_id(session, include_filing_type): + """Assert that court orders can be retrieved by business ID.""" + business = factory_business('CP1234567') + filing = factory_filing(business, data_dict={'filing': {'header': {'name': 'courtOrder'}}}, filing_type='courtOrder') + + court_order1 = CourtOrder( + filing_id=filing.id, + business_id=business.id, + file_number='12345', + effect_of_order='planOfArrangement', + order_details='Some details 1' + ) + court_order1.save() + + court_order2 = CourtOrder( + filing_id=filing.id, + business_id=business.id, + file_number='67890', + effect_of_order='planOfArrangement', + order_details='Some details 2' + ) + court_order2.save() + + fetched_list = CourtOrder.get_by_business_id(business.id, include_filing_type) + assert len(fetched_list) == 2 + if include_filing_type: + for fetched in fetched_list: + assert isinstance(fetched[0], CourtOrder) + assert isinstance(fetched[1], str) + assert fetched[1] == 'courtOrder' + else: + for fetched in fetched_list: + assert isinstance(fetched, CourtOrder) + + +def test_court_order_get_json_by_business_id(session): + """Assert that court orders can be retrieved by business ID.""" + business = factory_business('CP1234567') + filing = factory_filing(business, data_dict={'filing': {'header': {'name': 'courtOrder'}}}, filing_type='courtOrder') + + court_order1 = CourtOrder( + filing_id=filing.id, + business_id=business.id, + file_number='12345', + effect_of_order='planOfArrangement', + order_details='Some details 1' + ) + court_order1.save() + + court_order2 = CourtOrder( + filing_id=filing.id, + business_id=business.id, + file_number='67890', + effect_of_order='planOfArrangement', + order_details='Some details 2' + ) + court_order2.save() + + fetched_list = CourtOrder.get_json_with_filing_type(business.id) + assert len(fetched_list) == 2 + for fetched in fetched_list: + assert fetched['filingType'] == 'courtOrder' + assert fetched['orderDetails'] in ['Some details 1', 'Some details 2'] + assert fetched['fileNumber'] in ['12345', '67890'] + assert fetched['effectOfOrder'] == 'planOfArrangement' + assert fetched['filingId'] == filing.id + assert fetched['id'] in [court_order1.id, court_order2.id] + +def test_court_order_missing_getters(session): + """Assert that None/empty list is returned when ID is missing or not found.""" + assert CourtOrder.get_by_id(None) is None + assert CourtOrder.get_by_filing_id(None) is None + assert CourtOrder.get_by_business_id(None) == [] diff --git a/python/common/business-registry-model/tests/models/test_filing.py b/python/common/business-registry-model/tests/models/test_filing.py index 11190f8562..28b525b80e 100644 --- a/python/common/business-registry-model/tests/models/test_filing.py +++ b/python/common/business-registry-model/tests/models/test_filing.py @@ -38,7 +38,7 @@ from sqlalchemy.exc import IntegrityError from business_model.exceptions import BusinessException -from business_model.models import Business, Filing, User +from business_model.models import Business, CourtOrder, Filing, User from business_model.models.db import VersioningProxy from tests import EPOCH_DATETIME from tests.conftest import not_raises @@ -700,9 +700,13 @@ def test_alteration_filing_with_court_order(session): b = factory_business(identifier, datetime.datetime.utcnow(), None, Business.LegalTypes.COMP.value) factory_business_mailing_address(b) filing = factory_filing(b, ALTERATION_FILING_TEMPLATE) - filing.court_order_file_number = COURT_ORDER['fileNumber'] - filing.court_order_date = COURT_ORDER['orderDate'] - filing.court_order_effect_of_order = COURT_ORDER['effectOfOrder'] + order = CourtOrder() + order.filing_id = filing.id + order.business_id = b.id + order.file_number = COURT_ORDER['fileNumber'] + order.order_date = COURT_ORDER['orderDate'] + order.effect_of_order = COURT_ORDER['effectOfOrder'] + filing.court_orders.append(order) filing.filing_json = ALTERATION_FILING_TEMPLATE filing.save() assert filing.id is not None @@ -713,43 +717,6 @@ def test_alteration_filing_with_court_order(session): assert registry_schemas.validate(filing.json, 'alteration') -@pytest.mark.parametrize('invalid_court_order', [ - { - 'fileNumber': '123456789012345678901', # long fileNumber - 'orderDate': '2021-01-30T09:56:01+01:00', - 'effectOfOrder': 'planOfArrangement' - }, - { - 'fileNumber': 'Valid file number', - 'orderDate': 'a2021-01-30T09:56:01', # Invalid date - 'effectOfOrder': 'planOfArrangement' - }, - { - 'fileNumber': 'Valid File Number', - 'orderDate': '2021-01-30T09:56:01+01:00', - 'effectOfOrder': ('a' * 501) # long effectOfOrder - } -]) -def test_validate_invalid_court_orders(session, invalid_court_order): - """Assert not valid court orders.""" - identifier = 'BC1156677' - b = factory_business(identifier, datetime.datetime.utcnow(), None, Business.LegalTypes.COMP.value) - factory_business_mailing_address(b) - filing = factory_filing(b, ALTERATION_FILING_TEMPLATE) - filing.court_order_file_number = invalid_court_order['fileNumber'] - filing.court_order_date = invalid_court_order['orderDate'] - filing.court_order_effect_of_order = invalid_court_order['effectOfOrder'] - - # TODO: rescope the error after upgraing frameworks - # with pytest.raises(DataError, Exception, ProgrammingError) as excinfo: - with pytest.raises(Exception) as excinfo: - filing.save() - - assert excinfo - -# @pytest.mark.parametrize('test_name, json1, json2, expected', TEST_JSON_DIFF) - - def test_submitter_info(session): user = factory_user('idir/staff-person') filing = Filing() diff --git a/queue_services/business-bn/poetry.lock b/queue_services/business-bn/poetry.lock index 4945782566..70cdd48c1d 100644 --- a/queue_services/business-bn/poetry.lock +++ b/queue_services/business-bn/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "aiofiles" @@ -301,7 +301,7 @@ files = [ [[package]] name = "business-model" -version = "3.3.23" +version = "3.4.1" description = "" optional = false python-versions = ">=3.13,<3.14" @@ -321,14 +321,14 @@ pg8000 = ">=1.31.2,<2.0.0" pycountry = ">=24.6.1,<25.0.0" pydantic = ">=2.10.6,<3.0.0" pytz = ">=2025.1,<2026.0" -registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.62"} +registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.74"} sql-versioning = {git = "https://github.com/bcgov/lear.git", rev = "main", subdirectory = "python/common/sql-versioning-alt"} [package.source] type = "git" url = "https://github.com/bcgov/lear.git" reference = "main" -resolved_reference = "b745e248a60f21f23171b495cb5e1098ccf08079" +resolved_reference = "deecf506fc9e76fb9e52e91a69db0db10d93f068" subdirectory = "python/common/business-registry-model" [[package]] @@ -358,23 +358,25 @@ subdirectory = "python/common/business-registry-account" [[package]] name = "business-registry-common" -version = "0.1.0" +version = "0.1.1" description = "" optional = false -python-versions = ">=3.9,<4.0" +python-versions = ">=3.13,<3.14" groups = ["main"] files = [] develop = false [package.dependencies] -datedelta = "^1.4" -pytz = "^2025.1" +datedelta = ">=1.4,<2.0" +flask = ">=3.1.0,<4.0.0" +python-dateutil = ">=2.9.0,<3.0.0" +pytz = ">=2025.1,<2026.0" [package.source] type = "git" url = "https://github.com/bcgov/lear.git" reference = "main" -resolved_reference = "b745e248a60f21f23171b495cb5e1098ccf08079" +resolved_reference = "deecf506fc9e76fb9e52e91a69db0db10d93f068" subdirectory = "python/common/business-registry-common" [[package]] @@ -2761,7 +2763,7 @@ files = [ [[package]] name = "registry_schemas" -version = "2.18.62" +version = "2.18.74" description = "A short description of the project" optional = false python-versions = ">=3.6" @@ -2779,8 +2781,8 @@ strict-rfc3339 = "*" [package.source] type = "git" url = "https://github.com/bcgov/business-schemas.git" -reference = "2.18.62" -resolved_reference = "1cea81cf14104fb7d4989169236eff13b3df16c4" +reference = "2.18.74" +resolved_reference = "5abdc89a0f6eb06cbbac9ad66b580d7c76190ebc" [[package]] name = "requests" diff --git a/queue_services/business-bn/pyproject.toml b/queue_services/business-bn/pyproject.toml index 6a7d23ad84..b2cc15b1d6 100644 --- a/queue_services/business-bn/pyproject.toml +++ b/queue_services/business-bn/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "business-bn" -version = "0.1.5" +version = "0.1.6" description = "" authors = ["Patrick Wang "] readme = "README.md" diff --git a/queue_services/business-digital-credentials/poetry.lock b/queue_services/business-digital-credentials/poetry.lock index e7a6f29981..754fe170fa 100644 --- a/queue_services/business-digital-credentials/poetry.lock +++ b/queue_services/business-digital-credentials/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "aiofiles" @@ -286,7 +286,7 @@ files = [ [[package]] name = "business-model" -version = "3.3.30" +version = "3.4.1" description = "" optional = false python-versions = ">=3.13,<3.14" @@ -306,14 +306,14 @@ pg8000 = ">=1.31.2,<2.0.0" pycountry = ">=24.6.1,<25.0.0" pydantic = ">=2.10.6,<3.0.0" pytz = ">=2025.1,<2026.0" -registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.69"} +registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.74"} sql-versioning = {git = "https://github.com/bcgov/lear.git", rev = "main", subdirectory = "python/common/sql-versioning-alt"} [package.source] type = "git" url = "https://github.com/bcgov/lear.git" reference = "main" -resolved_reference = "2b62a37bf206edda1241cb2748213e2ef956bbe1" +resolved_reference = "deecf506fc9e76fb9e52e91a69db0db10d93f068" subdirectory = "python/common/business-registry-model" [[package]] @@ -2751,7 +2751,7 @@ rpds-py = ">=0.7.0" [[package]] name = "registry_schemas" -version = "2.18.62" +version = "2.18.74" description = "A short description of the project" optional = false python-versions = ">=3.6" @@ -2769,8 +2769,8 @@ strict-rfc3339 = "*" [package.source] type = "git" url = "https://github.com/bcgov/business-schemas.git" -reference = "2.18.69" -resolved_reference = "25282bd6cf4de87e9dc6bc44f619ed723b64c7f2" +reference = "2.18.74" +resolved_reference = "5abdc89a0f6eb06cbbac9ad66b580d7c76190ebc" [[package]] name = "requests" diff --git a/queue_services/business-digital-credentials/pyproject.toml b/queue_services/business-digital-credentials/pyproject.toml index 571fde9540..f5616d0f2f 100644 --- a/queue_services/business-digital-credentials/pyproject.toml +++ b/queue_services/business-digital-credentials/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "business-digital-credentials" -version = "0.2.0" +version = "0.2.1" description = "This module is the service worker for handling events that deal with Digital Business Card credential tasks." authors = ["Lucas O'Neil "] license = "BSD-3-Clause" diff --git a/queue_services/business-emailer/poetry.lock b/queue_services/business-emailer/poetry.lock index 8938b0dfbe..bf47849ab4 100644 --- a/queue_services/business-emailer/poetry.lock +++ b/queue_services/business-emailer/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "aiofiles" @@ -368,7 +368,7 @@ files = [ [[package]] name = "business-model" -version = "3.3.31" +version = "3.4.1" description = "" optional = false python-versions = ">=3.13,<3.14" @@ -388,14 +388,14 @@ pg8000 = ">=1.31.2,<2.0.0" pycountry = ">=24.6.1,<25.0.0" pydantic = ">=2.10.6,<3.0.0" pytz = ">=2025.1,<2026.0" -registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.71"} +registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.74"} sql-versioning = {git = "https://github.com/bcgov/lear.git", rev = "main", subdirectory = "python/common/sql-versioning-alt"} [package.source] type = "git" url = "https://github.com/bcgov/lear.git" reference = "HEAD" -resolved_reference = "76a8ad5a4f6665897dac21611558d7533f97a762" +resolved_reference = "deecf506fc9e76fb9e52e91a69db0db10d93f068" subdirectory = "python/common/business-registry-model" [[package]] @@ -2932,7 +2932,7 @@ rpds-py = ">=0.7.0" [[package]] name = "registry_schemas" -version = "2.18.71" +version = "2.18.74" description = "A short description of the project" optional = false python-versions = ">=3.6" @@ -2950,8 +2950,8 @@ strict-rfc3339 = "*" [package.source] type = "git" url = "https://github.com/bcgov/business-schemas.git" -reference = "2.18.71" -resolved_reference = "e8f9a9c1a2ae9a98f7fa8bb05a6f982cb765263c" +reference = "2.18.74" +resolved_reference = "5abdc89a0f6eb06cbbac9ad66b580d7c76190ebc" [[package]] name = "reportlab" diff --git a/queue_services/business-emailer/pyproject.toml b/queue_services/business-emailer/pyproject.toml index 175b5441c1..46e4e8cd9d 100644 --- a/queue_services/business-emailer/pyproject.toml +++ b/queue_services/business-emailer/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "business-emailer" -version = "0.1.7" +version = "0.1.13" description = "This module is the service worker for sending emails about entity related events." authors = ["Hrvoje Fekete "] license = "BSD-3-Clause" diff --git a/queue_services/business-emailer/src/business_emailer/email_processors/__init__.py b/queue_services/business-emailer/src/business_emailer/email_processors/__init__.py index 34e63c3b80..4db39f40c2 100644 --- a/queue_services/business-emailer/src/business_emailer/email_processors/__init__.py +++ b/queue_services/business-emailer/src/business_emailer/email_processors/__init__.py @@ -81,8 +81,7 @@ def get_recipients(option: str, filing_json: dict, token: str | None = None, fil ): recipients = f"{recipients}, {', '.join(party_emails)}" - elif not is_coop: - # only add business email recipient for non-coop + else: recipients = get_recipient_from_auth(identifier, token) return recipients @@ -99,7 +98,7 @@ def get_recipient_from_auth(identifier: str, token: str) -> str: f'{current_app.config.get("AUTH_URL")}/entities/{identifier}', headers=headers ) - contacts = contact_info.json()["contacts"] + contacts = (contact_info.json()).get("contacts") if not contacts: current_app.logger.error("Queue Error: No email in business (%s) profile to send output to.", identifier, exc_info=True) @@ -111,7 +110,7 @@ def get_recipient_from_auth(identifier: str, token: str) -> str: def get_user_email_from_auth(user_name: str, token: str) -> str: """Get user email from auth.""" user_info = get_user_from_auth(user_name, token) - contacts = user_info.json()["contacts"] + contacts = (user_info.json()).get("contacts") if not contacts: return user_info.json().get("email") # idir user @@ -181,6 +180,12 @@ def substitute_template_parts(template_code: str, file_type = "html") -> str: "business-number", "business-registry-footer", "business-tombstone", + "business-tombstone-basic", + "business-tombstone-out-filing", + "consent", + "consent-next-steps", + "continuation-application-details", + "out-details", "what-happens-next" ] else: @@ -191,7 +196,6 @@ def substitute_template_parts(template_code: str, file_type = "html") -> str: "business-info", "business-information", "consent-letter-information", - "continuation-application-details", "reg-business-info", "cra-notice", "nr-footer", @@ -219,25 +223,6 @@ def substitute_template_parts(template_code: str, file_type = "html") -> str: return template_code -def get_jurisdictions(identifier: str, token: str) -> dict: - """Get jurisdictions call.""" - headers = { - "Accept": "application/json", - "Authorization": f"Bearer {token}" - } - - response = requests.get( - f'{current_app.config.get("LEGAL_API_URL")}/mras/{identifier}', headers=headers - ) - if response.status_code != HTTPStatus.OK: - return None - try: - return response.json() - except Exception: - current_app.logger.error("Failed to get MRAS response") - return None - - def get_filing_document(business_identifier, filing_id, document_type, token, regenerate=False): """Get the filing documents.""" headers = { @@ -251,7 +236,7 @@ def get_filing_document(business_identifier, filing_id, document_type, token, re f'/documents/{document_type}', headers=headers, params=params ) - if document.status_code != HTTPStatus.OK: + if document.status_code not in [HTTPStatus.OK, HTTPStatus.CREATED]: current_app.logger.error("Failed to get %s pdf for filing: %s", document_type, filing_id) return None try: @@ -292,28 +277,21 @@ def _add_filing_document_pdf( # noqa: PLR0913 token: str, business: dict, filing: Filing, + file_attachment_name: str | None = None, regenerate=False ): """Add the specified filing document pdf to the pdfs list.""" # File name - file_name = (document_type[0].upper() + " ".join(re.findall("[a-zA-Z][^A-Z]*", document_type[1:]))).replace(" Of ", " of ") + if not (file_name := file_attachment_name): + file_name = (document_type[0].upper() + " ".join(re.findall("[a-zA-Z][^A-Z]*", document_type[1:]))).replace(" Of ", " of ") + if document_type == "annualReport" and (ar_date := filing.filing_json["filing"].get("annualReport", {}).get("annualReportDate")): file_name = f"{ar_date[:4]} {file_name}" - elif document_type == "changeOfAddress": - file_name = "Address Change" - elif document_type == "changeOfDirectors": - file_name = "Director Change" - elif document_type == "registration": - file_name = "Statement of Registration" - elif document_type == "continuationIn": - file_name = "Continuation Application" - elif document_type == "amalgamationApplication": - amalgamation_application_names = { - Amalgamation.AmalgamationTypes.regular.name: "Amalgamation Application (Regular)", - Amalgamation.AmalgamationTypes.vertical.name: "Amalgamation Application Short-form (Vertical)", - Amalgamation.AmalgamationTypes.horizontal.name: "Amalgamation Application Short-form (Horizontal)" - } - file_name = amalgamation_application_names.get(filing.filing_sub_type, file_name) + + if (filing.filing_type == "dissolution" and business.get("legalType") == Business.LegalTypes.COOP.value + and document_type in ["affidavit", "specialResolution"]): + # coop dissolution affidavit and special resolution attachments are certified copies + file_name = f"Certified {file_name}" # Get pdf and add it to the list filing_pdf_encoded = get_filing_document(business["identifier"], filing.id, document_type, token, regenerate=regenerate) @@ -326,70 +304,37 @@ def _add_filing_document_pdf( # noqa: PLR0913 "attachOrder": str(attach_order) } ) - return attach_order + 1 + attach_order += 1 - -def _add_receipt_pdf( # noqa: PLR0913 - pdfs: list[dict], - attach_order: int, - token: str, - business: dict, - filing: Filing, - filing_date_time: str, - effective_date: str -): - """Add the filing receipt pdf to the pdfs list.""" - headers = { - "Accept": "application/pdf", - "Authorization": f"Bearer {token}" - } - if not (corp_name := business.get("legalName")): # pylint: disable=superfluous-parens - legal_type = business.get("legalType") - corp_name = Business.BUSINESSES.get(legal_type, {}).get("numberedDescription") - - receipt = requests.post( - f'{current_app.config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - json={ - "corpName": corp_name, - "filingDateTime": filing_date_time, - "effectiveDateTime": effective_date if effective_date != filing_date_time else "", - "filingIdentifier": str(filing.id), - "businessNumber": business.get("taxId", "") - }, - headers=headers - ) - if receipt.status_code != HTTPStatus.CREATED: - current_app.logger.error("Failed to get receipt pdf for filing: %s", filing.id) - else: - receipt_encoded = base64.b64encode(receipt.content) - pdfs.append( - { - "fileName": "Receipt.pdf", - "fileBytes": receipt_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - return attach_order + 1 + return attach_order def get_pdfs( # noqa: PLR0913 token: str, business: dict, filing: Filing, - filing_date_time: str, - effective_date: str, extra_pdf_type_list: list[str], + filing_attachment_name: str | None, regenerate=False ) -> list: """Get the pdfs for the filing output.""" pdfs = [] attach_order = 1 - # add filing application document - attach_order = _add_filing_document_pdf(pdfs, attach_order, filing.filing_type, token, business, filing, regenerate=regenerate) + filings_with_unimplemented_outputs = ["amalgamationOut", "consentAmalgamationOut", "continuationOut"] + receipt_only_sub_filings = [ + ("changeOfLiquidators", "liquidationReport"), + ("changeOfReceivers", "appointReceiver"), + ("changeOfReceivers", "ceaseReceiver"), + ("changeOfReceivers", "changeAddressReceiver"), + ] + + filing_key = (filing.filing_type, filing.filing_sub_type) + if filing.filing_type not in filings_with_unimplemented_outputs and filing_key not in receipt_only_sub_filings: + # add filing application document + attach_order = _add_filing_document_pdf(pdfs, attach_order, filing.filing_type, token, business, filing, filing_attachment_name, regenerate=regenerate) # add extra documents for pdf_type in extra_pdf_type_list: attach_order = _add_filing_document_pdf(pdfs, attach_order, pdf_type, token, business, filing, regenerate=regenerate) # add receipt - attach_order = _add_receipt_pdf(pdfs, attach_order, token, business, filing, filing_date_time, effective_date) + attach_order = _add_filing_document_pdf(pdfs, attach_order, "receipt", token, business, filing, regenerate=regenerate) return pdfs diff --git a/queue_services/business-emailer/src/business_emailer/email_processors/appoint_receiver_notification.py b/queue_services/business-emailer/src/business_emailer/email_processors/appoint_receiver_notification.py deleted file mode 100644 index 709d023b8a..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_processors/appoint_receiver_notification.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright © 2025 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Email processing rules and actions for Appoint Receiver notifications.""" -import base64 -import re -from http import HTTPStatus -from pathlib import Path - -import requests -from flask import current_app -from jinja2 import Template - -from business_emailer.email_processors import ( - get_filing_document, - get_filing_info, - get_recipient_from_auth, - substitute_template_parts, -) -from business_model.models import Business, Filing - - -def process(email_info: dict, token: str) -> dict: # pylint: disable=too-many-locals - """Build the email for Appoint Receiver notification.""" - current_app.logger.debug("appoint_receiver: %s", email_info) - # get template and fill in parts - filing_type = email_info["type"] - - # get template variables from filing - filing, business, leg_tmz_filing_date, leg_tmz_effective_date = get_filing_info(email_info["filingId"]) - - template = Path( - f"{current_app.config.get("TEMPLATE_PATH")}/APPOINT_RECVR.html" - ).read_text() - filled_template = substitute_template_parts(template) - # render template with vars - jnja_template = Template(filled_template, autoescape=True) - filing_data = (filing.json)["filing"][f"{filing_type}"] - filing_name = filing.filing_type[0].upper() + " ".join(re.findall("[a-zA-Z][^A-Z]*", filing.filing_type[1:])) - html_out = jnja_template.render( - business=business, - filing=filing_data, - header=(filing.json)["filing"]["header"], - filing_date_time=leg_tmz_filing_date, - effective_date_time=leg_tmz_effective_date, - entity_dashboard_url=current_app.config.get("DASHBOARD_URL") + - (filing.json)["filing"]["business"].get("identifier", ""), - email_header=filing_name.upper(), - filing_type=filing_type - ) - - # get attachments - pdfs = _get_pdfs(token, business, filing, leg_tmz_filing_date, leg_tmz_effective_date) - - # get recipients - identifier = filing.filing_json["filing"]["business"]["identifier"] - recipients = [] - recipients.append(get_recipient_from_auth(identifier, token)) - - recipients = list(set(recipients)) - recipients = ", ".join(filter(None, recipients)).strip() - - # assign subject - subject = "Receiver appointed" - legal_name = business.get("legalName", None) - legal_name = "Numbered Company" if legal_name.startswith(identifier) else legal_name - subject = f"{legal_name} - {subject}" if legal_name else subject - - return { - "recipients": recipients, - "requestBy": "BCRegistries@gov.bc.ca", - "content": { - "subject": subject, - "body": f"{html_out}", - "attachments": pdfs - } - } - - -def _get_pdfs( - token: str, - business: dict, - filing: Filing, - filing_date_time: str, - effective_date: str) -> list: - """Get the PDFs for the Appoint Receiver output.""" - pdfs = [] - attach_order = 1 - headers = { - "Accept": "application/pdf", - "Authorization": f"Bearer {token}" - } - - # add filing PDF - filing_pdf_type = "appointReceiver" - filing_pdf_encoded = get_filing_document(business["identifier"], filing.id, filing_pdf_type, token) - if filing_pdf_encoded: - pdfs.append( - { - "fileName": "Appoint Receiver.pdf", - "fileBytes": filing_pdf_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - # add receipt PDF - corp_name = business.get("legalName") - if business.get("identifier").startswith("T"): - business_data = None - else: - business_data = Business.find_by_internal_id(filing.business_id) - receipt = requests.post( - f"{current_app.config.get("PAY_API_URL")}/{filing.payment_token}/receipts", - json={ - "corpName": corp_name, - "filingDateTime": filing_date_time, - "effectiveDateTime": effective_date if effective_date else "", - "filingIdentifier": str(filing.id), - "businessNumber": business_data.tax_id if business_data and business_data.tax_id else "" - }, headers=headers) - - if receipt.status_code != HTTPStatus.CREATED: - current_app.logger.error("Failed to get receipt pdf for filing: %s", filing.id) - else: - receipt_encoded = base64.b64encode(receipt.content) - pdfs.append( - { - "fileName": "Receipt.pdf", - "fileBytes": receipt_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - }) - attach_order += 1 - return pdfs diff --git a/queue_services/business-emailer/src/business_emailer/email_processors/cease_receiver_notification.py b/queue_services/business-emailer/src/business_emailer/email_processors/cease_receiver_notification.py deleted file mode 100644 index 53ba4e39f8..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_processors/cease_receiver_notification.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright © 2025 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Email processing rules and actions for Cease Receiver notifications.""" -import base64 -import re -from http import HTTPStatus -from pathlib import Path - -import requests -from flask import current_app -from jinja2 import Template - -from business_emailer.email_processors import ( - get_filing_document, - get_filing_info, - get_recipient_from_auth, - substitute_template_parts, -) -from business_model.models import Business, Filing - - -def process(email_info: dict, token: str) -> dict: # pylint: disable=too-many-locals - """Build the email for Cease Receiver notification.""" - current_app.logger.debug("cease_receiver: %s", email_info) - # get template and fill in parts - filing_type = email_info["type"] - - # get template variables from filing - filing, business, leg_tmz_filing_date, leg_tmz_effective_date = get_filing_info(email_info["filingId"]) - - template = Path( - f'{current_app.config.get("TEMPLATE_PATH")}/CEASE_RECVR.html' - ).read_text() - filled_template = substitute_template_parts(template) - # render template with vars - jnja_template = Template(filled_template, autoescape=True) - filing_data = (filing.json)["filing"][f"{filing_type}"] - filing_name = filing.filing_type[0].upper() + " ".join(re.findall("[a-zA-Z][^A-Z]*", filing.filing_type[1:])) - html_out = jnja_template.render( - business=business, - filing=filing_data, - header=(filing.json)["filing"]["header"], - filing_date_time=leg_tmz_filing_date, - effective_date_time=leg_tmz_effective_date, - entity_dashboard_url=current_app.config.get("DASHBOARD_URL") + - (filing.json)["filing"]["business"].get("identifier", ""), - email_header=filing_name.upper(), - filing_type=filing_type - ) - - # get attachments - pdfs = _get_pdfs(token, business, filing, leg_tmz_filing_date, leg_tmz_effective_date) - - # get recipients - identifier = filing.filing_json["filing"]["business"]["identifier"] - recipients = [] - recipients.append(get_recipient_from_auth(identifier, token)) - - recipients = list(set(recipients)) - recipients = ", ".join(filter(None, recipients)).strip() - - # assign subject - subject = "Receiver Ceased" - legal_name = business.get("legalName", None) - legal_name = "Numbered Company" if legal_name.startswith(identifier) else legal_name - subject = f"{legal_name} - {subject}" if legal_name else subject - - return { - "recipients": recipients, - "requestBy": "BCRegistries@gov.bc.ca", - "content": { - "subject": subject, - "body": f"{html_out}", - "attachments": pdfs - } - } - - -def _get_pdfs( - token: str, - business: dict, - filing: Filing, - filing_date_time: str, - effective_date: str) -> list: - """Get the PDFs for the Cease Receiver output.""" - pdfs = [] - attach_order = 1 - headers = { - "Accept": "application/pdf", - "Authorization": f"Bearer {token}" - } - - # add filing PDF - filing_pdf_type = "ceaseReceiver" - filing_pdf_encoded = get_filing_document(business["identifier"], filing.id, filing_pdf_type, token) - if filing_pdf_encoded: - pdfs.append( - { - "fileName": "Cease Receiver.pdf", - "fileBytes": filing_pdf_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - # add receipt PDF - corp_name = business.get("legalName") - if business.get("identifier").startswith("T"): - business_data = None - else: - business_data = Business.find_by_internal_id(filing.business_id) - receipt = requests.post( - f'{current_app.config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - json={ - "corpName": corp_name, - "filingDateTime": filing_date_time, - "effectiveDateTime": effective_date if effective_date else "", - "filingIdentifier": str(filing.id), - "businessNumber": business_data.tax_id if business_data and business_data.tax_id else "" - }, headers=headers) - - if receipt.status_code != HTTPStatus.CREATED: - current_app.logger.error("Failed to get receipt pdf for filing: %s", filing.id) - else: - receipt_encoded = base64.b64encode(receipt.content) - pdfs.append( - { - "fileName": "Receipt.pdf", - "fileBytes": receipt_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - }) - attach_order += 1 - return pdfs diff --git a/queue_services/business-emailer/src/business_emailer/email_processors/consent_continuation_out_notification.py b/queue_services/business-emailer/src/business_emailer/email_processors/consent_continuation_out_notification.py deleted file mode 100644 index a77114f3f2..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_processors/consent_continuation_out_notification.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright © 2023 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Email processing rules and actions for CCO notifications.""" -from __future__ import annotations - -import base64 -import re -from http import HTTPStatus -from pathlib import Path - -import requests -from flask import current_app -from jinja2 import Template - -from business_emailer.email_processors import ( - get_filing_document, - get_filing_info, - get_recipient_from_auth, - substitute_template_parts, -) -from business_model.models import Business, Filing, UserRoles - - -def _get_pdfs( - token: str, - business: dict, - filing: Filing, - filing_date_time: str, - effective_date: str -) -> list: - # pylint: disable=too-many-locals, too-many-branches, too-many-statements, too-many-arguments - """Get the pdfs for the Consent Continuation Out output.""" - pdfs = [] - attach_order = 1 - headers = { - "Accept": "application/pdf", - "Authorization": f"Bearer {token}" - } - - # add filing pdf - filing_pdf_type = "letterOfConsent" - filing_pdf_encoded = get_filing_document(business["identifier"], filing.id, filing_pdf_type, token) - if filing_pdf_encoded: - pdfs.append( - { - "fileName": "Letter of Consent.pdf", - "fileBytes": filing_pdf_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - # add receipt pdf - corp_name = business.get("legalName") - business_data = Business.find_by_internal_id(filing.business_id) - receipt = requests.post( - f'{current_app.config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - json={ - "corpName": corp_name, - "filingDateTime": filing_date_time, - "effectiveDateTime": effective_date if effective_date != filing_date_time else "", - "filingIdentifier": str(filing.id), - "businessNumber": business_data.tax_id if business_data and business_data.tax_id else "" - }, - headers=headers - ) - if receipt.status_code != HTTPStatus.CREATED: - current_app.logger.error("Failed to get receipt pdf for filing: %s", filing.id) - else: - receipt_encoded = base64.b64encode(receipt.content) - pdfs.append( - { - "fileName": "Receipt.pdf", - "fileBytes": receipt_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - return pdfs - - -def process(email_info: dict, token: str) -> dict: # pylint: disable=too-many-locals, too-many-branches - """Build the email for Consent Continuation Out notification.""" - current_app.logger.debug("consent_continuation_out_notification: %s", email_info) - # get template and fill in parts - filing_type, status = email_info["type"], email_info["option"] - # get template vars from filing - filing, business, leg_tmz_filing_date, leg_tmz_effective_date = get_filing_info(email_info["filingId"]) - filing_name = filing.filing_type[0].upper() + " ".join(re.findall("[a-zA-Z][^A-Z]*", filing.filing_type[1:])) - - template = Path( - f'{current_app.config.get("TEMPLATE_PATH")}/CCO-{status}.html' - ).read_text() - filled_template = substitute_template_parts(template) - # render template with vars - jnja_template = Template(filled_template, autoescape=True) - filing_data = (filing.json)["filing"][f"{filing_type}"] - html_out = jnja_template.render( - business=business, - filing=filing_data, - header=(filing.json)["filing"]["header"], - filing_date_time=leg_tmz_filing_date, - effective_date_time=leg_tmz_effective_date, - entity_dashboard_url=current_app.config.get("DASHBOARD_URL") + - (filing.json)["filing"]["business"].get("identifier", ""), - email_header=filing_name.upper(), - filing_type=filing_type - ) - - # get attachments - pdfs = _get_pdfs(token, business, filing, leg_tmz_filing_date, leg_tmz_effective_date) - - # get recipients - identifier = filing.filing_json["filing"]["business"]["identifier"] - recipients = [] - recipients.append(get_recipient_from_auth(identifier, token)) - - if filing.submitter_roles and UserRoles.staff in filing.submitter_roles: - # when staff file a CCO documentOptionalEmail may contain completing party email - recipients.append(filing.filing_json["filing"]["header"].get("documentOptionalEmail")) - - recipients = list(set(recipients)) - recipients = ", ".join(filter(None, recipients)).strip() - - # assign subject - subject = "Confirmation of Filing from the Business Registry" - - legal_name = business.get("legalName", None) - subject = f"{legal_name} - {subject}" if legal_name else subject - - return { - "recipients": recipients, - "requestBy": "BCRegistries@gov.bc.ca", - "content": { - "subject": subject, - "body": f"{html_out}", - "attachments": pdfs - } - } diff --git a/queue_services/business-emailer/src/business_emailer/email_processors/continuation_in_notification.py b/queue_services/business-emailer/src/business_emailer/email_processors/continuation_authorization_notification.py similarity index 83% rename from queue_services/business-emailer/src/business_emailer/email_processors/continuation_in_notification.py rename to queue_services/business-emailer/src/business_emailer/email_processors/continuation_authorization_notification.py index adb669365f..6aba07dbaa 100644 --- a/queue_services/business-emailer/src/business_emailer/email_processors/continuation_in_notification.py +++ b/queue_services/business-emailer/src/business_emailer/email_processors/continuation_authorization_notification.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Email processing rules and actions for Continuation In notifications.""" +"""Email processing rules and actions for Continuation Authorization notifications.""" from __future__ import annotations import re @@ -29,6 +29,13 @@ ) from business_model.models import Business, Filing, ReviewResult +CONTINUATION_IN_AUTHORIZATION_TEMPLATE_SUFFIXES = { + Filing.Status.APPROVED.value: "approved", + Filing.Status.AWAITING_REVIEW.value: "awaitingReview", + Filing.Status.CHANGE_REQUESTED.value: "changeRequested", + Filing.Status.REJECTED.value: "rejected", + "RESUBMITTED": "resubmitted", +} def _get_pdfs( status: str, @@ -78,7 +85,7 @@ def process(email_info: dict, token: str) -> dict: # pylint: disable=too-many-l numbered_description = Business.BUSINESSES.get(legal_type, {}).get("numberedDescription") review_result = ReviewResult.get_last_review_result(filing.id) # encode newlines in review comment only - latest_review_comment = review_result.comments.replace("\n", "\\n") if review_result else None + latest_review_comment = review_result.comments if review_result else None # compute Foreign Jurisdiction string as in report.py and business_document.py country_code = filing_data["foreignJurisdiction"]["country"] @@ -87,12 +94,13 @@ def process(email_info: dict, token: str) -> dict: # pylint: disable=too-many-l region = None if region_code and region_code.upper() != "FEDERAL": region = pycountry.subdivisions.get(code=f"{country_code}-{region_code}") - foreign_jurisdiction = f"{region.name}, {country.name}" if region else country.name + foreign_jurisdiction = region.name if region else country.name # get template and fill in parts - template = (Path(f'{current_app.config.get("TEMPLATE_PATH")}/CONT-IN-{status}.html') + template_suffix = CONTINUATION_IN_AUTHORIZATION_TEMPLATE_SUFFIXES[status] + template = (Path(f'{current_app.config.get("TEMPLATE_PATH")}/{filing_type}-{template_suffix}.md') .read_text(encoding="utf-8")) - filled_template = substitute_template_parts(template) + filled_template = substitute_template_parts(template, "md") jnja_template = Template(filled_template, autoescape=True) # render template with vars @@ -125,15 +133,15 @@ def process(email_info: dict, token: str) -> dict: # pylint: disable=too-many-l # assign subject legal_name = business.get("legalName", None) if status == Filing.Status.APPROVED.value: - subject = "Authorization Approved" + subject = "Continuation Authorization Results" if status == Filing.Status.REJECTED.value: - subject = "Authorization Rejected" + subject = "Continuation Authorization Results" elif status == Filing.Status.AWAITING_REVIEW.value: - subject = "Authorization Documents Received" + subject = "Continuation Authorization Documents Received" elif status == Filing.Status.CHANGE_REQUESTED.value: - subject = "Changes Needed to Authorization" + subject = "Change Requested for your Continuation Authorization" elif status == "RESUBMITTED": - subject = "Authorization Updates Received" + subject = "Updated Continuation Authorization Documents Received" subject = f"{legal_name} - {subject}" if legal_name else subject @@ -145,4 +153,4 @@ def process(email_info: dict, token: str) -> dict: # pylint: disable=too-many-l "body": f"{html_out}", "attachments": pdfs } - } \ No newline at end of file + } diff --git a/queue_services/business-emailer/src/business_emailer/email_processors/continuation_out_notification.py b/queue_services/business-emailer/src/business_emailer/email_processors/continuation_out_notification.py deleted file mode 100644 index ef8e7f942e..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_processors/continuation_out_notification.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright © 2023 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Email processing rules and actions for Continuation Out notifications.""" -from __future__ import annotations - -import base64 -import re -from http import HTTPStatus -from pathlib import Path - -import requests -from flask import current_app -from jinja2 import Template - -from business_emailer.email_processors import get_filing_info, get_recipient_from_auth, substitute_template_parts -from business_model.models import Business, Filing, UserRoles - - -def _get_pdfs( - token: str, - business: dict, - filing: Filing, - filing_date_time: str, - effective_date: str) -> list: - # pylint: disable=too-many-locals, too-many-branches, too-many-statements, too-many-arguments - """Get the pdfs for the Continuation Out output.""" - pdfs = [] - attach_order = 1 - headers = { - "Accept": "application/pdf", - "Authorization": f"Bearer {token}" - } - - # add receipt pdf - corp_name = business.get("legalName") - business_data = Business.find_by_internal_id(filing.business_id) - receipt = requests.post( - f'{current_app.config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - json={ - "corpName": corp_name, - "filingDateTime": filing_date_time, - "effectiveDateTime": effective_date if effective_date != filing_date_time else "", - "filingIdentifier": str(filing.id), - "businessNumber": business_data.tax_id if business_data and business_data.tax_id else "" - }, - headers=headers - ) - if receipt.status_code != HTTPStatus.CREATED: - current_app.logger.error("Failed to get receipt pdf for filing: %s", filing.id) - else: - receipt_encoded = base64.b64encode(receipt.content) - pdfs.append( - { - "fileName": "Receipt.pdf", - "fileBytes": receipt_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - return pdfs - - -def process(email_info: dict, token: str) -> dict: # pylint: disable=too-many-locals, too-many-branches - """Build the email for Continuation Out notification.""" - current_app.logger.debug("continuation_out_notification: %s", email_info) - # get template and fill in parts - filing_type, status = email_info["type"], email_info["option"] - # get template vars from filing - filing, business, leg_tmz_filing_date, leg_tmz_effective_date = get_filing_info(email_info["filingId"]) - filing_name = filing.filing_type[0].upper() + " ".join(re.findall("[a-zA-Z][^A-Z]*", filing.filing_type[1:])) - - template = Path( - f'{current_app.config.get("TEMPLATE_PATH")}/CO-{status}.html' - ).read_text() - filled_template = substitute_template_parts(template) - # render template with vars - jnja_template = Template(filled_template, autoescape=True) - filing_data = (filing.json)["filing"][f"{filing_type}"] - html_out = jnja_template.render( - business=business, - filing=filing_data, - header=(filing.json)["filing"]["header"], - filing_date_time=leg_tmz_filing_date, - effective_date_time=leg_tmz_effective_date, - entity_dashboard_url=current_app.config.get("DASHBOARD_URL") + - (filing.json)["filing"]["business"].get("identifier", ""), - email_header=filing_name.upper(), - filing_type=filing_type - ) - - # get attachments - pdfs = _get_pdfs(token, business, filing, leg_tmz_filing_date, leg_tmz_effective_date) - - # get recipients - identifier = filing.filing_json["filing"]["business"]["identifier"] - recipients = [] - recipients.append(get_recipient_from_auth(identifier, token)) - - if filing.submitter_roles and UserRoles.staff in filing.submitter_roles: - # when staff file a CO documentOptionalEmail may contain completing party email - recipients.append(filing.filing_json["filing"]["header"].get("documentOptionalEmail")) - - recipients = list(set(recipients)) - recipients = ", ".join(filter(None, recipients)).strip() - - # assign subject - subject = "Confirmation of Filing from the Business Registry" - - legal_name = business.get("legalName", None) - subject = f"{legal_name} - {subject}" if legal_name else subject - - return { - "recipients": recipients, - "requestBy": "BCRegistries@gov.bc.ca", - "content": { - "subject": subject, - "body": f"{html_out}", - "attachments": pdfs - } - } diff --git a/queue_services/business-emailer/src/business_emailer/email_processors/correction_notification.py b/queue_services/business-emailer/src/business_emailer/email_processors/correction_notification.py deleted file mode 100644 index d3b13737fc..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_processors/correction_notification.py +++ /dev/null @@ -1,259 +0,0 @@ -# Copyright © 2022 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Email processing rules and actions for Correction notifications.""" -from __future__ import annotations - -import base64 -import re -from http import HTTPStatus -from pathlib import Path - -import requests -from flask import current_app -from jinja2 import Template - -from business_emailer.email_processors import get_filing_document, get_filing_info, substitute_template_parts -from business_emailer.email_processors.special_resolution_helper import get_completed_pdfs -from business_emailer.filing_helper import is_special_resolution_correction_by_filing_json -from business_model.models import Business, Filing - - -# copied and pasted from legal_api.core.filing_helper -def _is_special_resolution_correction_by_meta_data(filing): - """Check whether it is a special resolution correction.""" - # Check by using the meta_data, this is more permanent than the filing json. - # This is used by reports (after the filer). - if filing.meta_data and (correction_meta_data := filing.meta_data.get("correction")): - # Note these come from the corrections filer. - sr_correction_meta_data_keys = ["hasResolution", "memorandumInResolution", "rulesInResolution", - "uploadNewRules", "uploadNewMemorandum", - "toCooperativeAssociationType", "toLegalName"] - for key in sr_correction_meta_data_keys: - if key in correction_meta_data: - return True - return False - - -def _get_pdfs( # noqa: PLR0913 - status: str, - token: str, - business: dict, - filing: Filing, - filing_date_time: str, - effective_date: str, - name_changed: bool) -> list: - # pylint: disable=too-many-locals, too-many-branches, too-many-statements, too-many-arguments - """Get the outputs for the correction notification.""" - pdfs = [] - attach_order = 1 - headers = { - "Accept": "application/pdf", - "Authorization": f"Bearer {token}" - } - legal_type = business.get("legalType") - is_cp_special_resolution = legal_type == "CP" and is_special_resolution_correction_by_filing_json( - filing.filing_json["filing"] - ) - - if status == Filing.Status.PAID.value: - # add filing pdf - filing_pdf_type = "correction" - filing_pdf_encoded = get_filing_document(business["identifier"], filing.id, filing_pdf_type, token) - if filing_pdf_encoded: - pdfs.append( - { - "fileName": "Register Correction Application.pdf", - "fileBytes": filing_pdf_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - corp_name = business.get("legalName") - receipt = requests.post( - f'{current_app.config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - json={ - "corpName": corp_name, - "filingDateTime": filing_date_time, - "effectiveDateTime": effective_date if effective_date != filing_date_time else "", - "filingIdentifier": str(filing.id), - "businessNumber": business.get("taxId", "") - }, - headers=headers - ) - if receipt.status_code != HTTPStatus.CREATED: - current_app.logger.error("Failed to get receipt pdf for filing: %s", filing.id) - else: - receipt_encoded = base64.b64encode(receipt.content) - pdfs.append( - { - "fileName": "Receipt.pdf", - "fileBytes": receipt_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - elif status == Filing.Status.COMPLETED.value: - if legal_type in ("SP", "GP"): - # add corrected registration statement - certificate_pdf_type = "correctedRegistrationStatement" - certificate_encoded = get_filing_document(business["identifier"], filing.id, certificate_pdf_type, token) - if certificate_encoded: - pdfs.append( - { - "fileName": "Corrected - Registration Statement.pdf", - "fileBytes": certificate_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - elif legal_type in Business.CORPS: - # add notice of articles - noa_pdf_type = "noticeOfArticles" - noa_encoded = get_filing_document(business["identifier"], filing.id, noa_pdf_type, token) - if noa_encoded: - pdfs.append( - { - "fileName": "Notice of Articles.pdf", - "fileBytes": noa_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - elif is_cp_special_resolution: - rules_changed = bool(filing.filing_json["filing"]["correction"].get("rulesFileKey")) - memorandum_changed = bool(filing.filing_json["filing"]["correction"].get("memorandumFileKey")) - pdfs = get_completed_pdfs(token, business, filing, name_changed, - rules_changed=rules_changed, memorandum_changed=memorandum_changed) - return pdfs - - -def _get_template(prefix: str, status: str, filing_type: str, filing: Filing, # pylint: disable=too-many-arguments # noqa: PLR0913 - business: dict, leg_tmz_filing_date: str, leg_tmz_effective_date: str, - name_changed: bool) -> str: - """Return rendered template.""" - filing_name = filing.filing_type[0].upper() + " ".join(re.findall("[a-zA-Z][^A-Z]*", filing.filing_type[1:])) - - template = Path( - f'{current_app.config.get("TEMPLATE_PATH")}/{prefix}-CRCTN-{status}.html' - ).read_text() - filled_template = substitute_template_parts(template) - # render template with vars - jnja_template = Template(filled_template, autoescape=True) - filing_data = (filing.json)["filing"][f"{filing_type}"] - html_out = jnja_template.render( - business=business, - filing=filing_data, - header=(filing.json)["filing"]["header"], - filing_date_time=leg_tmz_filing_date, - effective_date_time=leg_tmz_effective_date, - entity_dashboard_url=current_app.config.get("DASHBOARD_URL") + business.get("identifier", ""), - email_header=filing_name.upper(), - filing_type=filing_type, - name_changed=name_changed - ) - - return html_out - - -def _get_recipients(filing: Filing) -> list: - """Return recipients list.""" - recipients = [] - for party in filing.filing_json["filing"]["correction"].get("parties", []): - for role in party["roles"]: - if role["roleType"] in ("Partner", "Proprietor", "Completing Party"): - recipients.append(party["officer"].get("email")) - break - - if filing.filing_json["filing"]["correction"].get("contactPoint"): - recipients.append(filing.filing_json["filing"]["correction"]["contactPoint"]["email"]) - - recipients = list(set(recipients)) - recipients = list(filter(None, recipients)) - return recipients - - -def get_subject(status: str, prefix: str, business: dict) -> str: - """Return subject.""" - subjects = { - Filing.Status.PAID.value: "Confirmation of correction" if prefix == "CP-SR" else - "Confirmation of Filing from the Business Registry", - Filing.Status.COMPLETED.value: "Correction Documents from the Business Registry" - } - - subject = subjects[status] - if not subject: # fallback case - should never happen - subject = "Notification from the BC Business Registry" - - legal_name = business.get("legalName") - subject = f"{legal_name} - {subject}" if legal_name else subject - - return subject - - -def process(email_info: dict, token: str) -> dict | None: # pylint: disable=too-many-locals, , too-many-branches - """Build the email for Correction notification.""" - current_app.logger.debug("correction_notification: %s", email_info) - # get template and fill in parts - filing_type, status = email_info["type"], email_info["option"] - # get template vars from filing - filing, business, leg_tmz_filing_date, leg_tmz_effective_date = get_filing_info(email_info["filingId"]) - - prefix = "BC" - legal_type = business.get("legalType", None) - name_changed = False - - if legal_type in ["SP", "GP"]: - prefix = "FIRM" - elif legal_type in Business.CORPS: - original_filing_type = filing.filing_json["filing"]["correction"]["correctedFilingType"] - if original_filing_type in ["annualReport", "changeOfAddress", "changeOfDirectors"]: - return None - elif legal_type == "CP" and is_special_resolution_correction_by_filing_json( - filing.filing_json["filing"] - ): - prefix = "CP-SR" - name_changed = "requestType" in filing.filing_json["filing"]["correction"].get("nameRequest", {}) - else: - return None - - html_out = _get_template(prefix, # pylint: disable=too-many-function-args - status, - filing_type, - filing, - business, - leg_tmz_filing_date, - leg_tmz_effective_date, - name_changed) - # get attachments - pdfs = _get_pdfs(status, token, business, filing, leg_tmz_filing_date, leg_tmz_effective_date, name_changed) - # get recipients - recipients = _get_recipients(filing) - recipients = ", ".join(filter(None, recipients)).strip() - # assign subject - subject = get_subject(email_info["option"], prefix, business) - - return { - "recipients": recipients, - "requestBy": "BCRegistries@gov.bc.ca", - "content": { - "subject": subject, - "body": f"{html_out}", - "attachments": pdfs - } - } diff --git a/queue_services/business-emailer/src/business_emailer/email_processors/dissolution_notification.py b/queue_services/business-emailer/src/business_emailer/email_processors/dissolution_notification.py deleted file mode 100644 index b89c96057b..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_processors/dissolution_notification.py +++ /dev/null @@ -1,249 +0,0 @@ -# Copyright © 2021 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Email processing rules and actions for Dissolution Application notifications.""" -from __future__ import annotations - -import base64 -import re -from http import HTTPStatus -from pathlib import Path - -import requests -from flask import current_app -from jinja2 import Template - -from business_emailer.email_processors import ( - get_filing_document, - get_filing_info, - get_recipient_from_auth, - get_user_email_from_auth, - substitute_template_parts, -) -from business_model.models import Business, Filing, UserRoles - - -def _get_pdfs( # noqa: PLR0913,PLR0912 - status: str, - token: str, - business: dict, - filing: Filing, - filing_date_time: str, - effective_date: str -) -> list: - # pylint: disable=too-many-locals, too-many-branches, too-many-statements, too-many-arguments - """Get the pdfs for the dissolution output.""" - pdfs = [] - attach_order = 1 - headers = { - "Accept": "application/pdf", - "Authorization": f"Bearer {token}" - } - legal_type = business.get("legalType") - - if status == Filing.Status.PAID.value: - # add filing pdf - if legal_type not in ["SP", "GP"]: - filing_pdf_type = "dissolution" - filing_pdf_encoded = get_filing_document(business["identifier"], filing.id, filing_pdf_type, token) - if filing_pdf_encoded: - pdfs.append( - { - "fileName": "Voluntary Dissolution Application.pdf", - "fileBytes": filing_pdf_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - corp_name = business.get("legalName") - business_data = Business.find_by_internal_id(filing.business_id) - receipt = requests.post( - f'{current_app.config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - json={ - "corpName": corp_name, - "filingDateTime": filing_date_time, - "effectiveDateTime": effective_date if effective_date != filing_date_time else "", - "filingIdentifier": str(filing.id), - "businessNumber": business_data.tax_id if business_data and business_data.tax_id else "" - }, - headers=headers - ) - if receipt.status_code != HTTPStatus.CREATED: - current_app.logger.error("Failed to get receipt pdf for filing: %s", filing.id) - else: - receipt_encoded = base64.b64encode(receipt.content) - pdfs.append( - { - "fileName": "Receipt.pdf", - "fileBytes": receipt_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - elif status == Filing.Status.COMPLETED.value: - if legal_type in ["SP", "GP"]: - filing_pdf_type = "dissolution" - filing_pdf_encoded = get_filing_document(business["identifier"], filing.id, filing_pdf_type, token) - if filing_pdf_encoded: - pdfs.append( - { - "fileName": "Statement of Dissolution.pdf", - "fileBytes": filing_pdf_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - else: - if filing.filing_sub_type != "administrative": - # add certificateOfDissolution, suppress certificate of dissolution for admin dissolution - certificate_pdf_type = "certificateOfDissolution" - certificate_encoded = get_filing_document(business["identifier"], filing.id, - certificate_pdf_type, token) - if certificate_encoded: - pdfs.append( - { - "fileName": "Certificate of Dissolution.pdf", - "fileBytes": certificate_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - if legal_type == Business.LegalTypes.COOP.value: - # certifiedAffidavit - certified_affidavit_pdf_type = "affidavit" - certified_affidavit_encoded = get_filing_document(business["identifier"], filing.id, - certified_affidavit_pdf_type, token) - if certified_affidavit_encoded: - pdfs.append( - { - "fileName": "Certified Affidavit.pdf", - "fileBytes": certified_affidavit_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - # specialResolution - special_resolution_pdf_type = "specialResolution" - special_resolution_encoded = get_filing_document(business["identifier"], filing.id, - special_resolution_pdf_type, token) - if special_resolution_encoded: - pdfs.append( - { - "fileName": "Certified Special Resolution.pdf", - "fileBytes": special_resolution_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - return pdfs - - -def process(email_info: dict, token: str) -> dict: # pylint: disable=too-many-locals, , too-many-branches, # noqa: PLR0912 - """Build the email for Dissolution notification.""" - current_app.logger.debug("dissolution_notification: %s", email_info) - # get template and fill in parts - filing_type, status = email_info["type"], email_info["option"] - # get template vars from filing - filing, business, leg_tmz_filing_date, leg_tmz_effective_date = get_filing_info(email_info["filingId"]) - filing_name = filing.filing_type[0].upper() + " ".join(re.findall("[a-zA-Z][^A-Z]*", filing.filing_type[1:])) - legal_type = business.get("legalType", None) - - template = Path( - f'{current_app.config.get("TEMPLATE_PATH")}/DIS-{status}.html' - ).read_text() - filled_template = substitute_template_parts(template) - # render template with vars - jnja_template = Template(filled_template, autoescape=True) - filing_data = (filing.json)["filing"][f"{filing_type}"] - html_out = jnja_template.render( - business=business, - filing=filing_data, - header=(filing.json)["filing"]["header"], - filing_date_time=leg_tmz_filing_date, - effective_date_time=leg_tmz_effective_date, - entity_dashboard_url=current_app.config.get("DASHBOARD_URL") + - (filing.json)["filing"]["business"].get("identifier", ""), - email_header=filing_name.upper(), - filing_type=filing_type - ) - - # get attachments - pdfs = _get_pdfs(status, token, business, filing, leg_tmz_filing_date, leg_tmz_effective_date) - - # get recipients - identifier = filing.filing_json["filing"]["business"]["identifier"] - recipients = [] - recipients.append(get_recipient_from_auth(identifier, token)) - - if filing.submitter_roles and UserRoles.staff in filing.submitter_roles: - # when staff file a dissolution documentOptionalEmail may contain completing party email - recipients.append(filing.filing_json["filing"]["header"].get("documentOptionalEmail")) - else: - recipients.append(get_user_email_from_auth(filing.filing_submitter.username, token)) - - if legal_type in ["SP", "GP"]: # Send email to all proprietor, partner, completing party - business_data = Business.find_by_internal_id(filing.business_id) - for party in filing.filing_json["filing"]["dissolution"]["parties"]: - if party["officer"].get("email"): - recipients.append(party["officer"]["email"]) - for party_role in business_data.party_roles.all(): - if party_role.party.email: - recipients.append(party_role.party.email) - else: - for party in filing.filing_json["filing"]["dissolution"]["parties"]: - for role in party["roles"]: - if role["roleType"] == "Custodian": - recipients.append(party["officer"]["email"]) - break - - recipients = list(set(recipients)) - recipients = ", ".join(filter(None, recipients)).strip() - - # assign subject - if status == Filing.Status.PAID.value: - if legal_type in ["SP", "GP"]: - subject = "Confirmation of Filing from the Business Registry" - else: - subject = "Voluntary dissolution" - - elif status == Filing.Status.COMPLETED.value: - if legal_type in ["SP", "GP"]: - subject = "Dissolution Documents from the Business Registry" - else: - subject = "Confirmation of Dissolution from the Business Registry" - - if not subject: # fallback case - should never happen - subject = "Notification from the BC Business Registry" - - legal_name = business.get("businessName") or business.get("legalName", None) - subject = f"{legal_name} - {subject}" if legal_name else subject - - return { - "recipients": recipients, - "requestBy": "BCRegistries@gov.bc.ca", - "content": { - "subject": subject, - "body": f"{html_out}", - "attachments": pdfs - } - } diff --git a/queue_services/business-emailer/src/business_emailer/email_processors/filing_notification.py b/queue_services/business-emailer/src/business_emailer/email_processors/filing_notification.py index 85bd6489cd..2997dc00b8 100644 --- a/queue_services/business-emailer/src/business_emailer/email_processors/filing_notification.py +++ b/queue_services/business-emailer/src/business_emailer/email_processors/filing_notification.py @@ -14,13 +14,20 @@ """Email processing rules and actions corp filing notifications.""" from __future__ import annotations +import copy +from contextlib import suppress +from datetime import datetime + +import pycountry from flask import current_app from jinja2 import Template +from business_common.utils import LegislationDatetime from business_emailer.email_processors import ( get_filing_info, get_filled_template, get_pdfs, + get_recipient_from_auth, get_recipients, get_subject, get_user_email_from_auth, @@ -42,36 +49,72 @@ def _get_additional_info(filing: Filing) -> dict: if filing.filing_type == "alteration": meta_data_alteration = filing.meta_data.get("alteration", {}) if filing.meta_data else {} additional_info["nameChange"] = "toLegalName" in meta_data_alteration + elif filing.filing_type == "specialResolution": + additional_info["nameChange"] = filing.filing_json["filing"].get("changeOfName") + additional_info["rulesChange"] = bool(filing.filing_json["filing"].get("alteration", {}).get("rulesFileKey")) + elif filing.filing_type == "correction": + additional_info["nameChange"] = "requestTypeCd" in filing.filing_json["filing"]["correction"].get("nameRequest", {}) + additional_info["rulesChange"] = bool(filing.filing_json["filing"]["correction"].get("rulesFileKey")) + additional_info["memorandumChange"] = bool(filing.filing_json["filing"]["correction"].get("memorandumFileKey")) return additional_info -def _get_additional_recipients(filing: Filing, token: str) -> str: +def _get_additional_recipients(filing: Filing, token: str) -> str | None: """Get additional recipients for a filing type.""" - submitter_recipient_filings = ["alteration", "changeOfRegistration", "dissolution", "specialResolution"] + submitter_recipient_filings = [ + "alteration", + "changeOfRegistration", + "changeOfLiquidators", + "changeOfReceivers", + "consentContinuationOut", + "continuationOut", + "dissolution", + "specialResolution" + ] if filing.filing_type in submitter_recipient_filings: - optional_email = filing.filing_json["filing"]["header"].get("documentOptionalEmail") - if filing.submitter_roles and UserRoles.staff in filing.submitter_roles and optional_email: + if filing.submitter_roles and UserRoles.staff in filing.submitter_roles: # when staff do filing documentOptionalEmail may contain completing party email - return optional_email - else: - return get_user_email_from_auth(filing.filing_submitter.username, token) + return filing.filing_json["filing"]["header"].get("documentOptionalEmail") + return get_user_email_from_auth(filing.filing_submitter.username, token) def _get_attachments_and_extra_pdf_types(status: str, filing_type: str, filing: Filing, legal_type_key: str) -> tuple[list[str], list[str]]: """Get attachments for a filing type.""" - attachments = FILING_ATTACHMENTS.get(legal_type_key, {}).get(filing_type, {}).get("attachments", []) - extra_pdf_types = FILING_ATTACHMENTS.get(legal_type_key, {}).get(filing_type, {}).get("extraPdfTypes", []) + attachments = copy.deepcopy(FILING_ATTACHMENTS.get(legal_type_key, {}).get(filing_type, {}).get("attachments", [])) + extra_pdf_types = copy.deepcopy(FILING_ATTACHMENTS.get(legal_type_key, {}).get(filing_type, {}).get("extraPdfTypes", [])) + # filing sub type overrides attachments and extraPdfTypes if present - if filing.filing_sub_type and (attachments_sub := FILING_ATTACHMENTS.get(legal_type_key, {}).get(f"{filing_type}-{filing.filing_sub_type}", {})): + if filing.filing_sub_type and (attachments_sub := copy.deepcopy(FILING_ATTACHMENTS.get(legal_type_key, {}).get(f"{filing_type}-{filing.filing_sub_type}", {}))): attachments = attachments_sub.get("attachments", []) extra_pdf_types = attachments_sub.get("extraPdfTypes", []) - # filing attachments with change of name cert can override attachments and extraPdfTypes if present - if (_get_additional_info(filing).get("nameChange", False) - and (attachments_con := FILING_ATTACHMENTS.get(legal_type_key, {}).get(f"{filing_type}-con", {})) - ): - attachments = attachments_con.get("attachments", []) - extra_pdf_types = attachments_con.get("extraPdfTypes", []) + + def _remove_from_list(list: list, value: str): + """Remove the value from the list. + Suppresses ValueError + """ + with suppress(ValueError): + list.remove(value) + + if filing_type not in Filing.TempCorpFilingType: + # adjust attachments for some filings that have dynamic attachments based on the filing data + additional_info = _get_additional_info(filing) + if not additional_info.get("nameChange"): + # remove con if in the attachments list + _remove_from_list(attachments, "Certificate of Name Change") + _remove_from_list(attachments, "Certificate of Name Change Correction") + _remove_from_list(extra_pdf_types, "certificateOfNameChange") + _remove_from_list(extra_pdf_types, "certificateOfNameCorrection") + + if not additional_info.get("rulesChange"): + # remove cr if in the attachments list + _remove_from_list(attachments, "Certified Rules") + _remove_from_list(extra_pdf_types, "certifiedRules") + + if not additional_info.get("memorandumChange"): + # remove cm if in the attachments list + _remove_from_list(attachments, "Certified Memorandum") + _remove_from_list(extra_pdf_types, "certifiedMemorandum") if status != Filing.Status.COMPLETED.value: extra_pdf_types = [] @@ -79,15 +122,100 @@ def _get_attachments_and_extra_pdf_types(status: str, filing_type: str, filing: return attachments, extra_pdf_types +def _get_business_number_display(business: dict, filing_type: str, legal_type: str): + """Return the formatted business number to display in the email.""" + business_number = None + if len(business.get("taxId", "")) > 9: # noqa: PLR2004 + # Only show if bn15 is saved, format for ux + business_number = business["taxId"].replace("BC", " BC") + + # Coops do not get a BN so it will be omitted from the email + # Dissolution emails the BN label will be omitted when there is none available (unlikely to happen in prod) + if not business_number and legal_type != Business.LegalTypes.COOP.value and filing_type != "dissolution": + business_number = NOT_AVAILABLE + + return business_number + + +def _get_datetime_str_display(date_str: str | None): + """Return the formatted date string to display in the email.""" + if date_str: + try: + if "T" in date_str: + date_time = LegislationDatetime.as_legislation_timezone(datetime.fromisoformat(date_str)) + else: + # date-only strings are already in legislation timezone + date_time = LegislationDatetime.as_legislation_timezone_from_date_str(date_str) + return date_time.strftime("%B %-d, %Y") + except Exception as err: + current_app.logger.warning(err.with_traceback(None)) + + +def _get_jurisdiction_display(country_code: str, region_code: str | None): + """Return the formatted jurisdiction to display in the email.""" + country = pycountry.countries.get(alpha_2=country_code) + jurisdiction = country.name if country else country_code + if region_code and (region := pycountry.subdivisions.get(code=f"{country_code}-{region_code}")): + jurisdiction = f"{region.name}, {jurisdiction}" + return jurisdiction + + +def _get_out_filing_details(filing: Filing) -> dict | None: + """Return details for continuation/amalgation out filings or their consent filings when applicable.""" + out_filing_info = { + "amalgamationOut": { + "action": "completed its amalgamation" + }, + "consentAmalgamationOut": { + "action": "amalgamate", + "desc": "amalgamation" + }, + "consentContinuationOut": { + "action": "continue", + "desc": "continuation" + }, + "continuationOut": { + "action": "continued" + }, + } + if filing.filing_type in out_filing_info: + meta_details = filing.meta_data.get(filing.filing_type) or {} + jurisdiction = _get_jurisdiction_display(meta_details.get("country") or "Unknown", meta_details.get("region")) + return { + "consent_expiry_date": _get_datetime_str_display(meta_details.get("expiry")), + "consent_filing_desc": out_filing_info[filing.filing_type].get("desc"), + "new_jurisdiction": jurisdiction, + "out_action": out_filing_info[filing.filing_type].get("action"), + "out_date": _get_datetime_str_display(meta_details.get("amalgamationOutDate") or meta_details.get("continuationOutDate")), + "out_name": meta_details.get("legalName") or "Unknown" + } + + +def _get_filing_display_names(filing_type: str, filing_subtype: str | None): + """Return the filing display names for the given type and subtype.""" + filing_name, filing_name_short = FILING_TITLE.get(filing_type), FILING_TITLE_SHORT.get(filing_type) + + # check if this filing has different display names based on the filing subtype + if filing_subtype: + if isinstance(filing_name, dict): + filing_name = filing_name.get(filing_subtype) + if isinstance(filing_name_short, dict): + filing_name_short = filing_name_short.get(filing_subtype) + + return filing_name, filing_name_short or filing_name + + def _skip_email_check(status: str, filing: Filing, legal_type: str, filing_name: str, business_identifier: str) -> bool: """Determine if the email should be skipped.""" invalid_status = (status not in [Filing.Status.COMPLETED.value, Filing.Status.PAID.value] or (status == Filing.Status.PAID.value and not filing.is_future_effective)) invalid_data = not legal_type or not filing_name or not business_identifier - skipped_coop_filing_types = ["changeOfDirectors", "changeOfAddress"] + skipped_coop_filing_types = ["annualReport", "changeOfDirectors", "changeOfAddress"] invalid_coop_filing = legal_type == Business.LegalTypes.COOP.value and filing.filing_type in skipped_coop_filing_types - - return invalid_status or invalid_data or invalid_coop_filing + # no email for delay, involuntary handled separately + invalid_dissolution_filing = filing.filing_type == "dissolution" and filing.filing_sub_type in ["delay", "involuntary"] + + return invalid_status or invalid_data or invalid_coop_filing or invalid_dissolution_filing def process(email_info: dict, token: str) -> dict | None: @@ -98,9 +226,8 @@ def process(email_info: dict, token: str) -> dict | None: # get template vars from filing filing, business, leg_tmz_filing_date, leg_tmz_effective_date = get_filing_info(email_info["filingId"]) - new_business_filings = ["amalgamationApplication", "continuationIn", "incorporationApplication", "registration"] filing_data = filing.json.get("filing", {}).get(filing_type, {}) - if filing_type in new_business_filings and not business: + if filing_type in Filing.TempCorpFilingType and not business: # For new business filings, the nameRequest contains relevant business details. # We overwrite the business info from the nameRequest and then set the identifier back to the temp reg id. name_request = filing_data.get("nameRequest") @@ -108,7 +235,8 @@ def process(email_info: dict, token: str) -> dict | None: business["identifier"] = filing.temp_reg legal_type = business.get("legalType") - filing_name = FILING_TITLE.get(filing_type) + filing_name, filing_name_short = _get_filing_display_names(filing_type, filing.filing_sub_type) + what_happens_next_name = filing_name_short if filing_type in ["dissolution", "amalgamationApplication"] else filing_name business_identifier = business.get("identifier") if _skip_email_check(status, filing, legal_type, filing_name, business_identifier): @@ -117,41 +245,38 @@ def process(email_info: dict, token: str) -> dict | None: dashboard_url = current_app.config.get("DASHBOARD_URL") + business_identifier is_future_effective_paid = filing.is_future_effective and status == Filing.Status.PAID.value - if filing_type in new_business_filings and is_future_effective_paid: + if filing_type in Filing.TempCorpFilingType and is_future_effective_paid: business_identifier = NOT_AVAILABLE show_effective_date = filing.is_future_effective - business_name = business.get("legalName") or NOT_AVAILABLE - filing_name_short = FILING_TITLE_SHORT.get(filing_type) + + # businessName is added for FIRMs when legalName is set to the proprietor name or list of partners + business_name = business.get("businessName") or business.get("legalName") or NOT_AVAILABLE legal_type_key = get_legal_type_key(legal_type) business_description = "Business" if corp_type := CorpType.find_by_id(legal_type): business_description: str = corp_type.full_desc.replace("BC ", "") - business_number = None - if len(business.get("taxId", "")) > 9: # noqa: PLR2004 - # Only show if bn15 is saved, format for ux - business_number = business["taxId"].replace("BC", " BC") + business_number = _get_business_number_display(business, filing_type, legal_type) + out_filing_details = _get_out_filing_details(filing) or {} # get template and fill in parts filled_template = get_filled_template(filing.filing_type, is_future_effective_paid) - if not business_number and legal_type != Business.LegalTypes.COOP.value: - business_number = NOT_AVAILABLE - # attachments and future attachments - future_attachments, extra_pdf_types = _get_attachments_and_extra_pdf_types(status, filing_type, filing, legal_type_key) - - pdfs = get_pdfs(token, business, filing, leg_tmz_filing_date, leg_tmz_effective_date, extra_pdf_types) + full_attachments_list, extra_pdf_types = _get_attachments_and_extra_pdf_types(status, filing_type, filing, legal_type_key) + filing_attachment_name = full_attachments_list[0] if full_attachments_list else None + pdfs = get_pdfs(token, business, filing, extra_pdf_types, filing_attachment_name) + attachments_list = [pdf["fileName"].replace(".pdf", "") for pdf in pdfs] # render template with vars - attachments_list = [pdf["fileName"].replace(".pdf", "") for pdf in pdfs] jnja_template: Template = Template(filled_template, autoescape=True) rendered_template = jnja_template.render( ar_date=filing_data.get("annualReportDate","")[:4], filing_date_time=leg_tmz_filing_date, effective_date_time=leg_tmz_effective_date, entity_dashboard_url=dashboard_url, + filing_sub_type=filing.filing_sub_type, filing_type=filing_type, attachments_list=attachments_list, business_description=business_description, @@ -160,19 +285,26 @@ def process(email_info: dict, token: str) -> dict | None: business_number=business_number, filing_name=filing_name, filing_name_short=filing_name_short, - future_attachments_list=future_attachments, + future_attachments_list=full_attachments_list, office_name=OFFICE_NAME.get(legal_type_key), number_description="Registration" if legal_type_key == "FIRM" else "Incorporation", show_effective_date=show_effective_date, + what_happens_next_name=what_happens_next_name, + # consent/continuation/amalgamation out values + **out_filing_details ) # get recipients recipient_filing_type = None - if filing_type in new_business_filings or filing_type == "changeOfRegistration": + if filing_type in Filing.TempCorpFilingType or filing_type in ["changeOfRegistration", "correction", "dissolution"]: recipient_filing_type = filing_type recipients = get_recipients(status, filing.filing_json, token, recipient_filing_type) + if filing_type == "dissolution" and (business_email := get_recipient_from_auth(business_identifier, token)): + # dissolution also notifies the business contact email + recipients = f"{recipients}, {business_email}" if recipients else business_email + if additional_recipients := _get_additional_recipients(filing, token): recipients = f"{recipients}, {additional_recipients}" @@ -181,8 +313,11 @@ def process(email_info: dict, token: str) -> dict | None: return # assign subject - short_filing_name = FILING_TITLE_SHORT.get(filing_type) or filing_name - subject = get_subject(is_future_effective_paid, business_name, legal_type, filing_name, short_filing_name) + subject = get_subject(is_future_effective_paid, business_name, legal_type, filing_name, filing_name_short) + if filing_type in ["consentAmalgamationOut", "consentContinuationOut"]: + subject = f"{business_name} - {filing_name_short} Granted" + elif filing_type == "changeOfReceivers": + subject = f"{business_name} - Confirmation of Receiver Change" return { "recipients": recipients, diff --git a/queue_services/business-emailer/src/business_emailer/email_processors/intent_to_liquidate_notification.py b/queue_services/business-emailer/src/business_emailer/email_processors/intent_to_liquidate_notification.py deleted file mode 100644 index 054998dc92..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_processors/intent_to_liquidate_notification.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright © 2025 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Email processing rules and actions for Intent to Liquidate notifications.""" -import base64 -import re -from http import HTTPStatus -from pathlib import Path - -import requests -from flask import current_app -from jinja2 import Template - -from business_emailer.email_processors import ( - get_filing_document, - get_filing_info, - get_recipient_from_auth, - substitute_template_parts, -) -from business_model.models import Business, Filing, UserRoles - - -def process(email_info: dict, token: str) -> dict: # pylint: disable=too-many-locals - """Build the email for Intent to Liquidate notification.""" - current_app.logger.debug("intent_to_liquidate_notification: %s", email_info) - # get template and fill in parts - filing_type = email_info["type"] - - # get template variables from filing - filing, business, leg_tmz_filing_date, leg_tmz_effective_date = get_filing_info(email_info["filingId"]) - - template = Path( - f'{current_app.config.get("TEMPLATE_PATH")}/INTENTTOLIQUIDATE-COMPLETED.html' - ).read_text() - filled_template = substitute_template_parts(template) - # render template with vars - jnja_template = Template(filled_template, autoescape=True) - filing_data = (filing.json)["filing"][f"{filing_type}"] - filing_name = filing.filing_type[0].upper() + " ".join(re.findall("[a-zA-Z][^A-Z]*", filing.filing_type[1:])) - - html_out = jnja_template.render( - business=business, - filing=filing_data, - header=(filing.json)["filing"]["header"], - filing_date_time=leg_tmz_filing_date, - effective_date_time=leg_tmz_effective_date, - entity_dashboard_url=current_app.config.get("DASHBOARD_URL") + - (filing.json)["filing"]["business"].get("identifier", ""), - email_header=filing_name.upper(), - filing_type=filing_type - ) - - # get attachments - pdfs = _get_pdfs(token, business, filing, leg_tmz_filing_date, leg_tmz_effective_date) - - # get recipients - identifier = filing.filing_json["filing"]["business"]["identifier"] - recipients = [] - recipients.append(get_recipient_from_auth(identifier, token)) - - if filing.submitter_roles and UserRoles.staff in filing.submitter_roles: - # when staff file a documentOptionalEmail may contain completing party email - recipients.append(filing.filing_json["filing"]["header"].get("documentOptionalEmail")) - - recipients = list(set(recipients)) - recipients = ", ".join(filter(None, recipients)).strip() - - # assign subject - subject = "Statement of Intent to Liquidate" - legal_name = business.get("legalName", None) - subject = f"{legal_name} - {subject}" if legal_name else subject - - return { - "recipients": recipients, - "requestBy": "BCRegistries@gov.bc.ca", - "content": { - "subject": subject, - "body": f"{html_out}", - "attachments": pdfs - } - } - - -def _get_pdfs( - token: str, - business: dict, - filing: Filing, - filing_date_time: str, - effective_date: str) -> list: - """Get the PDFs for the Statement of Intent to Liquidate output.""" - pdfs = [] - attach_order = 1 - headers = { - "Accept": "application/pdf", - "Authorization": f"Bearer {token}" - } - - # add filing PDF - filing_pdf_type = "intentToLiquidate" - filing_pdf_encoded = get_filing_document(business["identifier"], filing.id, filing_pdf_type, token) - if filing_pdf_encoded: - pdfs.append( - { - "fileName": "Statement of Intent to Liquidate.pdf", - "fileBytes": filing_pdf_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - # add receipt PDF - corp_name = business.get("legalName") - business_data = Business.find_by_internal_id(filing.business_id) - receipt = requests.post( - f'{current_app.config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - json={ - "corpName": corp_name, - "filingDateTime": filing_date_time, - "effectiveDateTime": effective_date if effective_date else "", - "filingIdentifier": str(filing.id), - "businessNumber": business_data.tax_id if business_data and business_data.tax_id else "" - }, headers=headers) - - if receipt.status_code != HTTPStatus.CREATED: - current_app.logger.error("Failed to get receipt pdf for filing: %s", filing.id) - else: - receipt_encoded = base64.b64encode(receipt.content) - pdfs.append( - { - "fileName": "Receipt.pdf", - "fileBytes": receipt_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - }) - attach_order += 1 - return pdfs diff --git a/queue_services/business-emailer/src/business_emailer/email_processors/involuntary_dissolution_stage_1_notification.py b/queue_services/business-emailer/src/business_emailer/email_processors/involuntary_dissolution_stage_1_notification.py index 851bae10d0..82803eb3d1 100644 --- a/queue_services/business-emailer/src/business_emailer/email_processors/involuntary_dissolution_stage_1_notification.py +++ b/queue_services/business-emailer/src/business_emailer/email_processors/involuntary_dissolution_stage_1_notification.py @@ -23,7 +23,7 @@ from flask import current_app from jinja2 import Template -from business_emailer.email_processors import get_jurisdictions, substitute_template_parts +from business_emailer.email_processors import substitute_template_parts from business_model.models import Business, Furnishing PROCESSABLE_FURNISHING_NAMES = [ @@ -33,8 +33,51 @@ Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_TR_XPRO.name ] +XPRO_FURNISHING_NAMES = [ + Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_AR_XPRO, + Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_TR_XPRO +] + +# NWPTA partner jurisdictions a business can be extraprovincially registered in +NWPTA_JURISDICTION_IDS = ["AB", "MB", "SK"] + +FURNISHING_CONTENT = { + Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_AR.name: { + "action": "dissolved", + "delay_type": "dissolution", + "reason_title": "overdue annual reports", + "reason_description": "you haven't filed the required annual reports", + "next_step_action": "file any outstanding annual reports", + "attachment_name": "Notice of Commencement of Dissolution" + }, + Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_TR.name: { + "action": "dissolved", + "delay_type": "dissolution", + "reason_title": "failure to file a post restoration transition application", + "reason_description": "you haven't filed the required post restoration transition application", + "next_step_action": "file your post restoration transition application", + "attachment_name": "Notice of Commencement of Dissolution" + }, + Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_AR_XPRO.name: { + "action": "cancelled", + "delay_type": "cancellation", + "reason_title": "overdue annual reports", + "reason_description": "you haven't filed the required annual reports", + "next_step_action": "file any outstanding annual reports", + "attachment_name": "Notice of Commencement of Cancellation" + }, + Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_TR_XPRO.name: { + "action": "cancelled", + "delay_type": "cancellation", + "reason_title": "failure to file a post restoration transition application", + "reason_description": "you haven't filed the required post restoration transition application", + "next_step_action": "file your post restoration transition application", + "attachment_name": "Notice of Commencement of Cancellation" + } +} + -def process(email_info: dict, token: str) -> dict: # pylint: disable=too-many-locals, , too-many-branches +def process(email_info: dict, token: str) -> dict: """Build the email for Involuntary dissolution notification.""" current_app.logger.debug("involuntary_dissolution_stage_1_notification: %s", email_info) # get business @@ -42,29 +85,47 @@ def process(email_info: dict, token: str) -> dict: # pylint: disable=too-many-l furnishing = Furnishing.find_by_id(furnishing_id) business = furnishing.business business_identifier = business.identifier - # get template - template = Path( - f'{current_app.config.get("TEMPLATE_PATH")}/INVOL-DIS-STAGE-1.html' - ).read_text(encoding="utf-8") - filled_template = substitute_template_parts(template) - # render template with vars - jnja_template = Template(filled_template, autoescape=True) + business_name = business.legal_name + content = FURNISHING_CONTENT[furnishing.furnishing_name.name] extra_provincials = [] - if furnishing.furnishing_name not in \ - [Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_AR_XPRO, - Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_TR_XPRO]: + if furnishing.furnishing_name not in XPRO_FURNISHING_NAMES: # get response from get jurisdictions jurisdictions_response = get_jurisdictions(business_identifier, token) # get extra provincials array extra_provincials = get_extra_provincials(jurisdictions_response) - html_out = jnja_template.render( - business=business.json(), - entity_dashboard_url=current_app.config.get("DASHBOARD_URL") + business_identifier, - extra_provincials=extra_provincials, - furnishing_name=furnishing.furnishing_name.name + # get template + template = Path( + f'{current_app.config.get("TEMPLATE_PATH")}/dissolution-involuntary-stage-1.md' + ).read_text(encoding="utf-8") + filled_template = substitute_template_parts(template, "md") + # render template with vars + jnja_template = Template(filled_template, autoescape=True) + entity_dashboard_url = current_app.config.get("DASHBOARD_URL") + business_identifier + number_description = "Registration" \ + if business.legal_type == Business.LegalTypes.EXTRA_PRO_A.value else "Incorporation" + + business_number = None + if len(business.tax_id or "") > 9: # noqa: PLR2004 + # Only show if bn15 is saved, format for ux + business_number = business.tax_id.replace("BC", " BC") + + body = jnja_template.render( + action=content["action"], + attachment_name=content["attachment_name"], + business_identifier=business_identifier, + business_name=business_name, + business_number=business_number, + delay_type=content["delay_type"], + entity_dashboard_url=entity_dashboard_url, + extra_provincials_display=format_extra_provincials(extra_provincials), + next_step_action=content["next_step_action"], + number_description=number_description, + reason_description=content["reason_description"], + reason_title=content["reason_title"] ) + # get recipients recipients = [] recipients.append(furnishing.email) # furnishing email @@ -75,35 +136,61 @@ def process(email_info: dict, token: str) -> dict: # pylint: disable=too-many-l # get attachments pdfs = _get_pdfs(token, business, furnishing) - legal_name = business.legal_name - subject = f"Attention {business_identifier} - {legal_name}" + subject = f"{business_name} - URGENT - Your business is in the process of being {content['action']}" return { "recipients": recipients, "requestBy": "BCRegistries@gov.bc.ca", "content": { "subject": subject, - "body": f"{html_out}", + "body": body, "attachments": pdfs } } -def get_extra_provincials(response: dict): +def get_extra_provincials(response: dict) -> list[str]: """Get extra provincials name.""" extra_provincials = [] if response: jurisdictions = response.get("jurisdictions", []) - # List of NWPTA jurisdictions - nwpta_jurisdictions = ["Alberta", "British Columbia", "Manitoba", "Saskatchewan"] for jurisdiction in jurisdictions: - name = jurisdiction.get("name") - if name and (name in nwpta_jurisdictions): + if jurisdiction.get("id") in NWPTA_JURISDICTION_IDS and (name := jurisdiction.get("name")): extra_provincials.append(name) extra_provincials.sort() return extra_provincials +def format_extra_provincials(extra_provincials: list[str]) -> str: + """Format the extra provincial names into a display string.""" + if not extra_provincials: + return "" + if len(extra_provincials) == 1: + return extra_provincials[0] + if len(extra_provincials) == 2: # noqa:PLR2004 + return f"{extra_provincials[0]} and {extra_provincials[1]}" + return f'{", ".join(extra_provincials[:-1])}, and {extra_provincials[-1]}' + + +def get_jurisdictions(identifier: str, token: str) -> dict: + """Get jurisdictions call.""" + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {token}" + } + + response = requests.get( + f'{current_app.config.get("LEGAL_API_URL")}/mras/{identifier}', headers=headers + ) + if response.status_code != HTTPStatus.OK: + return None + try: + return response.json() + except Exception: + current_app.logger.error("Failed to get MRAS response") + return None + + def post_process(email_msg: dict, status: str): """Update corresponding furnishings entry as PROCESSED or FAILED depending on notification status.""" furnishing_id = email_msg["furnishing"]["furnishingId"] @@ -140,13 +227,7 @@ def _get_pdfs( current_app.logger.error("Failed to get pdf for furnishing: %s", furnishing.id) return [] - if furnishing.furnishing_name in \ - [Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_AR_XPRO, - Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_TR_XPRO]: - filename = "Notice of Commencement of Cancellation.pdf" - else: - filename = "Notice of Commencement of Dissolution.pdf" - + filename = f'{FURNISHING_CONTENT[furnishing.furnishing_name.name]["attachment_name"]}.pdf' furnishing_pdf_encoded = base64.b64encode(furnishing_pdf.content) return [{ diff --git a/queue_services/business-emailer/src/business_emailer/email_processors/restoration_notification.py b/queue_services/business-emailer/src/business_emailer/email_processors/restoration_notification.py deleted file mode 100644 index cf844f0bb5..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_processors/restoration_notification.py +++ /dev/null @@ -1,210 +0,0 @@ -# Copyright © 2022 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Email processing rules and actions for Restoration Application notifications.""" -from __future__ import annotations - -import base64 -import re -from http import HTTPStatus - -import requests -from flask import current_app -from jinja2 import Environment, FileSystemLoader - -from business_emailer.email_processors import get_filing_document, get_filing_info, get_recipient_from_auth -from business_model.models import Business, CorpType, Filing - - -def _get_completed_pdfs( - token: str, - business: dict, - filing: Filing) -> list: - # pylint: disable=too-many-locals, too-many-branches, too-many-statements, too-many-arguments - """Get the pdfs for the restoration output.""" - pdfs = [] - attach_order = 1 - - # add notice of articles - noa_pdf_type = "noticeOfArticles" - noa_encoded = get_filing_document(business["identifier"], filing.id, noa_pdf_type, token) - if noa_encoded: - pdfs.append( - { - "fileName": "Notice of Articles.pdf", - "fileBytes": noa_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - # add certificate of restoration - certificate_pdf_type = "certificateOfRestoration" - certificate_encoded = get_filing_document(business["identifier"], filing.id, certificate_pdf_type, token) - if certificate_encoded: - pdfs.append( - { - "fileName": "Certificate of Restoration.pdf", - "fileBytes": certificate_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - return pdfs - - -def _get_paid_pdfs( - token: str, - business: dict, - filing: Filing, - filing_date_time: str, - effective_date: str) -> list: - # pylint: disable=too-many-locals, too-many-branches, too-many-statements, too-many-arguments - """Get the pdfs for the restoration output.""" - pdfs = [] - attach_order = 1 - headers = { - "Accept": "application/pdf", - "Authorization": f"Bearer {token}" - } - - filing_pdf_type = "restoration" - filing_pdf_encoded = get_filing_document(business["identifier"], filing.id, filing_pdf_type, token) - if filing_pdf_encoded: - pdfs.append( - { - "fileName": "Restoration Application.pdf", - "fileBytes": filing_pdf_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - name_request = filing.json["filing"]["restoration"]["nameRequest"] - corp_name = name_request.get("legalName") - business_data = Business.find_by_internal_id(filing.business_id) - receipt = requests.post( - f'{current_app.config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - json={ - "corpName": corp_name, - "filingDateTime": filing_date_time, - "effectiveDateTime": effective_date if effective_date != filing_date_time else "", - "filingIdentifier": str(filing.id), - "businessNumber": business_data.tax_id if business_data and business_data.tax_id else "" - }, - headers=headers - ) - if receipt.status_code != HTTPStatus.CREATED: - current_app.logger.error("Failed to get receipt pdf for filing: %s", filing.id) - else: - receipt_encoded = base64.b64encode(receipt.content) - pdfs.append( - { - "fileName": "Receipt.pdf", - "fileBytes": receipt_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - return pdfs - - -def process(email_info: dict, token: str) -> dict: # pylint: disable=too-many-locals, , too-many-branches - """Build the email for Restoration notification.""" - current_app.logger.debug("registration_notification: %s", email_info) - # get template and fill in parts - filing_type, status = email_info["type"], email_info["option"] - # get template vars from filing - filing, business, leg_tmz_filing_date, leg_tmz_effective_date = get_filing_info(email_info["filingId"]) - filing_name = filing.filing_type[0].upper() + " ".join(re.findall("[a-zA-Z][^A-Z]*", filing.filing_type[1:])) - business = business if business else filing.json["filing"]["business"] - identifier = business.get("identifier") - filing_business = filing.json["filing"]["business"] - corp_type = CorpType.find_by_id(filing_business.get("legalType")) - restoration_type = filing.json["filing"]["restoration"]["type"] - - # create the jinja environment - jinja_env = Environment( - loader=FileSystemLoader(current_app.config.get("TEMPLATE_PATH")), - autoescape=True - ) - # look for a template in this format RES-fullRestoration-PAID.html - # if the template doesn't exists use RES-PAID.html - template = jinja_env.select_template([ - f"RES-{restoration_type}-{status}.jinja2", - f"RES-{status}.jinja2" - ]) - - filing_data = filing.json["filing"][f"{filing_type}"] - html_out = template.render( - business=business, - filing=filing_data, - header=filing.json["filing"]["header"], - filing_date_time=leg_tmz_filing_date, - effective_date_time=leg_tmz_effective_date, - entity_dashboard_url=current_app.config.get("DASHBOARD_URL") + identifier, - email_header=filing_name.upper(), - filing_type=filing_type, - status=status, - entityDescription=corp_type.full_desc if corp_type else "" - ) - - # get attachments - if status == Filing.Status.PAID.value: - pdfs = _get_paid_pdfs(token, business, filing, leg_tmz_filing_date, leg_tmz_effective_date) - if status == Filing.Status.COMPLETED.value: - pdfs = _get_completed_pdfs(token, business, filing) - - # get recipients - recipients = [] - - if contact_email := filing_data.get("contactPoint", {}).get("email"): - recipients.append(contact_email) - else: - recipients.append(get_recipient_from_auth(identifier, token)) - - for party in filing_data["parties"]: - for role in party["roles"]: - if role["roleType"] == "Applicant": - recipients.append(party["officer"].get("email")) - break - - recipients = list(set(recipients)) - recipients = ", ".join(filter(None, recipients)).strip() - - # assign subject - if status == Filing.Status.PAID.value: - subject = "Confirmation of Filing from the Business Registry" - - elif status == Filing.Status.COMPLETED.value: - subject = "Restoration Documents from the Business Registry" - - if not subject: # fallback case - should never happen - subject = "Notification from the BC Business Registry" - - legal_name = business.get("legalName", None) - subject = f"{legal_name} - {subject}" if legal_name else subject - - return { - "recipients": recipients, - "requestBy": "BCRegistries@gov.bc.ca", - "content": { - "subject": subject, - "body": f"{html_out}", - "attachments": pdfs - } - } diff --git a/queue_services/business-emailer/src/business_emailer/email_processors/special_resolution_helper.py b/queue_services/business-emailer/src/business_emailer/email_processors/special_resolution_helper.py deleted file mode 100644 index 25f38e1565..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_processors/special_resolution_helper.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright © 2020 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Common functions relate to Special Resolution.""" -import base64 -from http import HTTPStatus - -import requests -from flask import current_app - -from business_emailer.email_processors import get_filing_document -from business_model.models import Business, Filing - - -def get_completed_pdfs( # noqa: PLR0913 - token: str, - business: dict, - filing: Filing, - name_changed: bool, - rules_changed=False, - memorandum_changed=False -) -> list: - # pylint: disable=too-many-locals, too-many-branches, too-many-statements, too-many-arguments - """Get the completed pdfs for the special resolution output.""" - pdfs = [] - attach_order = 1 - - # specialResolution - special_resolution_pdf_type = "specialResolution" - special_resolution_encoded = get_filing_document(business["identifier"], filing.id, - special_resolution_pdf_type, token) - if special_resolution_encoded: - pdfs.append( - { - "fileName": "Special Resolution.pdf", - "fileBytes": special_resolution_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - # Change of Name - if name_changed: - certified_name_change_pdf_type = "certificateOfNameChange" - certified_name_change_encoded = get_filing_document(business["identifier"], filing.id, - certified_name_change_pdf_type, token) - - if certified_name_change_encoded: - pdfs.append( - { - "fileName": "Certificate of Name Change.pdf", - "fileBytes": certified_name_change_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - # Certified Rules - if rules_changed: - rules_pdf_type = "certifiedRules" - certified_rules_encoded = get_filing_document(business["identifier"], filing.id, rules_pdf_type, token) - if certified_rules_encoded: - pdfs.append( - { - "fileName": "Certified Rules.pdf", - "fileBytes": certified_rules_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - # Certified Memorandum - if memorandum_changed: - certified_memorandum_pdf_type = "certifiedMemorandum" - certified_memorandum_encoded = get_filing_document(business["identifier"], filing.id, - certified_memorandum_pdf_type, token) - if certified_memorandum_encoded: - pdfs.append( - { - "fileName": "Certified Memorandum.pdf", - "fileBytes": certified_memorandum_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - return pdfs - - -def get_paid_pdfs( - token: str, - business: dict, - filing: Filing, - filing_date_time: str, - effective_date: str) -> list: - # pylint: disable=too-many-locals, too-many-branches, too-many-statements, too-many-arguments - """Get the paid pdfs for the special resolution output.""" - pdfs = [] - attach_order = 1 - headers = { - "Accept": "application/pdf", - "Authorization": f"Bearer {token}" - } - - # add filing pdf - sr_filing_pdf_type = "specialResolutionApplication" - sr_filing_pdf_encoded = get_filing_document(business["identifier"], filing.id, sr_filing_pdf_type, token) - if sr_filing_pdf_encoded: - pdfs.append( - { - "fileName": "Special Resolution Application.pdf", - "fileBytes": sr_filing_pdf_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - legal_name = business.get("legalName") - origin_business = Business.find_by_internal_id(filing.business_id) - - sr_receipt = requests.post( - f'{current_app.config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - json={ - "corpName": legal_name, - "filingDateTime": filing_date_time, - "effectiveDateTime": effective_date if effective_date != filing_date_time else "", - "filingIdentifier": str(filing.id), - "businessNumber": origin_business.tax_id if origin_business and origin_business.tax_id else "" - }, - headers=headers - ) - - if sr_receipt.status_code != HTTPStatus.CREATED: - current_app.logger.error("Failed to get receipt pdf for filing: %s", filing.id) - else: - receipt_encoded = base64.b64encode(sr_receipt.content) - pdfs.append( - { - "fileName": "Receipt.pdf", - "fileBytes": receipt_encoded.decode("utf-8"), - "fileUrl": "", - "attachOrder": str(attach_order) - } - ) - attach_order += 1 - - return pdfs diff --git a/queue_services/business-emailer/src/business_emailer/email_processors/special_resolution_notification.py b/queue_services/business-emailer/src/business_emailer/email_processors/special_resolution_notification.py deleted file mode 100644 index 467cb026a9..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_processors/special_resolution_notification.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright © 2023 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Email processing rules and actions for Special Resolution notifications.""" -from __future__ import annotations - -import re -from pathlib import Path - -from flask import current_app -from jinja2 import Template - -from business_emailer.email_processors import ( - get_filing_info, - get_recipient_from_auth, - get_user_email_from_auth, - substitute_template_parts, -) -from business_emailer.email_processors.special_resolution_helper import get_completed_pdfs, get_paid_pdfs -from business_model.models import Filing, UserRoles - - -def process(email_info: dict, token: str) -> dict: # pylint: disable=too-many-locals, too-many-branches - """Build the email for Special Resolution notification.""" - current_app.logger.debug("special_resolution_notification: %s", email_info) - # get template and fill in parts - filing_type, status = email_info["type"], email_info["option"] - # get template vars from filing - filing, business, leg_tmz_filing_date, leg_tmz_effective_date = get_filing_info(email_info["filingId"]) - filing_name = filing.filing_type[0].upper() + " ".join(re.findall("[a-zA-Z][^A-Z]*", filing.filing_type[1:])) - - template = Path( - f'{current_app.config.get("TEMPLATE_PATH")}/SR-CP-{status}.html' - ).read_text() - filled_template = substitute_template_parts(template) - # render template with vars - jnja_template = Template(filled_template, autoescape=True) - filing_data = (filing.json)["filing"][f"{filing_type}"] - name_changed = filing.filing_json["filing"].get("changeOfName") - rules_changed = bool(filing.filing_json["filing"].get("alteration", {}).get("rulesFileKey")) - html_out = jnja_template.render( - business=business, - filing=filing_data, - header=(filing.json)["filing"]["header"], - filing_date_time=leg_tmz_filing_date, - effective_date_time=leg_tmz_effective_date, - entity_dashboard_url=current_app.config.get("DASHBOARD_URL") + - (filing.json)["filing"]["business"].get("identifier", ""), - email_header=filing_name.upper(), - filing_type=filing_type, - name_changed=name_changed, - rules_updated=rules_changed - ) - - # get attachments - if status == Filing.Status.PAID.value: - pdfs = get_paid_pdfs(token, business, filing, leg_tmz_filing_date, leg_tmz_effective_date) - if status == Filing.Status.COMPLETED.value: - pdfs = get_completed_pdfs(token, business, filing, name_changed, rules_changed) - - # get recipients - identifier = filing.filing_json["filing"]["business"]["identifier"] - recipients = [] - recipients.append(get_recipient_from_auth(identifier, token)) - - if filing.submitter_roles and UserRoles.staff in filing.submitter_roles: - # when staff file a dissolution documentOptionalEmail may contain completing party email - recipients.append(filing.filing_json["filing"]["header"].get("documentOptionalEmail")) - else: - recipients.append(get_user_email_from_auth(filing.filing_submitter.username, token)) - - recipients = list(set(recipients)) - recipients = ", ".join(filter(None, recipients)).strip() - - # assign subject - subject = "" - if status == Filing.Status.PAID.value: - subject = "Confirmation of Special Resolution from the Business Registry" - elif status == Filing.Status.COMPLETED.value: - subject = "Special Resolution Documents from the Business Registry" - - legal_name = business.get("legalName", None) - subject = f"{legal_name} - {subject}" if legal_name else subject - - return { - "recipients": recipients, - "requestBy": "BCRegistries@gov.bc.ca", - "content": { - "subject": subject, - "body": f"{html_out}", - "attachments": pdfs - } - } diff --git a/queue_services/business-emailer/src/business_emailer/email_processors/util.py b/queue_services/business-emailer/src/business_emailer/email_processors/util.py index c60716da52..f9f584c964 100644 --- a/queue_services/business-emailer/src/business_emailer/email_processors/util.py +++ b/queue_services/business-emailer/src/business_emailer/email_processors/util.py @@ -34,15 +34,50 @@ "changeOfDirectors": "Director Change", "changeOfAddress": "Address Change", "changeOfRegistration": "Change of Registration", + "consentContinuationOut": "Consent to Continue Out", "continuationIn": "Continuation Application", + "continuationOut": "Continuation Out", + "correction": "Correction", + "dissolution": { + "administrative": "Dissolution Application", + "voluntary": "Voluntary Dissolution Application" + }, "incorporationApplication": "Incorporation Application", "registration": "Registration", + "specialResolution": "Special Resolution", + "restoration": "Restoration", + "changeOfLiquidators": { + "intentToLiquidate": "Statement of Intent to Liquidate", + "appointLiquidator": "Notice of Liquidator Appointment", + "ceaseLiquidator": "Notice of Liquidator Cessation", + "changeAddressLiquidator": "Liquidator (or Records) Address Change", + "liquidationReport": "Liquidation Report", + }, + "changeOfReceivers": { + "appointReceiver": "Appoint Receiver/Receiver Manager", + "ceaseReceiver": "Cease Receiver or Receiver Manager", + "changeAddressReceiver": "Change of Address of Receiver/Receiver Manager", + }, } FILING_TITLE_SHORT = { "amalgamationApplication": "Amalgamation", "continuationIn": "Continuation In", - "incorporationApplication": "Incorporation" + "changeOfLiquidators": { + "intentToLiquidate": "Intent to Liquidate", + "appointLiquidator": "Liquidator Appointment", + "ceaseLiquidator": "Liquidator Cessation", + "changeAddressLiquidator": "Liquidator (or Records) Address Change", + "liquidationReport": "Liquidation Report", + }, + "dissolution": "Dissolution", + "incorporationApplication": "Incorporation", + "restoration": { + "fullRestoration": "Restoration", + "limitedRestoration": "Restoration", + "limitedRestorationExtension": "Extension of Limited Restoration", + "limitedRestorationToFull": "Conversion to Full Restoration", + } } FILING_ATTACHMENTS = { @@ -59,17 +94,29 @@ "attachments": ["Director Change","Receipt"], "extraPdfTypes": [], }, + "correction": { + "attachments": ["Register Correction Application","Certificate of Name Correction","Certified Rules","Certified Memorandum","Receipt"], + "extraPdfTypes": ["certificateOfNameCorrection","certifiedRules","certifiedMemorandum"] + }, + "dissolution-voluntary": { + "attachments": ["Voluntary Dissolution Application","Certificate of Dissolution","Certified Affidavit","Certified Special Resolution","Receipt"], + "extraPdfTypes": ["certificateOfDissolution","affidavit","specialResolution"], + }, + "dissolution-administrative": { + "attachments": ["Dissolution Application","Certified Affidavit","Certified Special Resolution","Receipt"], + "extraPdfTypes": ["affidavit","specialResolution"], + }, "incorporationApplication": { - "attachments": ["Incorporation Application","Certificate of Incorporation","Certified Rules","Memorandum","Receipt"], + "attachments": ["Incorporation Application","Certificate of Incorporation","Certified Rules","Certified Memorandum","Receipt"], "extraPdfTypes": ["certificateOfIncorporation","certifiedRules","certifiedMemorandum"], - } + }, + "specialResolution": { + "attachments": ["Special Resolution","Special Resolution Application","Certificate of Name Change","Certified Rules","Receipt"], + "extraPdfTypes": ["specialResolutionApplication","certificateOfNameChange","certifiedRules"], + }, }, "CORP": { "alteration": { - "attachments": ["Alteration","Notice of Articles","Receipt"], - "extraPdfTypes": ["noticeOfArticles"], - }, - "alteration-con": { "attachments": ["Alteration","Notice of Articles","Certificate of Name Change","Receipt"], "extraPdfTypes": ["noticeOfArticles","certificateOfNameChange"], }, @@ -96,14 +143,82 @@ "changeOfDirectors": { "attachments": ["Director Change","Notice of Articles","Receipt"], "extraPdfTypes": ["noticeOfArticles"], + }, + "changeOfLiquidators-intentToLiquidate": { + "attachments": ["Statement of Intent to Liquidate", "Receipt"], + "extraPdfTypes": [], + }, + "changeOfLiquidators-appointLiquidator": { + "attachments": ["Notice to Appoint Liquidators", "Receipt"], + "extraPdfTypes": [], + }, + "changeOfLiquidators-ceaseLiquidator": { + "attachments": ["Notice to Cease Liquidators", "Receipt"], + "extraPdfTypes": [], + }, + "changeOfLiquidators-changeAddressLiquidator": { + "attachments": ["Liquidators (or Records) Change of Address", "Receipt"], + "extraPdfTypes": [], + }, + "changeOfLiquidators-liquidationReport": { + "attachments": ["Receipt"], + "extraPdfTypes": [], + }, + "changeOfReceivers-appointReceiver": { + "attachments": ["Receipt"], + "extraPdfTypes": [], + }, + "changeOfReceivers-ceaseReceiver": { + "attachments": ["Receipt"], + "extraPdfTypes": [], + }, + "changeOfReceivers-changeAddressReceiver": { + "attachments": ["Receipt"], + "extraPdfTypes": [], + }, + "consentContinuationOut": { + "attachments": ["Continue Out Application", "Letter of Consent", "Receipt"], + "extraPdfTypes": ["letterOfConsent"] }, "continuationIn": { "attachments": ["Continuation Application","Notice of Articles","Certificate of Continuation","Receipt"], "extraPdfTypes": ["noticeOfArticles","certificateOfContinuation"], }, + "continuationOut": { + "attachments": ["Receipt"], + "extraPdfTypes": [] + }, + "correction": { + "attachments": ["Register Correction Application","Notice of Articles","Receipt"], + "extraPdfTypes": ["noticeOfArticles"] + }, + "dissolution-voluntary": { + "attachments": ["Voluntary Dissolution Application","Certificate of Dissolution","Receipt"], + "extraPdfTypes": ["certificateOfDissolution"], + }, + "dissolution-administrative": { + "attachments": ["Dissolution Application","Receipt"], + "extraPdfTypes": [], + }, "incorporationApplication": { "attachments": ["Incorporation Application","Notice of Articles","Certificate of Incorporation","Receipt"], "extraPdfTypes": ["noticeOfArticles","certificateOfIncorporation"], + }, + "restoration-fullRestoration": { + "attachments": ["Full Restoration Application","Notice of Articles","Certificate of Restoration","Receipt"], + "extraPdfTypes": ["noticeOfArticles","certificateOfRestoration"], + }, + "restoration-limitedRestoration": { + "attachments": ["Limited Restoration Application","Notice of Articles","Certificate of Restoration","Receipt"], + "extraPdfTypes": ["noticeOfArticles","certificateOfRestoration"], + }, + "restoration-limitedRestorationExtension": { + "attachments": ["Limited Restoration Extension Application","Notice of Articles","Certificate of Restoration","Receipt"], + "extraPdfTypes": ["noticeOfArticles","certificateOfRestoration"], + }, + "restoration-limitedRestorationToFull": { + "attachments": ["Conversion to Full Restoration Application","Notice of Articles","Certificate of Restoration","Receipt"], + "extraPdfTypes": ["noticeOfArticles","certificateOfRestoration"], } }, "FIRM": { @@ -111,6 +226,14 @@ "attachments": ["Change of Registration", "Amended Registration Statement", "Receipt"], "extraPdfTypes": ["amendedRegistrationStatement"], }, + "correction": { + "attachments": ["Register Correction Application","Corrected Registration Statement","Receipt"], + "extraPdfTypes": ["correctedRegistrationStatement"] + }, + "dissolution-voluntary": { + "attachments": ["Statement of Dissolution","Receipt"], + "extraPdfTypes": [], + }, "registration": { "attachments": ["Statement of Registration","Receipt"], "extraPdfTypes": [], @@ -133,4 +256,4 @@ def get_legal_type_key(legal_type: str) -> str: return "CP" elif legal_type in [Business.LegalTypes.SOLE_PROP.value, Business.LegalTypes.PARTNERSHIP.value]: return "FIRM" - return "CORP" \ No newline at end of file + return "CORP" diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/APPOINT_RECVR.html b/queue_services/business-emailer/src/business_emailer/email_templates/APPOINT_RECVR.html deleted file mode 100644 index c18edba5d2..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/APPOINT_RECVR.html +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - Confirmation of Appoint Receiver - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/BC-CRCTN-COMPLETED.html b/queue_services/business-emailer/src/business_emailer/email_templates/BC-CRCTN-COMPLETED.html deleted file mode 100644 index ab2a10bcd0..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/BC-CRCTN-COMPLETED.html +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - Correction Documents from the Business Registry - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/BC-CRCTN-PAID.html b/queue_services/business-emailer/src/business_emailer/email_templates/BC-CRCTN-PAID.html deleted file mode 100644 index dca93c8bd9..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/BC-CRCTN-PAID.html +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - - - Confirmation of Filing from the Business Registry - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/CCO-COMPLETED.html b/queue_services/business-emailer/src/business_emailer/email_templates/CCO-COMPLETED.html deleted file mode 100644 index f2d57e35a0..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/CCO-COMPLETED.html +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - We have received your Consent for Continuation Out. - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/CEASE_RECVR.html b/queue_services/business-emailer/src/business_emailer/email_templates/CEASE_RECVR.html deleted file mode 100644 index 702ec8ecb1..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/CEASE_RECVR.html +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - Confirmation of Cease Receiver - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/CO-COMPLETED.html b/queue_services/business-emailer/src/business_emailer/email_templates/CO-COMPLETED.html deleted file mode 100644 index 1500f913a9..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/CO-COMPLETED.html +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - Confirmation of continuation out - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/CONT-IN-APPROVED.html b/queue_services/business-emailer/src/business_emailer/email_templates/CONT-IN-APPROVED.html deleted file mode 100644 index a185765ded..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/CONT-IN-APPROVED.html +++ /dev/null @@ -1,55 +0,0 @@ - - - - - - - - - Results of your Continuation Authorization - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/CONT-IN-AWAITING_REVIEW.html b/queue_services/business-emailer/src/business_emailer/email_templates/CONT-IN-AWAITING_REVIEW.html deleted file mode 100644 index 1639bf5226..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/CONT-IN-AWAITING_REVIEW.html +++ /dev/null @@ -1,50 +0,0 @@ - - - - - - - - - Confirmation of Submission from the Business Registry - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/CONT-IN-CHANGE_REQUESTED.html b/queue_services/business-emailer/src/business_emailer/email_templates/CONT-IN-CHANGE_REQUESTED.html deleted file mode 100644 index 319551491b..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/CONT-IN-CHANGE_REQUESTED.html +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - - - - Continuation Authorization Results from the Business Registry - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/CONT-IN-REJECTED.html b/queue_services/business-emailer/src/business_emailer/email_templates/CONT-IN-REJECTED.html deleted file mode 100644 index 29daf600f0..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/CONT-IN-REJECTED.html +++ /dev/null @@ -1,57 +0,0 @@ - - - - - - - - - Continuation Authorization Results from the Business Registry - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/CONT-IN-RESUBMITTED.html b/queue_services/business-emailer/src/business_emailer/email_templates/CONT-IN-RESUBMITTED.html deleted file mode 100644 index 93e1d3ae2a..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/CONT-IN-RESUBMITTED.html +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - Confirmation of Submission from the Business Registry - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/CP-SR-CRCTN-COMPLETED.html b/queue_services/business-emailer/src/business_emailer/email_templates/CP-SR-CRCTN-COMPLETED.html deleted file mode 100644 index 992778e09d..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/CP-SR-CRCTN-COMPLETED.html +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - Corrected documents from the BC Business Registry - - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/CP-SR-CRCTN-PAID.html b/queue_services/business-emailer/src/business_emailer/email_templates/CP-SR-CRCTN-PAID.html deleted file mode 100644 index fe29e7bf5a..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/CP-SR-CRCTN-PAID.html +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - - Corrected documents from the BC Business Registry - - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/DIS-COMPLETED.html b/queue_services/business-emailer/src/business_emailer/email_templates/DIS-COMPLETED.html deleted file mode 100644 index 003f0c416f..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/DIS-COMPLETED.html +++ /dev/null @@ -1,65 +0,0 @@ - - - - - - - - - {% if business.legalType in ['GP', 'SP'] %} - Confirmation of dissolution - {% else %} - Confirmation of voluntary dissolution - {% endif %} - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/DIS-PAID.html b/queue_services/business-emailer/src/business_emailer/email_templates/DIS-PAID.html deleted file mode 100644 index 99800f2ef9..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/DIS-PAID.html +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - - - - - {% if business.legalType in ['GP', 'SP'] %} - You have successfully submitted your dissolution with the BC Business Registry. - {% else %} - You have successfully submitted your voluntary dissolution with the BC Business Registry. - {% endif %} - - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/FIRM-CRCTN-COMPLETED.html b/queue_services/business-emailer/src/business_emailer/email_templates/FIRM-CRCTN-COMPLETED.html deleted file mode 100644 index 4f63336f8a..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/FIRM-CRCTN-COMPLETED.html +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - Confirmation of correction of registration - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/FIRM-CRCTN-PAID.html b/queue_services/business-emailer/src/business_emailer/email_templates/FIRM-CRCTN-PAID.html deleted file mode 100644 index 3283fce63d..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/FIRM-CRCTN-PAID.html +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - You have successfully submitted your correction of registration with the BC Business Registry. - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/INTENTTOLIQUIDATE-COMPLETED.html b/queue_services/business-emailer/src/business_emailer/email_templates/INTENTTOLIQUIDATE-COMPLETED.html deleted file mode 100644 index e1283b3f8b..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/INTENTTOLIQUIDATE-COMPLETED.html +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - Statement of Intent to Liquidate - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/INVOL-DIS-STAGE-1.html b/queue_services/business-emailer/src/business_emailer/email_templates/INVOL-DIS-STAGE-1.html deleted file mode 100644 index b360ea554b..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/INVOL-DIS-STAGE-1.html +++ /dev/null @@ -1,123 +0,0 @@ - - - - - - - - - Attention {{ business.identifier }} {{ business.legalName }} - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/RES-COMPLETED.jinja2 b/queue_services/business-emailer/src/business_emailer/email_templates/RES-COMPLETED.jinja2 deleted file mode 100644 index f8c2e71cd1..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/RES-COMPLETED.jinja2 +++ /dev/null @@ -1,15 +0,0 @@ -{% extends 'RES-common.jinja2' %} - -{% block page_title %}You have successfully restored your business with the BC Business Registry.{% endblock %} - -{% block title_message %}You have successfully restored your business with the BC Business Registry.{% endblock %} - -{% block attachments %} -
    -
  • Notice of Articles
    • -
  • Certificate of Restoration
    • -
-{% endblock %} - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/RES-PAID.jinja2 b/queue_services/business-emailer/src/business_emailer/email_templates/RES-PAID.jinja2 deleted file mode 100644 index 10c2555f0b..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/RES-PAID.jinja2 +++ /dev/null @@ -1,17 +0,0 @@ -{% extends 'RES-common.jinja2' %} - -{% block page_title %}You have successfully filed your restoration with the BC Business Registry.{% endblock %} -{% block title_message %}You have successfully filed your restoration with the BC Business Registry.{% endblock %} - -{% block attachments %}{% endblock %} - -{% block after_processing %} -

- Once your filing has been processed, the business will be - registered, and the company will receive the below outputs: -

-
    -
  • Notice of Articles
    • -
  • Certificate of Restoration
    • -
-{% endblock %} diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/RES-common.jinja2 b/queue_services/business-emailer/src/business_emailer/email_templates/RES-common.jinja2 deleted file mode 100644 index 01dd3116a6..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/RES-common.jinja2 +++ /dev/null @@ -1,15 +0,0 @@ -{% extends './common/base_template.jinja2' %} - -{% block title_message %}You have successfully filed your restoration with the BC Business Registry.{% endblock %} - -{% block content %} - {% include './common/business-information-block.jinja2' %} -

- The following documents are attached to this email: -

- {% block attachments %}{% endblock %} -

- These documents are also available through your BC Business Registry account. -

- {% block after_processing %}{% endblock %} -{% endblock %} diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/RES-fullRestoration-PAID.jinja2 b/queue_services/business-emailer/src/business_emailer/email_templates/RES-fullRestoration-PAID.jinja2 deleted file mode 100644 index bb4fcb508b..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/RES-fullRestoration-PAID.jinja2 +++ /dev/null @@ -1,8 +0,0 @@ -{% extends 'RES-PAID.jinja2' %} - -{% block attachments %} -
    -
  • Full Restoration Application
    • -
  • Receipt
    • -
-{% endblock %} diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/RES-limitedRestoration-PAID.jinja2 b/queue_services/business-emailer/src/business_emailer/email_templates/RES-limitedRestoration-PAID.jinja2 deleted file mode 100644 index a237ff5f4f..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/RES-limitedRestoration-PAID.jinja2 +++ /dev/null @@ -1,8 +0,0 @@ -{% extends 'RES-PAID.jinja2' %} - -{% block attachments %} -
    -
  • Limited Restoration Application
    • -
  • Receipt
    • -
-{% endblock %} diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/RES-limitedRestorationExtension-COMPLETED.jinja2 b/queue_services/business-emailer/src/business_emailer/email_templates/RES-limitedRestorationExtension-COMPLETED.jinja2 deleted file mode 100644 index 833fca9c6c..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/RES-limitedRestorationExtension-COMPLETED.jinja2 +++ /dev/null @@ -1,5 +0,0 @@ -{% extends 'RES-COMPLETED.jinja2' %} - -{% block page_title %}You have successfully extended the period of restoration with the BC Business Registry.{% endblock %} - -{% block title_message %}You have successfully extended the period of restoration with the BC Business Registry.{% endblock %} diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/RES-limitedRestorationExtension-PAID.jinja2 b/queue_services/business-emailer/src/business_emailer/email_templates/RES-limitedRestorationExtension-PAID.jinja2 deleted file mode 100644 index aadf3351f7..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/RES-limitedRestorationExtension-PAID.jinja2 +++ /dev/null @@ -1,8 +0,0 @@ -{% extends 'RES-PAID.jinja2' %} - -{% block attachments %} -
    -
  • Limited Restoration Extension Application
    • -
  • Receipt
    • -
-{% endblock %} diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/RES-limitedRestorationToFull-PAID.jinja2 b/queue_services/business-emailer/src/business_emailer/email_templates/RES-limitedRestorationToFull-PAID.jinja2 deleted file mode 100644 index 8fb5175efb..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/RES-limitedRestorationToFull-PAID.jinja2 +++ /dev/null @@ -1,8 +0,0 @@ -{% extends 'RES-PAID.jinja2' %} - -{% block attachments %} -
    -
  • Conversion to Full Restoration Application
    • -
  • Receipt
    • -
-{% endblock %} diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/SR-CP-COMPLETED.html b/queue_services/business-emailer/src/business_emailer/email_templates/SR-CP-COMPLETED.html deleted file mode 100644 index eb41dfd1f6..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/SR-CP-COMPLETED.html +++ /dev/null @@ -1,50 +0,0 @@ - - - - - - - - - Confirmation of Special Resolution - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/SR-CP-PAID.html b/queue_services/business-emailer/src/business_emailer/email_templates/SR-CP-PAID.html deleted file mode 100644 index 44c13385d3..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/SR-CP-PAID.html +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - - - - You have successfully filed your special resolution with the BC Business Registry. - - [[style.html]] - - - - - - - - - - diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/changeOfLiquidators.md b/queue_services/business-emailer/src/business_emailer/email_templates/changeOfLiquidators.md new file mode 100644 index 0000000000..498a878a46 --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/changeOfLiquidators.md @@ -0,0 +1,13 @@ +# You have successfully filed your {{ filing_name | lower }} with the BC Business Registry + +--- + +[[business-tombstone.md]] + +--- + +[[attachments.md]] + +--- + +[[business-registry-footer.md]] diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/changeOfReceivers.md b/queue_services/business-emailer/src/business_emailer/email_templates/changeOfReceivers.md new file mode 100644 index 0000000000..51ee013af8 --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/changeOfReceivers.md @@ -0,0 +1,13 @@ +# Your receiver/receiver manager information has been successfully updated + +--- + +[[business-tombstone.md]] + +--- + +[[attachments.md]] + +--- + +[[business-registry-footer.md]] diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/common/business-tombstone-basic.md b/queue_services/business-emailer/src/business_emailer/email_templates/common/business-tombstone-basic.md new file mode 100644 index 0000000000..b858c27d86 --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/common/business-tombstone-basic.md @@ -0,0 +1,5 @@ +**Business Name:** {{ business_name }} +**{{ number_description }} Number:** {{ business_identifier }} +{% if business_number -%} +**Business Number:** {{ business_number }} +{% endif %} diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/common/business-tombstone-out-filing.md b/queue_services/business-emailer/src/business_emailer/email_templates/common/business-tombstone-out-filing.md new file mode 100644 index 0000000000..0c39c34d25 --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/common/business-tombstone-out-filing.md @@ -0,0 +1,12 @@ +**Business Name:** {{ business_name }} +**{{number_description}} Number:** {{ business_identifier }} +**Business Number:** {{ business_number }} +{% if consent_expiry_date -%} +**Effective Until:** {{ consent_expiry_date }} +{% endif -%} +**New Jurisdiction:** {{ new_jurisdiction }} +{% if out_date and filing_type == 'amalgamationOut' -%} +**Amalgamate Out Effective Date:** {{ out_date }} +{% elif out_date and filing_type == 'continuationOut' -%} +**Continue Out Effective Date:** {{ out_date }} +{% endif %} diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/common/consent-next-steps.md b/queue_services/business-emailer/src/business_emailer/email_templates/common/consent-next-steps.md new file mode 100644 index 0000000000..c4459f6171 --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/common/consent-next-steps.md @@ -0,0 +1,2 @@ +## Next Steps +Once you have completed your {{ consent_filing_desc }} into the jurisdiction of {{ new_jurisdiction }}, you must forward any documents showing your status in your new jurisdiction so that we may complete your {{ consent_filing_desc }} out and make {{ business_name }} - {{ business_identifier }} historical in British Columbia. \ No newline at end of file diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/common/consent.md b/queue_services/business-emailer/src/business_emailer/email_templates/common/consent.md new file mode 100644 index 0000000000..51a61e2dc9 --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/common/consent.md @@ -0,0 +1,2 @@ +## Consent +You have been granted a 6 month consent to {{ out_action }} your business out of British Columbia, to the jurisdiction of {{ new_jurisdiction }}. {% if consent_expiry_date -%}This consent expires on {{ consent_expiry_date }}.{% endif %} \ No newline at end of file diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/common/continuation-application-details.html b/queue_services/business-emailer/src/business_emailer/email_templates/common/continuation-application-details.html deleted file mode 100644 index f63ce03b85..0000000000 --- a/queue_services/business-emailer/src/business_emailer/email_templates/common/continuation-application-details.html +++ /dev/null @@ -1,25 +0,0 @@ -
-

Your Continuation Application Details

-
-
Previous Jurisdiction:
-
{{ foreign_jurisdiction }}
-
- [[16px.html]] -
-
Identifying Number in Previous Jurisdiction:
-
{{ filing.foreignJurisdiction.identifier }}
-
- [[16px.html]] -
-
Name in Previous Jurisdiction:
-
{{ filing.foreignJurisdiction.legalName }}
-
- - {% if filing.business %} - [[16px.html]] -
-
Extraprovincial Registration Number in B.C.:
-
{{ filing.business.identifier }}
-
- {% endif %} -
diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/common/continuation-application-details.md b/queue_services/business-emailer/src/business_emailer/email_templates/common/continuation-application-details.md new file mode 100644 index 0000000000..6774bf8144 --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/common/continuation-application-details.md @@ -0,0 +1,7 @@ +## Your Continuation Application Details +**Previous Jurisdiction:** {{ foreign_jurisdiction }} +**Name in Previous Jurisdiction:** {{ filing.foreignJurisdiction.legalName }} +**Number in Previous Jurisdiction:** {{ filing.foreignJurisdiction.identifier }} +{% if filing.business %} +**Extraprovincial Registration Number in BC:** {{ filing.business.identifier }} +{% endif %} diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/common/out-details.md b/queue_services/business-emailer/src/business_emailer/email_templates/common/out-details.md new file mode 100644 index 0000000000..f3af042d09 --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/common/out-details.md @@ -0,0 +1 @@ +Your business {{ business_name }} - {{ business_identifier }} has been made historical in British Columbia as of {{ out_date }} after having successfully {{ out_action }} into {{ new_jurisdiction }} under the name {{ out_name | upper }}. \ No newline at end of file diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/common/what-happens-next.md b/queue_services/business-emailer/src/business_emailer/email_templates/common/what-happens-next.md index b7fa7141f1..2e0f869639 100644 --- a/queue_services/business-emailer/src/business_emailer/email_templates/common/what-happens-next.md +++ b/queue_services/business-emailer/src/business_emailer/email_templates/common/what-happens-next.md @@ -1,5 +1,5 @@ ## What happens next -Once the {{ filing_name | lower }} is effective on {{ effective_date_time }}, you will receive the following documents by email: +Once the {{ what_happens_next_name | lower }} is effective on {{ effective_date_time }}, you will receive the following documents by email: {% for attachment in future_attachments_list -%} - {{ attachment }} {% endfor %} diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/consentContinuationOut.md b/queue_services/business-emailer/src/business_emailer/email_templates/consentContinuationOut.md new file mode 100644 index 0000000000..873c57d587 --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/consentContinuationOut.md @@ -0,0 +1,21 @@ +# Your request for consent to continue out of B.C. has been granted for 6 months + +--- + +[[business-tombstone-out-filing.md]] + +--- + +[[consent.md]] + +--- + +[[attachments.md]] + +--- + +[[consent-next-steps.md]] + +--- + +[[business-registry-footer.md]] \ No newline at end of file diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/continuationIn-approved.md b/queue_services/business-emailer/src/business_emailer/email_templates/continuationIn-approved.md new file mode 100644 index 0000000000..97f88e44fa --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/continuationIn-approved.md @@ -0,0 +1,19 @@ +# Results of your Continuation Authorization + +--- + +Your Continuation Authorization was **approved**. Follow the steps below to complete your continuation application. + +--- + +## Next Steps + +1. Return to [My Business Registry]({{ entity_dashboard_url }}) to resume your Continuation Application + +--- + +[[continuation-application-details.md]] + +--- + +[[business-registry-footer.md]] diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/continuationIn-awaitingReview.md b/queue_services/business-emailer/src/business_emailer/email_templates/continuationIn-awaitingReview.md new file mode 100644 index 0000000000..ba1a03caee --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/continuationIn-awaitingReview.md @@ -0,0 +1,19 @@ +# We have received your Continuation Authorization documents + +--- + +BC Registries will review your Continuation Authorization documents and contact you with the results within 5 business days. + +--- + +## Next Steps + +Once your authorization has been approved, you will be notified and can then return to the [BC Business Registry]({{ entity_dashboard_url }}) to complete your continuation application. + +--- + +[[continuation-application-details.md]] + +--- + +[[business-registry-footer.md]] diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/continuationIn-changeRequested.md b/queue_services/business-emailer/src/business_emailer/email_templates/continuationIn-changeRequested.md new file mode 100644 index 0000000000..b67ac9359b --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/continuationIn-changeRequested.md @@ -0,0 +1,24 @@ +# Change Requested to your Continuation Authorization + +--- + +BC Registries would like you to make changes to your Continuation Authorization. Follow the steps below to make changes and resubmit your application. + +--- + +## Next Steps + +Follow these steps to complete your application: + +1. Go to [My Business Registry]({{ entity_dashboard_url }}). +2. Select "Make Changes" to satisfy the request from BC Registries as described below: + +***{{ latest_review_comment }}*** + +--- + +[[continuation-application-details.md]] + +--- + +[[business-registry-footer.md]] diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/continuationIn-rejected.md b/queue_services/business-emailer/src/business_emailer/email_templates/continuationIn-rejected.md new file mode 100644 index 0000000000..f5eb444ad5 --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/continuationIn-rejected.md @@ -0,0 +1,21 @@ +# Results of your Continuation Authorization + +--- + +Your Continuation Authorization was **rejected**. + +--- + +## Next Steps + +1. Review the reasons your authorization was rejected below: + ***{{ latest_review_comment }}*** +2. Visit [My Business Registry]({{ entity_dashboard_url }}) to submit a new Continuation Application. + +--- + +[[continuation-application-details.md]] + +--- + +[[business-registry-footer.md]] diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/continuationIn-resubmitted.md b/queue_services/business-emailer/src/business_emailer/email_templates/continuationIn-resubmitted.md new file mode 100644 index 0000000000..392301891a --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/continuationIn-resubmitted.md @@ -0,0 +1,19 @@ +# We have received your updated Continuation Authorization documents + +--- + +BC Registries will review your updated Continuation Authorization documents and contact you with the results within 5 business days. + +--- + +## Next Steps + +Once your authorization has been approved, you will be notified and can then return to the [BC Business Registry]({{ entity_dashboard_url }}) to complete your continuation application. + +--- + +[[continuation-application-details.md]] + +--- + +[[business-registry-footer.md]] diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/continuationOut.md b/queue_services/business-emailer/src/business_emailer/email_templates/continuationOut.md new file mode 100644 index 0000000000..621f69c991 --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/continuationOut.md @@ -0,0 +1,17 @@ +# You have successfully continued out of B.C. + +--- + +[[business-tombstone-out-filing.md]] + +--- + +[[out-details.md]] + +--- + +[[attachments.md]] + +--- + +[[business-registry-footer.md]] \ No newline at end of file diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/correction.md b/queue_services/business-emailer/src/business_emailer/email_templates/correction.md new file mode 100644 index 0000000000..05c04325c3 --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/correction.md @@ -0,0 +1,13 @@ +# You have successfully completed your correction with the BC Business Registry + +--- + +[[business-tombstone.md]] + +--- + +[[attachments.md]] + +--- + +[[business-registry-footer.md]] \ No newline at end of file diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/dissolution-future.md b/queue_services/business-emailer/src/business_emailer/email_templates/dissolution-future.md new file mode 100644 index 0000000000..3b4129fda2 --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/dissolution-future.md @@ -0,0 +1,17 @@ +# Your {{ filing_name | lower }} has been filed + +--- + +[[business-tombstone.md]] + +--- + +[[attachments.md]] + +--- + +[[what-happens-next.md]] + +--- + +[[business-registry-footer.md]] \ No newline at end of file diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/dissolution-involuntary-stage-1.md b/queue_services/business-emailer/src/business_emailer/email_templates/dissolution-involuntary-stage-1.md new file mode 100644 index 0000000000..b7ba1985ba --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/dissolution-involuntary-stage-1.md @@ -0,0 +1,37 @@ +# Your business is in the process of being {{ action }} for {{ reason_title }} + +--- + +[[business-tombstone-basic.md]] + +--- + +## Attention + +Your business is in the process of being {{ action }} because {{ reason_description }}. + +{% if extra_provincials_display -%} +Our records indicate your business is registered in {{ extra_provincials_display }} as an extraprovincial company. If your business is {{ action }}, its registration as an extraprovincial company in {{ extra_provincials_display }} will automatically be cancelled as well. + +{% endif -%} +See the attached document for more information. + +--- + +## Next Steps + +Log into your [BC Business Registry account]({{ entity_dashboard_url }}) and {{ next_step_action }}. + +If you need more time, you can request a delay of {{ delay_type }} from your account. + +--- + +## Attachments + +The following document is attached to this email: + +- {{ attachment_name }} + +--- + +[[business-registry-footer.md]] diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/dissolution.md b/queue_services/business-emailer/src/business_emailer/email_templates/dissolution.md new file mode 100644 index 0000000000..fe7c8fd147 --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/dissolution.md @@ -0,0 +1,13 @@ +# You have successfully completed your dissolution with the BC Business Registry + +--- + +[[business-tombstone.md]] + +--- + +[[attachments.md]] + +--- + +[[business-registry-footer.md]] \ No newline at end of file diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/restoration.md b/queue_services/business-emailer/src/business_emailer/email_templates/restoration.md new file mode 100644 index 0000000000..21b822fbac --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/restoration.md @@ -0,0 +1,17 @@ +{% if filing_sub_type == 'limitedRestorationExtension' -%} +# You have successfully extended your period of restoration with the BC Business Registry +{% else -%} +# You have successfully restored your business with the BC Business Registry +{% endif -%} + +--- + +[[business-tombstone.md]] + +--- + +[[attachments.md]] + +--- + +[[business-registry-footer.md]] \ No newline at end of file diff --git a/queue_services/business-emailer/src/business_emailer/email_templates/specialResolution.md b/queue_services/business-emailer/src/business_emailer/email_templates/specialResolution.md new file mode 100644 index 0000000000..fd13969376 --- /dev/null +++ b/queue_services/business-emailer/src/business_emailer/email_templates/specialResolution.md @@ -0,0 +1,13 @@ +# Your have successfully filed your special resolution with the BC Business Registry + +--- + +[[business-tombstone.md]] + +--- + +[[attachments.md]] + +--- + +[[business-registry-footer.md]] \ No newline at end of file diff --git a/queue_services/business-emailer/src/business_emailer/meta/filing.py b/queue_services/business-emailer/src/business_emailer/meta/filing.py index 22982e5de7..d4c4e81944 100644 --- a/queue_services/business-emailer/src/business_emailer/meta/filing.py +++ b/queue_services/business-emailer/src/business_emailer/meta/filing.py @@ -877,22 +877,22 @@ def alter_outputs_dissolution(filing, outputs): if filing.filing_type == "dissolution": # Suppress Certificate of Dissolution for Admin Dissolution if filing.filing_sub_type == "administrative": - outputs.remove("certificateOfDissolution") + outputs.discard("certificateOfDissolution") # Suppress Certified Memorandum and Certified Rules for Coop Voluntary Dissolution if filing.filing_sub_type == "voluntary" and filing.json_legal_type == Business.LegalTypes.COOP: - outputs.remove("certifiedRules") - outputs.remove("certifiedMemorandum") + outputs.discard("certifiedRules") + outputs.discard("certifiedMemorandum") return outputs @staticmethod def alter_outputs_special_resolution(filing, outputs): """Handle output file list modification for special resolution.""" if filing.filing_type == "specialResolution": - outputs.remove("certifiedMemorandum") + outputs.discard("certifiedMemorandum") if "changeOfName" in filing.meta_data.get("legalFilings", []): outputs.add("certificateOfNameChange") if not filing.meta_data.get("alteration", {}).get("uploadNewRules"): - outputs.remove("certifiedRules") + outputs.discard("certifiedRules") return outputs @staticmethod diff --git a/queue_services/business-emailer/src/business_emailer/resources/business_emailer.py b/queue_services/business-emailer/src/business_emailer/resources/business_emailer.py index 9be2a3e0ef..e15256c444 100644 --- a/queue_services/business-emailer/src/business_emailer/resources/business_emailer.py +++ b/queue_services/business-emailer/src/business_emailer/resources/business_emailer.py @@ -46,25 +46,16 @@ agm_extension_notification, agm_location_change_notification, amalgamation_out_notification, - appoint_receiver_notification, ar_reminder_notification, bn_notification, - cease_receiver_notification, consent_amalgamation_out_notification, - consent_continuation_out_notification, - continuation_in_notification, - continuation_out_notification, - correction_notification, - dissolution_notification, + continuation_authorization_notification, filing_notification, - intent_to_liquidate_notification, involuntary_dissolution_stage_1_notification, mras_notification, name_request, notice_of_withdrawal_notification, nr_notification, - restoration_notification, - special_resolution_notification, ) from business_emailer.email_processors.util import FILING_TITLE from business_emailer.exceptions import EmailException, QueueException @@ -166,6 +157,7 @@ def send_email(email: dict, token: str): ) current_app.logger.debug("NOTIFY API response: %s", resp.status_code) if resp.status_code != HTTPStatus.OK: + current_app.logger.debug("NOTIFY API error: %s", resp.json()) raise EmailException except Exception: # this should log the error and put the email msg back on the queue @@ -235,51 +227,24 @@ def process_email(ce: SimpleCloudEvent): # pylint: disable=too-many-branches, t elif etype == "agmExtension" and option == Filing.Status.COMPLETED.value: email = agm_extension_notification.process(email_msg["email"], token) send_email(email, token) - elif etype == "dissolution": - email = dissolution_notification.process(email_msg["email"], token) - send_email(email, token) - elif etype == "restoration": - email_object = restoration_notification.process(email_msg["email"], token) - send_email(email_object, token) - elif etype == "correction": - email = correction_notification.process(email_msg["email"], token) - send_email(email, token) elif etype == "consentAmalgamationOut": email = consent_amalgamation_out_notification.process(email_msg["email"], token) send_email(email, token) elif etype == "amalgamationOut": email = amalgamation_out_notification.process(email_msg["email"], token) send_email(email, token) - elif etype == "consentContinuationOut" and option == Filing.Status.COMPLETED.value: - email = consent_continuation_out_notification.process(email_msg["email"], token) - send_email(email, token) - elif etype == "continuationOut" and option == Filing.Status.COMPLETED.value: - email = continuation_out_notification.process(email_msg["email"], token) - send_email(email, token) - elif etype == "specialResolution": - email = special_resolution_notification.process(email_msg["email"], token) - send_email(email, token) elif etype == "continuationIn" and option in ReviewStatus._member_names_: # Special case for review step of continuation in filing. Regular filing notifications are handled by the filing_notification processor. - email = continuation_in_notification.process(email_msg["email"], token) - send_email(email, token) - elif etype == "intentToLiquidate": - email = intent_to_liquidate_notification.process(email_msg["email"], token) + email = continuation_authorization_notification.process(email_msg["email"], token) send_email(email, token) elif etype == "noticeOfWithdrawal" and option == Filing.Status.COMPLETED.value: email = notice_of_withdrawal_notification.process(email_msg["email"], token) send_email(email, token) - elif etype == "appointReceiver" and option == Filing.Status.COMPLETED.value: - email = appoint_receiver_notification.process(email_msg["email"], token) - send_email(email, token) - elif etype == "ceaseReceiver" and option == Filing.Status.COMPLETED.value: - email = cease_receiver_notification.process(email_msg["email"], token) - send_email(email, token) elif etype in FILING_TITLE: if email := filing_notification.process(email_msg["email"], token): send_email(email, token) else: - # should only be if this was for a coops filing + # should only be if this was for a coops filing or an invalid publish option current_app.logger.debug("No email to send for: %s", email_msg) else: current_app.logger.debug("No email to send for: %s", email_msg) diff --git a/queue_services/business-emailer/tests/unit/__init__.py b/queue_services/business-emailer/tests/unit/__init__.py index 482bd192c6..b81c63f735 100644 --- a/queue_services/business-emailer/tests/unit/__init__.py +++ b/queue_services/business-emailer/tests/unit/__init__.py @@ -15,7 +15,7 @@ import copy import json from datetime import datetime, timedelta -from random import randrange, choices +from random import randrange from unittest.mock import Mock @@ -41,7 +41,6 @@ CORRECTION_CP_SPECIAL_RESOLUTION, CORRECTION_INCORPORATION, CORRECTION_REGISTRATION, - CP_SPECIAL_RESOLUTION_TEMPLATE, DISSOLUTION, FILING_HEADER, FILING_TEMPLATE, @@ -49,6 +48,7 @@ NOTICE_OF_WITHDRAWAL, REGISTRATION, RESTORATION, + SPECIAL_RESOLUTION, ) from tests.unit.helpers import generate_temp_filing @@ -58,19 +58,37 @@ AMALGAMATION_APPLICATION_TEMPLATE = copy.deepcopy(FILING_TEMPLATE) AMALGAMATION_APPLICATION_TEMPLATE['filing']['header']['name'] = 'amalgamationApplication' AMALGAMATION_APPLICATION_TEMPLATE['filing']['amalgamationApplication'] = copy.deepcopy(AMALGAMATION_APPLICATION) +REGISTRATION_TEMPLATE = copy.deepcopy(FILING_TEMPLATE) +REGISTRATION_TEMPLATE['filing']['header']['name'] = 'registration' +REGISTRATION_TEMPLATE['filing']['registration'] = copy.deepcopy(REGISTRATION) BOOTSTRAP_TYPE_MAPPER = { 'amalgamationApplication': AMALGAMATION_APPLICATION_TEMPLATE, 'continuationIn': CONTINUATION_IN_FILING_TEMPLATE, 'incorporationApplication': INCORPORATION_FILING_TEMPLATE, + 'registration': REGISTRATION_TEMPLATE, } # NB: These do not include the header or business in their templates FILING_TYPE_MAPPER = { + 'alteration': ALTERATION, 'annualReport': ANNUAL_REPORT['filing']['annualReport'], 'changeOfAddress': CORP_CHANGE_OF_ADDRESS, 'changeOfDirectors': CHANGE_OF_DIRECTORS, - 'alteration': ALTERATION + 'changeOfLiquidators': CHANGE_OF_LIQUIDATORS, + 'changeOfReceivers': CHANGE_OF_RECEIVERS, + 'changeOfRegistration': CHANGE_OF_REGISTRATION, + 'consentContinuationOut': CONSENT_CONTINUATION_OUT, + 'continuationOut': CONTINUATION_OUT, + 'dissolution': DISSOLUTION, + 'restoration': RESTORATION, + 'specialResolution': SPECIAL_RESOLUTION +} + +CORRECTION_TYPE_MAPPER = { + 'incorporationApplication': CORRECTION_INCORPORATION['filing']['correction'], + 'registration': CORRECTION_REGISTRATION['filing']['correction'], + 'specialResolution': CORRECTION_CP_SPECIAL_RESOLUTION } COMP_PARTY_EMAIL = 'comp_party@email.com' @@ -80,6 +98,34 @@ PARTY_EMAIL_2 = 'party2@email.com' +def _prep_parties_for_filing(filing_template: dict, filing_type: str, legal_type: str): + """Update the parties in the filing template with expected data.""" + if filing_template['filing'][filing_type].get('parties'): + filing_template['filing'][filing_type]['parties'] = filing_template['filing'][filing_type]['parties'][:2] + for index, party in enumerate(filing_template['filing'][filing_type]['parties']): + for role in party['roles']: + if legal_type not in ['SP', 'GP'] and role['roleType'] == 'Completing Party': + party['officer']['email'] = COMP_PARTY_EMAIL + break + elif index == 0: + party['officer']['email'] = PARTY_EMAIL_1 + else: + party['officer']['email'] = PARTY_EMAIL_2 + if legal_type == 'SP': + filing_template['filing'][filing_type]['parties'] = filing_template['filing'][filing_type]['parties'][:1] + filing_template['filing'][filing_type]['parties'][0]['roles'] = [ + { + 'roleType': 'Completing Party', + 'appointmentDate': '2022-01-01' + + }, + { + 'roleType': 'Proprietor', + 'appointmentDate': '2022-01-01' + + } + ] + def create_user(user_name: str): """Return a new user model.""" user = User() @@ -130,7 +176,7 @@ def create_filing(token=None, filing_json=None, business_id=None, return filing -def prep_bootstrap_filing(session, filing_type, identifier, legal_type, option, legal_name=None, filing_sub_type=None): +def prep_bootstrap_filing(session, filing_type, identifier, legal_type, option, legal_name=None, filing_sub_type=None, parties=None): """Return a new bootstrap filing prepped for email notification.""" if not filing_sub_type and filing_type == 'amalgamationApplication': filing_sub_type = 'regular' @@ -140,7 +186,7 @@ def prep_bootstrap_filing(session, filing_type, identifier, legal_type, option, 'legalType': legal_type } filing_template['filing'][filing_type]['nameRequest'] = {'legalType': legal_type} - filing_template['filing'][filing_type]['contactPoint']['email'] = CONTACT_POINT + filing_template['filing'][filing_type]['contactPoint'] = {'email': CONTACT_POINT} if legal_name: filing_template['filing'][filing_type]['nameRequest']['legalName'] = legal_name @@ -148,12 +194,24 @@ def prep_bootstrap_filing(session, filing_type, identifier, legal_type, option, if filing_sub_type: sub_type_key = Filing.FILING_SUB_TYPE_KEYS.get(filing_type, 'type') filing_template['filing'][filing_type][sub_type_key] = filing_sub_type - - for party in filing_template['filing'][filing_type]['parties']: - for role in party['roles']: - if role['roleType'] == 'Completing Party': - party['officer']['email'] = COMP_PARTY_EMAIL - + + _prep_parties_for_filing(filing_template, filing_type, legal_type) + + if filing_type == 'registration': + filing_template['filing']['business']['naics'] = { + 'naicsCode': '112320', + 'naicsDescription': 'Broiler and other meat-type chicken production' + } + filing_template['filing']['registration']['startDate'] = datetime.now().strftime('%Y-%m-%d') + if legal_type != 'GP': + filing_template['filing']['registration']['businessType'] = legal_type + + if legal_type == 'CP': + filing_template['filing'][filing_type]['cooperative'] = { + 'cooperativeAssociationType': 'CP', + 'rulesFileKey': 'rulekey1234', + 'memorandumFileKey': 'memkey1234' + } temp_identifier = generate_temp_filing() filing_template['filing']['business']['identifier'] = temp_identifier filing = create_filing(token='1', filing_json=filing_template, bootstrap_id=temp_identifier) @@ -162,7 +220,7 @@ def prep_bootstrap_filing(session, filing_type, identifier, legal_type, option, if option in ['COMPLETED', 'bn', 'mras']: legal_name = legal_name or f'{identifier[2:]} B.C. Ltd.' - business = create_business(identifier, legal_type=legal_type, legal_name=legal_name) + business = create_business(identifier, legal_type=legal_type, legal_name=legal_name, parties=parties) filing.business_id = business.id transaction_id = VersioningProxy.get_transaction_id(session()) filing.transaction_id = transaction_id @@ -176,105 +234,9 @@ def prep_incorp_filing(session, identifier, option, legal_type='BC', legal_name= return prep_bootstrap_filing(session, 'incorporationApplication', identifier, legal_type, option, legal_name=legal_name) -def prep_registration_filing(session, identifier, payment_id, option, legal_type, legal_name, parties=None): +def prep_registration_filing(session, identifier, option, legal_type, legal_name, parties=None): """Return a new registration filing prepped for email notification.""" - now = datetime.now().strftime('%Y-%m-%d') - REGISTRATION['business']['naics'] = { - 'naicsCode': '112320', - 'naicsDescription': 'Broiler and other meat-type chicken production' - } - - gp_registration = copy.deepcopy(FILING_HEADER) - gp_registration['filing']['header']['name'] = 'registration' - gp_registration['filing']['registration'] = copy.deepcopy(REGISTRATION) - gp_registration['filing']['registration']['startDate'] = now - gp_registration['filing']['registration']['nameRequest']['legalName'] = legal_name - gp_registration['filing']['registration']['parties'][0]['officer']['email'] = PARTY_EMAIL_1 - gp_registration['filing']['registration']['parties'][1]['officer']['email'] = PARTY_EMAIL_2 - gp_registration['filing']['registration']['contactPoint']['email'] = CONTACT_POINT - - sp_registration = copy.deepcopy(FILING_HEADER) - sp_registration['filing']['header']['name'] = 'registration' - sp_registration['filing']['registration'] = copy.deepcopy(REGISTRATION) - sp_registration['filing']['registration']['startDate'] = now - sp_registration['filing']['registration']['parties'][0]['officer']['email'] = PARTY_EMAIL_1 - sp_registration['filing']['registration']['businessType'] = 'SP' - sp_registration['filing']['registration']['contactPoint']['email'] = CONTACT_POINT - sp_registration['filing']['registration']['parties'][0]['roles'] = [ - { - 'roleType': 'Completing Party', - 'appointmentDate': '2022-01-01' - - }, - { - 'roleType': 'Proprietor', - 'appointmentDate': '2022-01-01' - - } - ] - del sp_registration['filing']['registration']['parties'][1] - - if legal_type == Business.LegalTypes.SOLE_PROP.value: - filing_template = sp_registration - elif legal_type == Business.LegalTypes.PARTNERSHIP.value: - filing_template = gp_registration - - temp_identifier = generate_temp_filing() - filing_template['filing']['business'] = { - 'identifier': temp_identifier, - 'legalType': legal_type - } - filing = create_filing(token=payment_id, filing_json=filing_template, - business_id=None, bootstrap_id=temp_identifier) - filing.payment_completion_date = filing.filing_date - - if option in ['COMPLETED']: - business = create_business(identifier, legal_type, parties=parties) - business.founding_date = datetime.fromisoformat(now) - filing.business_id = business.id - transaction_id = VersioningProxy.get_transaction_id(session()) - filing.transaction_id = transaction_id - - filing.save() - return filing - - -def prep_dissolution_filing(session, identifier, payment_id, option, legal_type, - legal_name, submitter_role, parties=None): - """Return a new dissolution filing prepped for email notification.""" - business = create_business(identifier, legal_type, legal_name, parties) - filing_template = copy.deepcopy(FILING_HEADER) - filing_template['filing']['header']['name'] = 'dissolution' - if submitter_role: - filing_template['filing']['header']['documentOptionalEmail'] = f'{submitter_role}@email.com' - - filing_template['filing']['dissolution'] = copy.deepcopy(DISSOLUTION) - filing_template['filing']['business'] = { - 'identifier': business.identifier, - 'legalType': legal_type, - 'legalName': legal_name - } - - for party in filing_template['filing']['dissolution']['parties']: - for role in party['roles']: - if role['roleType'] == 'Custodian': - party['officer']['email'] = 'custodian@email.com' - elif role['roleType'] == 'Completing Party': - party['officer']['email'] = 'cp@email.com' - - filing = create_filing( - token=payment_id, - filing_json=filing_template, - business_id=business.id) - filing.payment_completion_date = filing.filing_date - - user = create_user('test_user') - filing.submitter_id = user.id - if submitter_role: - filing.submitter_roles = submitter_role - - filing.save() - return filing + return prep_bootstrap_filing(session, 'registration', identifier, legal_type, option, legal_name=legal_name, parties=parties) def prep_consent_amalgamation_out_filing(session, identifier, payment_id, legal_type, legal_name, submitter_role): @@ -354,155 +316,10 @@ def prep_amalgamation_out_filing(session, identifier, payment_id, legal_type, le return filing -def prep_consent_continuation_out_filing(session, identifier, payment_id, legal_type, legal_name, submitter_role): - """Return a new consent continuation out filing prepped for email notification.""" - business = create_business(identifier, legal_type, legal_name) - filing_template = copy.deepcopy(FILING_HEADER) - filing_template['filing']['header']['name'] = 'consentContinuationOut' - if submitter_role: - filing_template['filing']['header']['documentOptionalEmail'] = f'{submitter_role}@email.com' - - filing_template['filing']['consentContinuationOut'] = copy.deepcopy(CONSENT_CONTINUATION_OUT) - filing_template['filing']['business'] = { - 'identifier': business.identifier, - 'legalType': legal_type, - 'legalName': legal_name - } - - filing = create_filing( - token=payment_id, - filing_json=filing_template, - business_id=business.id) - filing.payment_completion_date = filing.filing_date - - user = create_user('test_user') - filing.submitter_id = user.id - if submitter_role: - filing.submitter_roles = submitter_role - - filing.save() - return filing - - -def prep_continuation_out_filing(session, identifier, payment_id, legal_type, legal_name, submitter_role): - """Return a new continuation out filing prepped for email notification.""" - business = create_business(identifier, legal_type, legal_name) - filing_template = copy.deepcopy(FILING_HEADER) - filing_template['filing']['header']['name'] = 'continuationOut' - if submitter_role: - filing_template['filing']['header']['documentOptionalEmail'] = f'{submitter_role}@email.com' - - filing_template['filing']['continuationOut'] = copy.deepcopy(CONTINUATION_OUT) - filing_template['filing']['business'] = { - 'identifier': business.identifier, - 'legalType': legal_type, - 'legalName': legal_name - } - - filing = create_filing( - token=payment_id, - filing_json=filing_template, - business_id=business.id) - filing.payment_completion_date = filing.filing_date - - user = create_user('test_user') - filing.submitter_id = user.id - if submitter_role: - filing.submitter_roles = submitter_role - - filing.save() - return filing - - -def prep_restoration_filing(identifier, payment_id, legal_type, legal_name, r_type='fullRestoration'): - """Return a new restoration filing prepped for email notification. - - @param r_type: - @param identifier: - @param payment_id: - @param legal_type: - @param legal_name: - @return: - """ - business = create_business(identifier, legal_type, legal_name) - filing_template = copy.deepcopy(FILING_HEADER) - filing_template['filing']['header']['name'] = 'restoration' - filing_template['filing']['restoration'] = copy.deepcopy(RESTORATION) - filing_template['filing']['restoration']['type'] = r_type - filing_template['filing']['business'] = { - 'identifier': business.identifier, - 'legalType': legal_type, - 'legalName': legal_name - } - - filing = create_filing( - token=payment_id, - filing_json=filing_template, - business_id=business.id) - filing.payment_completion_date = filing.filing_date - - user = create_user('test_user') - filing.submitter_id = user.id - - filing.save() - return filing - - def prep_change_of_registration_filing(session, identifier, payment_id, legal_type, legal_name, submitter_role, parties=None): """Return a new change of registration filing prepped for email notification.""" - business = create_business(identifier, legal_type, legal_name, parties) - - gp_change_of_registration = copy.deepcopy(FILING_HEADER) - gp_change_of_registration['filing']['header']['name'] = 'changeOfRegistration' - gp_change_of_registration['filing']['changeOfRegistration'] = copy.deepcopy(CHANGE_OF_REGISTRATION) - gp_change_of_registration['filing']['changeOfRegistration']['parties'][0]['officer']['email'] = PARTY_EMAIL_1 - gp_change_of_registration['filing']['changeOfRegistration']['contactPoint']['email'] = CONTACT_POINT - - sp_change_of_registration = copy.deepcopy(FILING_HEADER) - sp_change_of_registration['filing']['header']['name'] = 'changeOfRegistration' - sp_change_of_registration['filing']['changeOfRegistration'] = copy.deepcopy(CHANGE_OF_REGISTRATION) - sp_change_of_registration['filing']['changeOfRegistration']['parties'][0]['roles'] = [ - { - 'roleType': 'Completing Party', - 'appointmentDate': '2022-01-01' - - }, - { - 'roleType': 'Proprietor', - 'appointmentDate': '2022-01-01' - - } - ] - sp_change_of_registration['filing']['changeOfRegistration']['parties'][0]['officer']['email'] = PARTY_EMAIL_1 - sp_change_of_registration['filing']['changeOfRegistration']['contactPoint']['email'] = CONTACT_POINT - - if legal_type == Business.LegalTypes.SOLE_PROP.value: - filing_template = sp_change_of_registration - elif legal_type == Business.LegalTypes.PARTNERSHIP.value: - filing_template = gp_change_of_registration - - filing_template['filing']['business'] = { - 'identifier': business.identifier, - 'legalType': legal_type, - 'legalName': legal_name - } - if submitter_role: - filing_template['filing']['header']['documentOptionalEmail'] = f'{submitter_role}@email.com' - - filing = create_filing( - token=payment_id, - filing_json=filing_template, - business_id=business.id) - filing.payment_completion_date = filing.filing_date - - user = create_user('test_user') - filing.submitter_id = user.id - if submitter_role: - filing.submitter_roles = submitter_role - - filing.save() - return filing + return prep_maintenance_filing(session, identifier, payment_id, 'COMPLETED', 'changeOfRegistration', None, submitter_role, legal_type, parties) def prep_alteration_filing(session, identifier, option, company_name): @@ -569,19 +386,46 @@ def prep_agm_extension_filing(identifier, payment_id, legal_type, legal_name): return filing -def prep_maintenance_filing(session, identifier, payment_id, status, filing_type, submitter_role=None): +def prep_maintenance_filing(session, identifier, payment_id, status, filing_type, + filing_sub_type=None, submitter_role=None, legal_type=None, parties=None, template_overrides={}): """Return a new maintenance filing prepped for email notification.""" - legal_type = Business.LegalTypes.COOP.value if identifier.startswith('CP') else Business.LegalTypes.BCOMP.value - business = create_business(identifier, legal_type, LEGAL_NAME) + if not legal_type: + legal_type = Business.LegalTypes.COOP.value if identifier.startswith('CP') else Business.LegalTypes.BCOMP.value + business = create_business(identifier, legal_type, LEGAL_NAME, parties) filing_template = copy.deepcopy(FILING_TEMPLATE) filing_template['filing']['header']['name'] = filing_type - filing_template['filing']['business'] = \ - {'identifier': f'{identifier}', 'legalType': legal_type, 'legalName': LEGAL_NAME} + filing_template['filing']['business'] = { + 'identifier': identifier, + 'legalType': legal_type, + 'legalName': business.legal_name + } filing_template['filing'][filing_type] = copy.deepcopy(FILING_TYPE_MAPPER[filing_type]) + filing_template['filing'][filing_type]['contactPoint'] = {'email': CONTACT_POINT} + _prep_parties_for_filing(filing_template, filing_type, legal_type) + + if filing_sub_type: + sub_type_key = Filing.FILING_SUB_TYPE_KEYS.get(filing_type, 'type') + filing_template['filing'][filing_type][sub_type_key] = filing_sub_type if submitter_role: filing_template['filing']['header']['documentOptionalEmail'] = f'{submitter_role}@email.com' - filing = create_filing(token=payment_id, filing_json=filing_template, business_id=business.id) + + filing_template['filing'] = { + **filing_template['filing'], + # add any extra data or overwrite as required + **template_overrides + } + + meta_data = None + if filing_type == 'consentContinuationOut': + meta_data = {'consentContinuationOut': { + 'expiry': '2025-10-31T06:59:00+00:00', 'region': 'AB', 'country': 'CA'}} + elif filing_type == 'continuationOut': + meta_data = {'continuationOut': { + 'continuationOutDate': '2025-04-29', 'legalName': 'new test business', + 'region': 'AB', 'country': 'CA'}} + + filing = create_filing(token=payment_id, filing_json=filing_template, business_id=business.id, meta_data=meta_data) user = create_user('test_user') filing.submitter_id = user.id @@ -597,156 +441,96 @@ def prep_maintenance_filing(session, identifier, payment_id, status, filing_type return filing -def prep_incorporation_correction_filing(session, business, original_filing_id, payment_id, option): - """Return a new incorporation correction filing prepped for email notification.""" - filing_template = copy.deepcopy(CORRECTION_INCORPORATION) +def prep_special_resolution_filing(session, identifier='CP1234567', submitter_role=None, has_name_change=False, has_rule_change=False): + """Return a new special resolution out filing prepped for email notification.""" + filing_template_overrides = {} + if has_name_change: + filing_template_overrides['changeOfName'] = { + 'nameRequest': { + 'nrNumber': 'NR 8798956', + 'legalName': 'HAULER MEDIA INC.', + 'legalType': 'BC' + } + } + if has_rule_change: + filing_template_overrides['alteration'] = { + 'rulesInResolution': True, + 'rulesFileKey': 'cooperative/a8abe1a6-4f45-4105-8a05-822baee3b743.pdf' + } + return prep_maintenance_filing(session, identifier, '1', 'COMPLETED', 'specialResolution', None, submitter_role, template_overrides=filing_template_overrides) + + +def prep_correction_filing(session, + business: Business, + original_filing_id: int, + original_filing_type: str, + status: str, + has_name_change=False, + has_rule_change=False, + has_memorandum_change=False): + """Return a correction filing prepped for email notification.""" + filing_template = copy.deepcopy(FILING_HEADER) + filing_template['filing']['header']['name'] = 'correction' + filing_template['filing']['correction'] = copy.deepcopy(CORRECTION_TYPE_MAPPER[original_filing_type]) filing_template['filing']['business'] = {'identifier': business.identifier} - for party in filing_template['filing']['correction']['parties']: - for role in party['roles']: - if role['roleType'] == 'Completing Party': - party['officer']['email'] = 'comp_party@email.com' - filing_template['filing']['correction']['contactPoint']['email'] = 'test@test.com' + filing_template['filing']['correction']['contactPoint']['email'] = CONTACT_POINT filing_template['filing']['correction']['correctedFilingId'] = original_filing_id - filing = create_filing(token=payment_id, filing_json=filing_template, business_id=business.id) - filing.payment_completion_date = filing.filing_date - filing.save() - if option in ['COMPLETED']: - transaction_id = VersioningProxy.get_transaction_id(session()) - filing.transaction_id = transaction_id - filing.save() - return filing - - -def prep_firm_correction_filing(session, identifier, payment_id, legal_type, legal_name, submitter_role, parties=None): - """Return a firm correction filing prepped for email notification.""" - business = create_business(identifier, legal_type, legal_name, parties) - - gp_correction = copy.deepcopy(CORRECTION_REGISTRATION) - gp_correction['filing']['correction']['parties'][0]['officer']['email'] = 'party@email.com' - - sp_correction = copy.deepcopy(CORRECTION_REGISTRATION) - sp_correction['filing']['correction']['parties'][0]['officer']['email'] = 'party@email.com' - sp_correction['filing']['correction']['parties'][0]['roles'] = [ - { - 'roleType': 'Completing Party', - 'appointmentDate': '2022-01-01' - - }, - { - 'roleType': 'Proprietor', - 'appointmentDate': '2022-01-01' - - } + filing_template['filing']['correction']['correctedFilingType'] = original_filing_type + # Update the parties (schema examples do not reflect a real payload for the correction case) + filing_template['filing']['correction']['parties'] = filing_template['filing']['correction']['parties'][:1] + # Remove completing party role (examples contain it as an extra role, but the real payload does not) + filing_template['filing']['correction']['parties'][0]['roles'] = [ + role for role in filing_template['filing']['correction']['parties'][0]['roles'] + if role['roleType'] != 'Completing Party' ] - sp_correction['filing']['correction']['parties'][0]['officer']['email'] = 'party@email.com' - - if legal_type == Business.LegalTypes.SOLE_PROP.value: - filing_template = sp_correction - elif legal_type == Business.LegalTypes.PARTNERSHIP.value: - filing_template = gp_correction - - filing_template['filing']['business'] = { - 'identifier': business.identifier, - 'legalType': legal_type, - 'legalName': legal_name - } - - filing = create_filing( - token=payment_id, - filing_json=filing_template, - business_id=business.id) - filing.payment_completion_date = filing.filing_date - - user = create_user('test_user') - filing.submitter_id = user.id - if submitter_role: - filing.submitter_roles = submitter_role - - filing.save() - return filing - - -def prep_cp_special_resolution_filing(identifier, payment_id, legal_type, legal_name, submitter_role=None): - """Return a new cp special resolution out filing prepped for email notification.""" - business = create_business(identifier, legal_type=legal_type, legal_name=legal_name) - filing_template = copy.deepcopy(CP_SPECIAL_RESOLUTION_TEMPLATE) - filing_template['filing']['business'] = \ - {'identifier': f'{identifier}', 'legalype': legal_type, 'legalName': legal_name} - filing_template['filing']['alteration'] = { - 'business': { - 'identifier': 'BC1234567', - 'legalType': 'BEN' + # Add completing party as a separate party with no email + filing_template['filing']['correction']['parties'].append({ + 'officer': { + 'firstName': 'Completing', + 'lastName': 'Party', + 'partyType': 'person' }, - 'contactPoint': { - 'email': 'joe@email.com' + 'mailingAddress': { + 'streetAddress': 'mailing_address - address line one', + 'addressCity': 'mailing_address city', + 'addressCountry': 'CA', + 'postalCode': 'H0H0H0', + 'addressRegion': 'BC' }, - 'rulesInResolution': True, - 'rulesFileKey': 'cooperative/a8abe1a6-4f45-4105-8a05-822baee3b743.pdf' - } - if submitter_role: - filing_template['filing']['header']['documentOptionalEmail'] = f'{submitter_role}@email.com' - filing = create_filing(token=payment_id, filing_json=filing_template, business_id=business.id) - - user = create_user('cp_test_user') - filing.submitter_id = user.id - if submitter_role: - filing.submitter_roles = submitter_role - filing.save() - return filing + 'roles': [ + { + 'roleType': 'Completing Party', + 'appointmentDate': datetime.now().strftime('%Y-%m-%d') + } + ] + }) + _prep_parties_for_filing(filing_template, 'correction', business.legal_type) -def prep_cp_special_resolution_correction_filing(session, business, original_filing_id, - payment_id, option, corrected_filing_type): - """Return a cp special resolution correction filing prepped for email notification.""" - filing_template = copy.deepcopy(FILING_HEADER) - filing_template['filing']['header']['name'] = 'correction' - filing_template['filing']['correction'] = copy.deepcopy(CORRECTION_CP_SPECIAL_RESOLUTION) - filing_template['filing']['business'] = {'identifier': business.identifier} - filing_template['filing']['correction']['contactPoint']['email'] = 'cp_sr@test.com' - filing_template['filing']['correction']['correctedFilingId'] = original_filing_id - filing_template['filing']['correction']['correctedFilingType'] = corrected_filing_type filing_template['filing']['correction']['nameRequest'] = { 'nrNumber': 'NR 8798956', 'legalName': 'HAULER MEDIA INC.', - 'legalType': 'BC', - 'requestType': 'CHG' + 'legalType': business.legal_type, + 'requestTypeCd': 'CHG' } - filing = create_filing(token=payment_id, filing_json=filing_template, business_id=business.id) - filing.payment_completion_date = filing.filing_date - # Triggered from the filer. - filing._meta_data = {'correction': {'uploadNewRules': True, 'toLegalName': True}} - filing.save() - if option in ['COMPLETED']: - transaction_id = VersioningProxy.get_transaction_id(session()) - filing.transaction_id = transaction_id - filing.save() - return filing + filing_template['filing']['correction']['rulesFileKey'] = 'ruleskey1234' + filing_template['filing']['correction']['memorandumFileKey'] = 'memkey1234' + if not has_name_change: + del filing_template['filing']['correction']['nameRequest'] + if not has_rule_change: + del filing_template['filing']['correction']['rulesFileKey'] + if not has_memorandum_change: + del filing_template['filing']['correction']['memorandumFileKey'] + + filing = create_filing(token='1', filing_json=filing_template, business_id=business.id) -def prep_cp_special_resolution_correction_upload_memorandum_filing(session, business, - original_filing_id, - payment_id, option, - corrected_filing_type): - """Return a cp special resolution correction filing prepped for email notification.""" - filing_template = copy.deepcopy(FILING_HEADER) - filing_template['filing']['header']['name'] = 'correction' - filing_template['filing']['correction'] = copy.deepcopy(CORRECTION_CP_SPECIAL_RESOLUTION) - filing_template['filing']['business'] = {'identifier': business.identifier} - filing_template['filing']['correction']['contactPoint']['email'] = 'cp_sr@test.com' - filing_template['filing']['correction']['correctedFilingId'] = original_filing_id - filing_template['filing']['correction']['correctedFilingType'] = corrected_filing_type - del filing_template['filing']['correction']['resolution'] - filing_template['filing']['correction']['memorandumFileKey'] = '28f73dc4-8e7c-4c89-bef6-a81dff909ca6.pdf' - filing_template['filing']['correction']['memorandumFileName'] = 'test.pdf' - filing = create_filing(token=payment_id, filing_json=filing_template, business_id=business.id) - filing.payment_completion_date = filing.filing_date - # Triggered from the filer. - filing._meta_data = {'correction': {'uploadNewMemorandum': True}} filing.save() - if option in ['COMPLETED']: + if status == 'COMPLETED': transaction_id = VersioningProxy.get_transaction_id(session()) filing.transaction_id = transaction_id filing.save() + return filing @@ -789,58 +573,6 @@ def prep_intent_to_liquidate_filing(session, identifier, payment_id, legal_type, return filing -def prep_cease_receiver_filing(identifier, payment_id, legal_type, legal_name): - """Return a new Cease Receiver filing prepped for email notification.""" - business = create_business(identifier, legal_type, legal_name) - filing_template = copy.deepcopy(FILING_HEADER) - filing_template['filing']['header']['name'] = 'changeOfReceivers' - filing_template['filing']['changeOfReceivers'] = copy.deepcopy(CHANGE_OF_RECEIVERS) - filing_template['filing']['changeOfReceivers']['type'] = 'ceaseReceiver' - filing_template['filing']['business'] = { - 'identifier': business.identifier, - 'legalType': legal_type, - 'legalName': legal_name - } - - filing = create_filing( - token=payment_id, - filing_json=filing_template, - business_id=business.id) - filing.payment_completion_date = filing.filing_date - - user = create_user('test_user') - filing.submitter_id = user.id - - filing.save() - return filing - - -def prep_appoint_receiver_filing(identifier, payment_id, legal_type, legal_name): - """Return a new Appoint Receiver filing prepped for email notification.""" - business = create_business(identifier, legal_type, legal_name) - filing_template = copy.deepcopy(FILING_HEADER) - filing_template['filing']['header']['name'] = 'changeOfReceivers' - - filing_template['filing']['appointReceiver'] = copy.deepcopy(CHANGE_OF_RECEIVERS) - filing_template['filing']['business'] = { - 'identifier': business.identifier, - 'legalType': legal_type, - 'legalName': legal_name - } - - filing = create_filing( - token=payment_id, - filing_json=filing_template, - business_id=business.id) - filing.payment_completion_date = filing.filing_date - - user = create_user('test_user') - filing.submitter_id = user.id - - filing.save() - return filing - - def prep_notice_of_withdraw_filing( identifier, payment_id, diff --git a/queue_services/business-emailer/tests/unit/email_processors/test_appoint_receiver_notification.py b/queue_services/business-emailer/tests/unit/email_processors/test_appoint_receiver_notification.py deleted file mode 100644 index 279d2f2032..0000000000 --- a/queue_services/business-emailer/tests/unit/email_processors/test_appoint_receiver_notification.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright © 2026 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -"""The Unit Tests for the Appoint Receiver email processor.""" -import base64 -from unittest.mock import patch - -import pytest -import requests_mock -from business_model.models import Business - -from business_emailer.email_processors import appoint_receiver_notification -from tests.unit import prep_appoint_receiver_filing - - -@pytest.mark.skip('This email processor is invalid and needs to be updated') -@pytest.mark.parametrize('status,legal_name,is_numbered', [ - ('COMPLETED', 'test business', False), - ('COMPLETED', 'BC1234567', True), -]) -def test_appoint_receiver_notification(app, session, status, legal_name, is_numbered): - """Assert that the appoint_receiver email processor builds the expected email.""" - # setup filing + business for email - filing = prep_appoint_receiver_filing( - 'BC1234567', '1', Business.LegalTypes.COMP.value, legal_name) - token = 'token' - # test processor - with patch.object(appoint_receiver_notification, '_get_pdfs', return_value=[]) as mock_get_pdfs: - with patch.object(appoint_receiver_notification, 'get_recipient_from_auth', - return_value='recipient@email.com'): - email = appoint_receiver_notification.process( - {'filingId': filing.id, 'type': 'appointReceiver', 'option': status}, token) - - if is_numbered: - assert email['content']['subject'] == 'Numbered Company - Receiver appointed' - else: - assert email['content']['subject'] == f'{legal_name} - Receiver appointed' - - assert email['recipients'] == 'recipient@email.com' - assert email['requestBy'] == 'BCRegistries@gov.bc.ca' - assert email['content']['body'] - assert email['content']['attachments'] == [] - - assert mock_get_pdfs.call_args[0][0] == token - assert mock_get_pdfs.call_args[0][1]['identifier'] == 'BC1234567' - assert mock_get_pdfs.call_args[0][1]['legalName'] == legal_name - assert mock_get_pdfs.call_args[0][1]['legalType'] == Business.LegalTypes.COMP.value - assert mock_get_pdfs.call_args[0][2] == filing - - -@pytest.mark.skip('This email processor is invalid and needs to be updated') -def test_appoint_receiver_attachments(session, config): - """Assert _get_pdfs assembles the Appoint Receiver filing PDF and receipt.""" - identifier = 'BC1234567' - filing = prep_appoint_receiver_filing( - identifier, '1', Business.LegalTypes.COMP.value, 'test business') - token = 'token' - with patch.object(appoint_receiver_notification, 'get_recipient_from_auth', - return_value='recipient@email.com'): - with requests_mock.Mocker() as m: - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{identifier}' - f'/filings/{filing.id}/documents/appointReceiver', - content=b'pdf_content_1', - status_code=200, - ) - m.post( - f'{config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - content=b'pdf_content_2', - status_code=201, - ) - output = appoint_receiver_notification.process( - {'filingId': filing.id, 'type': 'appointReceiver', 'option': 'COMPLETED'}, token) - - attachments = output['content']['attachments'] - assert len(attachments) == 2 - assert attachments[0]['fileName'] == 'Appoint Receiver.pdf' - assert base64.b64decode(attachments[0]['fileBytes']).decode('utf-8') == 'pdf_content_1' - assert attachments[1]['fileName'] == 'Receipt.pdf' - assert base64.b64decode(attachments[1]['fileBytes']).decode('utf-8') == 'pdf_content_2' diff --git a/queue_services/business-emailer/tests/unit/email_processors/test_bn_notification.py b/queue_services/business-emailer/tests/unit/email_processors/test_bn_notification.py index 597135701f..748a897fa4 100644 --- a/queue_services/business-emailer/tests/unit/email_processors/test_bn_notification.py +++ b/queue_services/business-emailer/tests/unit/email_processors/test_bn_notification.py @@ -54,7 +54,7 @@ def test_bn_move_notificaton(app, session): """Assert that the bn move email processor builds the email correctly.""" # setup filing + business for email identifier = 'FM1234567' - filing = prep_registration_filing(session, identifier, '1', 'COMPLETED', + filing = prep_registration_filing(session, identifier, 'COMPLETED', Business.LegalTypes.SOLE_PROP.value, 'test business') token = 'token' business = Business.find_by_identifier(identifier) diff --git a/queue_services/business-emailer/tests/unit/email_processors/test_cease_receiver_notification.py b/queue_services/business-emailer/tests/unit/email_processors/test_cease_receiver_notification.py deleted file mode 100644 index 22be22b6f4..0000000000 --- a/queue_services/business-emailer/tests/unit/email_processors/test_cease_receiver_notification.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright © 2026 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -"""The Unit Tests for the Cease Receiver email processor.""" -import base64 -from unittest.mock import patch - -import pytest -import requests_mock -from business_model.models import Business - -from business_emailer.email_processors import cease_receiver_notification -from tests.unit import prep_cease_receiver_filing - - -@pytest.mark.skip('This email processor is invalid and needs to be updated') -@pytest.mark.parametrize('status,legal_name,is_numbered', [ - ('COMPLETED', 'test business', False), - ('COMPLETED', 'BC1234567', True), -]) -def test_cease_receiver_notification(app, session, status, legal_name, is_numbered): - """Assert that the cease_receiver email processor builds the expected email.""" - # setup filing + business for email - filing = prep_cease_receiver_filing( - 'BC1234567', '1', Business.LegalTypes.COMP.value, legal_name) - token = 'token' - # test processor - with patch.object(cease_receiver_notification, '_get_pdfs', return_value=[]) as mock_get_pdfs: - with patch.object(cease_receiver_notification, 'get_recipient_from_auth', - return_value='recipient@email.com'): - email = cease_receiver_notification.process( - {'filingId': filing.id, 'type': 'ceaseReceiver', 'option': status}, token) - - if is_numbered: - assert email['content']['subject'] == 'Numbered Company - Receiver Ceased' - else: - assert email['content']['subject'] == f'{legal_name} - Receiver Ceased' - - assert email['recipients'] == 'recipient@email.com' - assert email['requestBy'] == 'BCRegistries@gov.bc.ca' - assert email['content']['body'] - assert email['content']['attachments'] == [] - - assert mock_get_pdfs.call_args[0][0] == token - assert mock_get_pdfs.call_args[0][1]['identifier'] == 'BC1234567' - assert mock_get_pdfs.call_args[0][1]['legalName'] == legal_name - assert mock_get_pdfs.call_args[0][1]['legalType'] == Business.LegalTypes.COMP.value - assert mock_get_pdfs.call_args[0][2] == filing - - -@pytest.mark.skip('This email processor is invalid and needs to be updated') -def test_cease_receiver_attachments(session, config): - """Assert _get_pdfs assembles the Cease Receiver filing PDF and receipt.""" - identifier = 'BC1234567' - filing = prep_cease_receiver_filing( - identifier, '1', Business.LegalTypes.COMP.value, 'test business') - token = 'token' - with patch.object(cease_receiver_notification, 'get_recipient_from_auth', - return_value='recipient@email.com'): - with requests_mock.Mocker() as m: - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{identifier}' - f'/filings/{filing.id}/documents/ceaseReceiver', - content=b'pdf_content_1', - status_code=200, - ) - m.post( - f'{config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - content=b'pdf_content_2', - status_code=201, - ) - output = cease_receiver_notification.process( - {'filingId': filing.id, 'type': 'ceaseReceiver', 'option': 'COMPLETED'}, token) - - attachments = output['content']['attachments'] - assert len(attachments) == 2 - assert attachments[0]['fileName'] == 'Cease Receiver.pdf' - assert base64.b64decode(attachments[0]['fileBytes']).decode('utf-8') == 'pdf_content_1' - assert attachments[1]['fileName'] == 'Receipt.pdf' - assert base64.b64decode(attachments[1]['fileBytes']).decode('utf-8') == 'pdf_content_2' diff --git a/queue_services/business-emailer/tests/unit/email_processors/test_consent_continuation_out_notification.py b/queue_services/business-emailer/tests/unit/email_processors/test_consent_continuation_out_notification.py deleted file mode 100644 index 27c7fc0797..0000000000 --- a/queue_services/business-emailer/tests/unit/email_processors/test_consent_continuation_out_notification.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright © 2023 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""The Unit Tests for the Consent Continuation Out email processor.""" -import base64 -from unittest.mock import patch - -import pytest -import requests_mock -from business_model.models import Business - -from business_emailer.email_processors import consent_continuation_out_notification -from tests.unit import prep_consent_continuation_out_filing - - -@pytest.mark.parametrize('status,legal_type,submitter_role', [ - ('COMPLETED', Business.LegalTypes.COMP.value, None), - ('COMPLETED', Business.LegalTypes.BCOMP.value, None), - ('COMPLETED', Business.LegalTypes.BC_CCC.value, None), - ('COMPLETED', Business.LegalTypes.BC_ULC_COMPANY.value, None), - ('COMPLETED', Business.LegalTypes.COMP.value, 'staff'), - ('COMPLETED', Business.LegalTypes.BCOMP.value, 'staff'), - ('COMPLETED', Business.LegalTypes.BC_CCC.value, 'staff'), - ('COMPLETED', Business.LegalTypes.BC_ULC_COMPANY.value, 'staff') -]) -def test_consent_continuation_out_notification(app, session, status, legal_type, submitter_role): - """Assert that the consent_continuation_out email processor for corps works as expected.""" - # setup filing + business for email - legal_name = 'test business' - filing = prep_consent_continuation_out_filing(session, 'BC1234567', '1', legal_type, legal_name, submitter_role) - token = 'token' - # test processor - with patch.object(consent_continuation_out_notification, '_get_pdfs', return_value=[]) as mock_get_pdfs: - with patch.object(consent_continuation_out_notification, 'get_recipient_from_auth', - return_value='recipient@email.com'): - email = consent_continuation_out_notification.process( - {'filingId': filing.id, 'type': 'consentContinuationOut', 'option': status}, token) - assert email['content']['subject'] == \ - legal_name + ' - Confirmation of Filing from the Business Registry' - - if submitter_role: - assert f'{submitter_role}@email.com' in email['recipients'] - assert 'recipient@email.com' in email['recipients'] - assert email['content']['body'] - assert email['content']['attachments'] == [] - assert mock_get_pdfs.call_args[0][0] == token - assert mock_get_pdfs.call_args[0][1]['identifier'] == 'BC1234567' - assert mock_get_pdfs.call_args[0][1]['legalName'] == legal_name - assert mock_get_pdfs.call_args[0][1]['legalType'] == legal_type - assert mock_get_pdfs.call_args[0][2] == filing - - -def test_consent_continuation_out_attachments(session, config): - """Assert _get_pdfs assembles the Letter of Consent and receipt.""" - identifier = 'BC1234567' - filing = prep_consent_continuation_out_filing( - session, identifier, '1', Business.LegalTypes.COMP.value, 'test business', None) - token = 'token' - with patch.object(consent_continuation_out_notification, 'get_recipient_from_auth', - return_value='recipient@email.com'): - with requests_mock.Mocker() as m: - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{identifier}' - f'/filings/{filing.id}/documents/letterOfConsent', - content=b'pdf_content_1', - status_code=200, - ) - m.post( - f'{config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - content=b'pdf_content_2', - status_code=201, - ) - output = consent_continuation_out_notification.process( - {'filingId': filing.id, 'type': 'consentContinuationOut', 'option': 'COMPLETED'}, token) - - attachments = output['content']['attachments'] - assert len(attachments) == 2 - assert attachments[0]['fileName'] == 'Letter of Consent.pdf' - assert base64.b64decode(attachments[0]['fileBytes']).decode('utf-8') == 'pdf_content_1' - assert attachments[1]['fileName'] == 'Receipt.pdf' - assert base64.b64decode(attachments[1]['fileBytes']).decode('utf-8') == 'pdf_content_2' diff --git a/queue_services/business-emailer/tests/unit/email_processors/test_continuation_in_notification.py b/queue_services/business-emailer/tests/unit/email_processors/test_continuation_in_notification.py index a399c4d88b..3d06624812 100644 --- a/queue_services/business-emailer/tests/unit/email_processors/test_continuation_in_notification.py +++ b/queue_services/business-emailer/tests/unit/email_processors/test_continuation_in_notification.py @@ -19,16 +19,16 @@ import requests_mock from business_model.models import Filing -from business_emailer.email_processors import continuation_in_notification +from business_emailer.email_processors import continuation_authorization_notification from tests.unit import CONTACT_POINT, prep_bootstrap_filing @pytest.mark.parametrize('status, subject, content', [ - (Filing.Status.APPROVED.value, 'Authorization Approved', 'Results of your Continuation Authorization'), - (Filing.Status.AWAITING_REVIEW.value, 'Authorization Documents Received', 'We have received your Continuation Authorization documents'), - (Filing.Status.CHANGE_REQUESTED.value, 'Changes Needed to Authorization', 'Make changes to your Continuation Authorization'), - (Filing.Status.REJECTED.value, 'Authorization Rejected', 'rejected'), - ('RESUBMITTED', 'Authorization Updates Received', 'your updated'), + (Filing.Status.APPROVED.value, 'Continuation Authorization Results', 'Results of your Continuation Authorization'), + (Filing.Status.AWAITING_REVIEW.value, 'Continuation Authorization Documents Received', 'We have received your Continuation Authorization documents'), + (Filing.Status.CHANGE_REQUESTED.value, 'Change Requested for your Continuation Authorization', 'Change Requested to your Continuation Authorization'), + (Filing.Status.REJECTED.value, 'Continuation Authorization Results', 'Results of your Continuation Authorization'), + ('RESUBMITTED', 'Updated Continuation Authorization Documents Received', 'We have received your updated Continuation Authorization documents'), ]) def test_continuation_in_notification(app, session, mocker, status, subject, content): """Assert Continuation review notification is created.""" @@ -37,8 +37,8 @@ def test_continuation_in_notification(app, session, mocker, status, subject, con token = 'token' # test processor - with patch.object(continuation_in_notification, '_get_pdfs', return_value=[]) as mock_get_pdfs: - email = continuation_in_notification.process( + with patch.object(continuation_authorization_notification, '_get_pdfs', return_value=[]) as mock_get_pdfs: + email = continuation_authorization_notification.process( {'filingId': filing.id, 'type': 'continuationIn', 'option': status}, token) assert CONTACT_POINT in email['recipients'] @@ -65,7 +65,7 @@ def test_continuation_in_attachments_resubmitted(session, mocker, config): content=b'pdf_content_1', status_code=200, ) - output = continuation_in_notification.process( + output = continuation_authorization_notification.process( {'filingId': filing.id, 'type': 'continuationIn', 'option': 'RESUBMITTED'}, token) attachments = output['content']['attachments'] diff --git a/queue_services/business-emailer/tests/unit/email_processors/test_continuation_out_notification.py b/queue_services/business-emailer/tests/unit/email_processors/test_continuation_out_notification.py deleted file mode 100644 index 841486eccd..0000000000 --- a/queue_services/business-emailer/tests/unit/email_processors/test_continuation_out_notification.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright © 2023 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""The Unit Tests for the Continuation Out email processor.""" -import base64 -from unittest.mock import patch - -import pytest -import requests_mock -from business_model.models import Business - -from business_emailer.email_processors import continuation_out_notification -from tests.unit import prep_continuation_out_filing - - -@pytest.mark.parametrize('status,legal_type,submitter_role', [ - ('COMPLETED', Business.LegalTypes.COMP.value, None), - ('COMPLETED', Business.LegalTypes.BCOMP.value, None), - ('COMPLETED', Business.LegalTypes.BC_CCC.value, None), - ('COMPLETED', Business.LegalTypes.BC_ULC_COMPANY.value, None), - ('COMPLETED', Business.LegalTypes.COMP.value, 'staff'), - ('COMPLETED', Business.LegalTypes.BCOMP.value, 'staff'), - ('COMPLETED', Business.LegalTypes.BC_CCC.value, 'staff'), - ('COMPLETED', Business.LegalTypes.BC_ULC_COMPANY.value, 'staff') -]) -def test_continuation_out_notification(app, session, status, legal_type, submitter_role): - """Assert that the continuation_out email processor for corps works as expected.""" - # setup filing + business for email - legal_name = 'test business' - filing = prep_continuation_out_filing(session, 'BC1234567', '1', legal_type, legal_name, submitter_role) - token = 'token' - # test processor - with patch.object(continuation_out_notification, '_get_pdfs', return_value=[]) as mock_get_pdfs: - with patch.object(continuation_out_notification, 'get_recipient_from_auth', - return_value='recipient@email.com'): - email = continuation_out_notification.process( - {'filingId': filing.id, 'type': 'continuationOut', 'option': status}, token) - assert email['content']['subject'] == \ - legal_name + ' - Confirmation of Filing from the Business Registry' - - if submitter_role: - assert f'{submitter_role}@email.com' in email['recipients'] - assert 'recipient@email.com' in email['recipients'] - assert email['content']['body'] - assert email['content']['attachments'] == [] - assert mock_get_pdfs.call_args[0][0] == token - assert mock_get_pdfs.call_args[0][1]['identifier'] == 'BC1234567' - assert mock_get_pdfs.call_args[0][1]['legalName'] == legal_name - assert mock_get_pdfs.call_args[0][1]['legalType'] == legal_type - assert mock_get_pdfs.call_args[0][2] == filing - - -def test_continuation_out_attachments(session, config): - """Assert _get_pdfs assembles just the receipt (no filing PDF for Continuation Out).""" - identifier = 'BC1234567' - filing = prep_continuation_out_filing( - session, identifier, '1', Business.LegalTypes.COMP.value, 'test business', None) - token = 'token' - with patch.object(continuation_out_notification, 'get_recipient_from_auth', - return_value='recipient@email.com'): - with requests_mock.Mocker() as m: - m.post( - f'{config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - content=b'pdf_content_1', - status_code=201, - ) - output = continuation_out_notification.process( - {'filingId': filing.id, 'type': 'continuationOut', 'option': 'COMPLETED'}, token) - - attachments = output['content']['attachments'] - assert len(attachments) == 1 - assert attachments[0]['fileName'] == 'Receipt.pdf' - assert base64.b64decode(attachments[0]['fileBytes']).decode('utf-8') == 'pdf_content_1' diff --git a/queue_services/business-emailer/tests/unit/email_processors/test_correction_notification.py b/queue_services/business-emailer/tests/unit/email_processors/test_correction_notification.py deleted file mode 100644 index 6d3e5b8e1a..0000000000 --- a/queue_services/business-emailer/tests/unit/email_processors/test_correction_notification.py +++ /dev/null @@ -1,319 +0,0 @@ -# Copyright © 2022 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""The Unit Tests for the Correction email processor.""" -import base64 -from unittest.mock import patch - -import pytest -import requests_mock -from business_model.models import Business - -from business_emailer.email_processors import correction_notification -from tests.unit import ( - prep_cp_special_resolution_correction_filing, - prep_cp_special_resolution_correction_upload_memorandum_filing, - prep_cp_special_resolution_filing, - prep_firm_correction_filing, - prep_incorp_filing, - prep_incorporation_correction_filing, -) - - -COMPLETED_SUBJECT_SUFIX = ' - Correction Documents from the Business Registry' -CP_IDENTIFIER = 'CP1234567' -SPECIAL_RESOLUTION_FILING_TYPE = 'specialResolution' - - -@pytest.mark.parametrize('status,legal_type', [ - ('PAID', Business.LegalTypes.SOLE_PROP.value), - ('COMPLETED', Business.LegalTypes.SOLE_PROP.value), - ('PAID', Business.LegalTypes.PARTNERSHIP.value), - ('COMPLETED', Business.LegalTypes.PARTNERSHIP.value), -]) -def test_firm_correction_notification(app, session, status, legal_type): - """Assert that email attributes are correct.""" - # setup filing + business for email - legal_name = 'test business' - parties = [{ - 'firstName': 'Jane', - 'lastName': 'Doe', - 'middleInitial': 'A', - 'partyType': 'person', - 'organizationName': '' - }] - filing = prep_firm_correction_filing(session, 'FM1234567', '1', legal_type, legal_name, 'staff', parties) - token = 'token' - # test processor - with patch.object(correction_notification, '_get_pdfs', return_value=[]) as mock_get_pdfs: - email = correction_notification.process( - {'filingId': filing.id, 'type': 'correction', 'option': status}, token) - if status == 'PAID': - assert email['content']['subject'] == 'JANE A DOE - Confirmation of Filing from the Business Registry' - else: - assert email['content']['subject'] == \ - 'JANE A DOE' + COMPLETED_SUBJECT_SUFIX - - if status == 'COMPLETED': - assert 'no_one@never.get' in email['recipients'] - if legal_type == Business.LegalTypes.PARTNERSHIP.value: - assert 'party@email.com' in email['recipients'] - - assert email['content']['body'] - assert email['content']['attachments'] == [] - assert mock_get_pdfs.call_args[0][0] == status - assert mock_get_pdfs.call_args[0][1] == token - if status == 'COMPLETED': - assert mock_get_pdfs.call_args[0][2]['identifier'] == 'FM1234567' - assert mock_get_pdfs.call_args[0][3] == filing - - -@pytest.mark.parametrize('status,legal_type', [ - ('PAID', Business.LegalTypes.COMP.value), - ('COMPLETED', Business.LegalTypes.COMP.value), - ('PAID', Business.LegalTypes.BCOMP.value), - ('COMPLETED', Business.LegalTypes.BCOMP.value), - ('PAID', Business.LegalTypes.BC_CCC.value), - ('COMPLETED', Business.LegalTypes.BC_CCC.value), - ('PAID', Business.LegalTypes.BC_ULC_COMPANY.value), - ('COMPLETED', Business.LegalTypes.BC_ULC_COMPANY.value), -]) -def test_bc_correction_notification(app, session, status, legal_type): - """Assert that email attributes are correct.""" - # setup filing + business for email - legal_name = 'test business' - original_filing = prep_incorp_filing(session, 'BC1234567', 'COMPLETED', legal_type=legal_type) - token = 'token' - business = Business.find_by_identifier('BC1234567') - filing = prep_incorporation_correction_filing(session, business, original_filing.id, '1', status) - # test processor - with patch.object(correction_notification, '_get_pdfs', return_value=[]) as mock_get_pdfs: - email = correction_notification.process( - {'filingId': filing.id, 'type': 'correction', 'option': status}, token) - if status == 'PAID': - assert email['content']['subject'] == legal_name + ' - Confirmation of Filing from the Business Registry' - else: - assert email['content']['subject'] == \ - legal_name + COMPLETED_SUBJECT_SUFIX - - assert 'comp_party@email.com' in email['recipients'] - assert 'test@test.com' in email['recipients'] - - assert email['content']['body'] - assert email['content']['attachments'] == [] - - assert mock_get_pdfs.call_args[0][0] == status - assert mock_get_pdfs.call_args[0][1] == token - - if status == 'COMPLETED': - assert mock_get_pdfs.call_args[0][2]['identifier'] == 'BC1234567' - assert mock_get_pdfs.call_args[0][3] == filing - - -@pytest.mark.parametrize('status,legal_type', [ - ('PAID', Business.LegalTypes.COOP.value), - ('COMPLETED', Business.LegalTypes.COOP.value), -]) -def test_cp_special_resolution_correction_notification(app, session, status, legal_type): - """Assert that email attributes are correct.""" - # setup filing + business for email - legal_name = 'cp business' - original_filing = prep_cp_special_resolution_filing(CP_IDENTIFIER, '1', legal_type, legal_name, submitter_role=None) - token = 'token' - business = Business.find_by_identifier(CP_IDENTIFIER) - filing = prep_cp_special_resolution_correction_filing(session, business, original_filing.id, - '1', status, SPECIAL_RESOLUTION_FILING_TYPE) - # test processor - with patch.object(correction_notification, '_get_pdfs', return_value=[]) as mock_get_pdfs: - email = correction_notification.process( - {'filingId': filing.id, 'type': 'correction', 'option': status}, token) - if status == 'PAID': - assert email['content']['subject'] == legal_name + ' - Confirmation of correction' - else: - assert email['content']['subject'] == \ - legal_name + COMPLETED_SUBJECT_SUFIX - - assert 'cp_sr@test.com' in email['recipients'] - - assert email['content']['body'] - assert email['content']['attachments'] == [] - - assert mock_get_pdfs.call_args[0][0] == status - assert mock_get_pdfs.call_args[0][1] == token - - if status == 'COMPLETED': - assert mock_get_pdfs.call_args[0][2]['identifier'] == CP_IDENTIFIER - assert mock_get_pdfs.call_args[0][3] == filing - - -def test_complete_special_resolution_correction_attachments(session, config): - """Test completed special resolution correction notification.""" - # setup filing + business for email - legal_type = Business.LegalTypes.COOP.value - legal_name = 'test cp sr business' - token = 'token' - status = 'COMPLETED' - original_filing = prep_cp_special_resolution_filing(CP_IDENTIFIER, '1', legal_type, legal_name, submitter_role=None) - business = Business.find_by_identifier(CP_IDENTIFIER) - filing = prep_cp_special_resolution_correction_filing(session, business, original_filing.id, - '1', status, SPECIAL_RESOLUTION_FILING_TYPE) - with requests_mock.Mocker() as m: - m.get( - ( - f'{config.get("LEGAL_API_URL")}' - f'/businesses/{CP_IDENTIFIER}' - f'/filings/{filing.id}' - '/documents/specialResolution' - ), - content=b'pdf_content_1', - status_code=200 - ) - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{CP_IDENTIFIER}/filings/{filing.id}' - '/documents/certificateOfNameChange', - content=b'pdf_content_2', - status_code=200 - ) - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{CP_IDENTIFIER}/filings/{filing.id}/documents/certifiedRules', - content=b'pdf_content_3', - status_code=200 - ) - - output = correction_notification.process({ - 'filingId': filing.id, - 'type': 'correction', - 'option': status - }, token) - assert 'content' in output - assert 'attachments' in output['content'] - assert len(output['content']['attachments']) == 3 - assert output['content']['attachments'][0]['fileName'] == 'Special Resolution.pdf' - assert base64.b64decode(output['content']['attachments'][0]['fileBytes']).decode('utf-8') == 'pdf_content_1' - assert output['content']['attachments'][1]['fileName'] == 'Certificate of Name Change.pdf' - assert base64.b64decode(output['content']['attachments'][1]['fileBytes']).decode('utf-8') == 'pdf_content_2' - assert output['content']['attachments'][2]['fileName'] == 'Certified Rules.pdf' - assert base64.b64decode(output['content']['attachments'][2]['fileBytes']).decode('utf-8') == 'pdf_content_3' - - -def test_paid_special_resolution_correction_attachments(session, config): - """Test paid special resolution correction notification.""" - # setup filing + business for email - legal_type = Business.LegalTypes.COOP.value - legal_name = 'test cp sr business' - token = 'token' - status = 'PAID' - original_filing = prep_cp_special_resolution_filing(CP_IDENTIFIER, '1', legal_type, legal_name, submitter_role=None) - business = Business.find_by_identifier(CP_IDENTIFIER) - filing = prep_cp_special_resolution_correction_filing(session, business, original_filing.id, - '1', status, SPECIAL_RESOLUTION_FILING_TYPE) - with requests_mock.Mocker() as m: - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{CP_IDENTIFIER}/filings/{filing.id}/documents/correction', - content=b'pdf_content_1', - status_code=200 - ) - m.post( - f'{config.get("PAY_API_URL")}/1/receipts', - content=b'pdf_content_2', - status_code=201 - ) - output = correction_notification.process({ - 'filingId': filing.id, - 'type': 'correction', - 'option': status - }, token) - assert 'content' in output - assert 'attachments' in output['content'] - assert len(output['content']['attachments']) == 2 - assert output['content']['attachments'][0]['fileName'] == 'Register Correction Application.pdf' - assert base64.b64decode(output['content']['attachments'][0]['fileBytes']).decode('utf-8') == 'pdf_content_1' - assert output['content']['attachments'][1]['fileName'] == 'Receipt.pdf' - assert base64.b64decode(output['content']['attachments'][1]['fileBytes']).decode('utf-8') == 'pdf_content_2' - - -@pytest.mark.parametrize('legal_type, filing_type', [ - (Business.LegalTypes.COOP.value, SPECIAL_RESOLUTION_FILING_TYPE), - (Business.LegalTypes.CCC_CONTINUE_IN.value, SPECIAL_RESOLUTION_FILING_TYPE), - (Business.LegalTypes.COOP.value, 'registration'), -]) -def test_paid_special_resolution_correction_on_correction(session, config, legal_type, filing_type): - """Assert that email attributes are correct.""" - # setup filing + business for email - legal_name = 'cp business' - original_filing = prep_cp_special_resolution_filing(CP_IDENTIFIER, '1', legal_type, - legal_name, submitter_role=None) - token = 'token' - business = Business.find_by_identifier(CP_IDENTIFIER) - filing_correction = prep_cp_special_resolution_correction_filing(session, business, original_filing.id, - '1', 'COMPLETED', filing_type) - filing = prep_cp_special_resolution_correction_filing(session, business, filing_correction.id, - '1', 'PAID', 'correction') - # test processor - with patch.object(correction_notification, '_get_pdfs', return_value=[]): - email = correction_notification.process( - {'filingId': filing.id, 'type': 'correction', 'option': 'PAID'}, token) - if legal_type == Business.LegalTypes.COOP.value and filing_type == 'specialResolution': - assert email['content']['subject'] == legal_name + ' - Confirmation of correction' - assert 'cp_sr@test.com' in email['recipients'] - assert email['content']['body'] - assert email['content']['attachments'] == [] - - -def test_complete_special_resolution_correction_memorandum_attachments(session, config): - """Completed CP SR correction with memorandumFileKey → Certified Memorandum attached. - - The upload_memorandum prep adds `memorandumFileKey` but leaves the template's - default `rulesFileKey` in place, so `rules_changed` is also True in the - processor — we expect Special Resolution + Certified Rules + Certified - Memorandum (3 attachments). The point of this test is to exercise the - `if memorandum_changed:` branch in special_resolution_helper.get_completed_pdfs. - """ - legal_type = Business.LegalTypes.COOP.value - legal_name = 'test cp sr business' - token = 'token' - original_filing = prep_cp_special_resolution_filing( - CP_IDENTIFIER, '1', legal_type, legal_name, submitter_role=None) - business = Business.find_by_identifier(CP_IDENTIFIER) - filing = prep_cp_special_resolution_correction_upload_memorandum_filing( - session, business, original_filing.id, '1', 'COMPLETED', SPECIAL_RESOLUTION_FILING_TYPE) - with requests_mock.Mocker() as m: - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{CP_IDENTIFIER}' - f'/filings/{filing.id}/documents/specialResolution', - content=b'pdf_content_1', - status_code=200, - ) - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{CP_IDENTIFIER}' - f'/filings/{filing.id}/documents/certifiedRules', - content=b'pdf_content_2', - status_code=200, - ) - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{CP_IDENTIFIER}' - f'/filings/{filing.id}/documents/certifiedMemorandum', - content=b'pdf_content_3', - status_code=200, - ) - output = correction_notification.process( - {'filingId': filing.id, 'type': 'correction', 'option': 'COMPLETED'}, token) - - attachments = output['content']['attachments'] - assert len(attachments) == 3 - assert attachments[0]['fileName'] == 'Special Resolution.pdf' - assert base64.b64decode(attachments[0]['fileBytes']).decode('utf-8') == 'pdf_content_1' - assert attachments[1]['fileName'] == 'Certified Rules.pdf' - assert base64.b64decode(attachments[1]['fileBytes']).decode('utf-8') == 'pdf_content_2' - assert attachments[2]['fileName'] == 'Certified Memorandum.pdf' - assert base64.b64decode(attachments[2]['fileBytes']).decode('utf-8') == 'pdf_content_3' diff --git a/queue_services/business-emailer/tests/unit/email_processors/test_cp_special_resolution_notification.py b/queue_services/business-emailer/tests/unit/email_processors/test_cp_special_resolution_notification.py deleted file mode 100644 index ca5c719a94..0000000000 --- a/queue_services/business-emailer/tests/unit/email_processors/test_cp_special_resolution_notification.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright © 2021 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""The Unit Tests for the Special Resolution email processor.""" -import base64 -from unittest.mock import patch - -import pytest -import requests_mock -from business_model.models import Business - -from business_emailer.email_processors import special_resolution_notification -from tests.unit import prep_cp_special_resolution_filing - - -LEGAL_TYPE = Business.LegalTypes.COOP.value -LEGAL_NAME = 'test business' -IDENTIFIER = 'CP1234567' -TOKEN = 'token' -RECIPIENT_EMAIL = 'recipient@email.com' -USER_EMAIL_FROM_AUTH = 'user@email.com' - - -@pytest.mark.parametrize('status', [ - ('PAID'), - ('COMPLETED') -]) -def test_cp_special_resolution_notification(session, app, config, status): - """Assert that the special resolution email processor works as expected.""" - # setup filing + business for email - filing = prep_cp_special_resolution_filing(IDENTIFIER, '1', LEGAL_TYPE, LEGAL_NAME, submitter_role=None) - get_pdf_function = 'get_paid_pdfs' if status == 'PAID' else 'get_completed_pdfs' - # test processor - with patch.object(special_resolution_notification, get_pdf_function, return_value=[]) as mock_get_pdfs: - with patch.object(special_resolution_notification, 'get_recipient_from_auth', - return_value=RECIPIENT_EMAIL): - with patch.object(special_resolution_notification, 'get_user_email_from_auth', - return_value=USER_EMAIL_FROM_AUTH): - email = special_resolution_notification.process( - {'filingId': filing.id, 'type': 'specialResolution', 'option': status}, TOKEN) - if status == 'PAID': - assert email['content']['subject'] == LEGAL_NAME + \ - ' - Confirmation of Special Resolution from the Business Registry' - else: - assert email['content']['subject'] == \ - LEGAL_NAME + ' - Special Resolution Documents from the Business Registry' - - assert RECIPIENT_EMAIL in email['recipients'] - assert USER_EMAIL_FROM_AUTH in email['recipients'] - assert email['content']['body'] - assert email['content']['attachments'] == [] - assert mock_get_pdfs.call_args[0][0] == TOKEN - assert mock_get_pdfs.call_args[0][1]['identifier'] == IDENTIFIER - assert mock_get_pdfs.call_args[0][2] == filing - - -def test_complete_special_resolution_attachments(session, config): - """Test completed special resolution notification.""" - # setup filing + business for email - status = 'COMPLETED' - filing = prep_cp_special_resolution_filing(IDENTIFIER, '1', LEGAL_TYPE, LEGAL_NAME, submitter_role=None) - with requests_mock.Mocker() as m: - with patch.object(special_resolution_notification, 'get_recipient_from_auth', - return_value=RECIPIENT_EMAIL): - with patch.object(special_resolution_notification, 'get_user_email_from_auth', - return_value=USER_EMAIL_FROM_AUTH): - m.get( - ( - f'{config.get("LEGAL_API_URL")}' - f'/businesses/{IDENTIFIER}' - f'/filings/{filing.id}' - f'/documents/specialResolution' - ), - content=b'pdf_content_1', - status_code=200 - ) - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{IDENTIFIER}/filings/{filing.id}' - '/documents/certificateOfNameChange', - content=b'pdf_content_2', - status_code=200 - ) - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{IDENTIFIER}/filings/{filing.id}' - '/documents/certifiedRules', - content=b'pdf_content_3', - status_code=200 - ) - - output = special_resolution_notification.process({ - 'filingId': filing.id, - 'type': 'specialResolution', - 'option': status - }, TOKEN) - assert 'content' in output - assert 'attachments' in output['content'] - assert len(output['content']['attachments']) == 3 - assert output['content']['attachments'][0]['fileName'] == 'Special Resolution.pdf' - assert (base64.b64decode(output['content']['attachments'][0]['fileBytes']).decode('utf-8') - == 'pdf_content_1') - assert output['content']['attachments'][1]['fileName'] == 'Certificate of Name Change.pdf' - assert (base64.b64decode(output['content']['attachments'][1]['fileBytes']).decode('utf-8') - == 'pdf_content_2') - assert output['content']['attachments'][2]['fileName'] == 'Certified Rules.pdf' - assert (base64.b64decode(output['content']['attachments'][2]['fileBytes']).decode('utf-8') - == 'pdf_content_3') - - -def test_paid_special_resolution_attachments(session, config): - """Test paid special resolution notification.""" - # setup filing + business for email - status = 'PAID' - filing = prep_cp_special_resolution_filing(IDENTIFIER, '1', LEGAL_TYPE, LEGAL_NAME, submitter_role=None) - with requests_mock.Mocker() as m: - with patch.object(special_resolution_notification, 'get_recipient_from_auth', - return_value=RECIPIENT_EMAIL): - with patch.object(special_resolution_notification, 'get_user_email_from_auth', - return_value=USER_EMAIL_FROM_AUTH): - m.get( - ( - f'{config.get("LEGAL_API_URL")}' - f'/businesses/{IDENTIFIER}' - f'/filings/{filing.id}' - f'/documents/specialResolutionApplication' - ), - content=b'pdf_content_1', - status_code=200 - ) - m.post( - f'{config.get("PAY_API_URL")}/1/receipts', - content=b'pdf_content_2', - status_code=201 - ) - output = special_resolution_notification.process({ - 'filingId': filing.id, - 'type': 'specialResolution', - 'option': status - }, TOKEN) - assert 'content' in output - assert 'attachments' in output['content'] - assert len(output['content']['attachments']) == 2 - assert output['content']['attachments'][0]['fileName'] == 'Special Resolution Application.pdf' - assert (base64.b64decode(output['content']['attachments'][0]['fileBytes']).decode('utf-8') - == 'pdf_content_1') - assert output['content']['attachments'][1]['fileName'] == 'Receipt.pdf' - assert (base64.b64decode(output['content']['attachments'][1]['fileBytes']).decode('utf-8') - == 'pdf_content_2') diff --git a/queue_services/business-emailer/tests/unit/email_processors/test_dissolution_notification.py b/queue_services/business-emailer/tests/unit/email_processors/test_dissolution_notification.py deleted file mode 100644 index 64891fe54b..0000000000 --- a/queue_services/business-emailer/tests/unit/email_processors/test_dissolution_notification.py +++ /dev/null @@ -1,272 +0,0 @@ -# Copyright © 2021 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""The Unit Tests for the Dissolution email processor.""" -import base64 -from unittest.mock import patch - -import pytest -import requests_mock -from business_model.models import Business - -from business_emailer.email_processors import dissolution_notification -from tests.unit import prep_dissolution_filing - - -@pytest.mark.parametrize('status,legal_type,submitter_role', [ - ('PAID', Business.LegalTypes.COMP.value, None), - ('COMPLETED', Business.LegalTypes.COMP.value, None), - ('PAID', Business.LegalTypes.BCOMP.value, None), - ('COMPLETED', Business.LegalTypes.BCOMP.value, None), - ('PAID', Business.LegalTypes.BC_CCC.value, None), - ('COMPLETED', Business.LegalTypes.BC_CCC.value, None), - ('PAID', Business.LegalTypes.BC_ULC_COMPANY.value, None), - ('COMPLETED', Business.LegalTypes.BC_ULC_COMPANY.value, None), - ('PAID', Business.LegalTypes.COOP.value, None), - ('COMPLETED', Business.LegalTypes.COOP.value, None), - - ('PAID', Business.LegalTypes.COMP.value, 'staff'), - ('COMPLETED', Business.LegalTypes.COMP.value, 'staff'), -]) -def test_dissolution_notification(app, session, status, legal_type, submitter_role): - """Assert that the dissolution email processor for corps works as expected.""" - # setup filing + business for email - legal_name = 'test business' - filing = prep_dissolution_filing(session, 'BC1234567', '1', status, legal_type, legal_name, submitter_role) - token = 'token' - # test processor - with patch.object(dissolution_notification, '_get_pdfs', return_value=[]) as mock_get_pdfs: - with patch.object(dissolution_notification, 'get_recipient_from_auth', return_value='recipient@email.com'): - with patch.object(dissolution_notification, 'get_user_email_from_auth', return_value='user@email.com'): - email = dissolution_notification.process( - {'filingId': filing.id, 'type': 'dissolution', 'option': status}, token) - if status == 'PAID': - assert email['content']['subject'] == legal_name + ' - Voluntary dissolution' - else: - assert email['content']['subject'] == \ - legal_name + ' - Confirmation of Dissolution from the Business Registry' - - if submitter_role: - assert f'{submitter_role}@email.com' in email['recipients'] - else: - assert 'user@email.com' in email['recipients'] - assert 'recipient@email.com' in email['recipients'] - assert 'custodian@email.com' in email['recipients'] - assert email['content']['body'] - assert email['content']['attachments'] == [] - assert mock_get_pdfs.call_args[0][0] == status - assert mock_get_pdfs.call_args[0][1] == token - assert mock_get_pdfs.call_args[0][2]['identifier'] == 'BC1234567' - assert mock_get_pdfs.call_args[0][2]['legalName'] == legal_name - assert mock_get_pdfs.call_args[0][2]['legalType'] == legal_type - assert mock_get_pdfs.call_args[0][3] == filing - - -@pytest.mark.parametrize('status,legal_type,submitter_role', [ - ('PAID', Business.LegalTypes.SOLE_PROP.value, None), - ('COMPLETED', Business.LegalTypes.SOLE_PROP.value, None), - ('PAID', Business.LegalTypes.PARTNERSHIP.value, None), - ('COMPLETED', Business.LegalTypes.PARTNERSHIP.value, None), -]) -def test_firms_dissolution_notification(app, session, status, legal_type, submitter_role): - """Assert that the dissolution email processor for firms works as expected.""" - # setup filing + business for email - legal_name = 'test business' - parties = [{ - 'firstName': 'Jane', - 'lastName': 'Doe', - 'middleInitial': 'A', - 'partyType': 'person', - 'organizationName': '' - }] - filing = prep_dissolution_filing(session, 'FM1234567', '1', status, legal_type, legal_name, submitter_role, parties) - token = 'token' - # test processor - with patch.object(dissolution_notification, '_get_pdfs', return_value=[]) as mock_get_pdfs: - with patch.object(dissolution_notification, 'get_recipient_from_auth', return_value='recipient@email.com'): - with patch.object(dissolution_notification, 'get_user_email_from_auth', return_value='user@email.com'): - email = dissolution_notification.process( - {'filingId': filing.id, 'type': 'dissolution', 'option': status}, token) - if status == 'PAID': - assert email['content']['subject'] == \ - f'{legal_name} - Confirmation of Filing from the Business Registry' - else: - assert email['content']['subject'] == \ - f'{legal_name} - Dissolution Documents from the Business Registry' - - if submitter_role: - assert f'{submitter_role}@email.com' in email['recipients'] - else: - assert 'user@email.com' in email['recipients'] - assert 'recipient@email.com' in email['recipients'] - assert 'cp@email.com' in email['recipients'] - assert email['content']['body'] - assert email['content']['attachments'] == [] - assert mock_get_pdfs.call_args[0][0] == status - assert mock_get_pdfs.call_args[0][1] == token - assert mock_get_pdfs.call_args[0][2]['identifier'] == 'FM1234567' - assert mock_get_pdfs.call_args[0][2]['legalName'] == 'JANE A DOE' - assert mock_get_pdfs.call_args[0][2]['legalType'] == legal_type - assert mock_get_pdfs.call_args[0][3] == filing - - -def test_dissolution_attachments_paid_bc(session, config): - """PAID BC: Voluntary Dissolution Application + receipt.""" - identifier = 'BC1234567' - filing = prep_dissolution_filing( - session, identifier, '1', 'PAID', Business.LegalTypes.COMP.value, 'test business', None) - token = 'token' - with patch.object(dissolution_notification, 'get_recipient_from_auth', return_value='recipient@email.com'): - with patch.object(dissolution_notification, 'get_user_email_from_auth', return_value='user@email.com'): - with requests_mock.Mocker() as m: - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{identifier}' - f'/filings/{filing.id}/documents/dissolution', - content=b'pdf_content_1', - status_code=200, - ) - m.post( - f'{config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - content=b'pdf_content_2', - status_code=201, - ) - output = dissolution_notification.process( - {'filingId': filing.id, 'type': 'dissolution', 'option': 'PAID'}, token) - - attachments = output['content']['attachments'] - assert len(attachments) == 2 - assert attachments[0]['fileName'] == 'Voluntary Dissolution Application.pdf' - assert base64.b64decode(attachments[0]['fileBytes']).decode('utf-8') == 'pdf_content_1' - assert attachments[1]['fileName'] == 'Receipt.pdf' - assert base64.b64decode(attachments[1]['fileBytes']).decode('utf-8') == 'pdf_content_2' - - -def test_dissolution_attachments_paid_sp_receipt_only(session, config): - """PAID SP: no filing PDF (SP/GP excluded) — receipt only.""" - identifier = 'FM1234567' - parties = [{ - 'firstName': 'Jane', 'lastName': 'Doe', 'middleInitial': 'A', - 'partyType': 'person', 'organizationName': '' - }] - filing = prep_dissolution_filing( - session, identifier, '1', 'PAID', Business.LegalTypes.SOLE_PROP.value, - 'test business', None, parties) - token = 'token' - with patch.object(dissolution_notification, 'get_recipient_from_auth', return_value='recipient@email.com'): - with patch.object(dissolution_notification, 'get_user_email_from_auth', return_value='user@email.com'): - with requests_mock.Mocker() as m: - m.post( - f'{config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - content=b'pdf_content_1', - status_code=201, - ) - output = dissolution_notification.process( - {'filingId': filing.id, 'type': 'dissolution', 'option': 'PAID'}, token) - - attachments = output['content']['attachments'] - assert len(attachments) == 1 - assert attachments[0]['fileName'] == 'Receipt.pdf' - assert base64.b64decode(attachments[0]['fileBytes']).decode('utf-8') == 'pdf_content_1' - - -def test_dissolution_attachments_completed_sp(session, config): - """COMPLETED SP: Statement of Dissolution only.""" - identifier = 'FM1234567' - parties = [{ - 'firstName': 'Jane', 'lastName': 'Doe', 'middleInitial': 'A', - 'partyType': 'person', 'organizationName': '' - }] - filing = prep_dissolution_filing( - session, identifier, '1', 'COMPLETED', Business.LegalTypes.SOLE_PROP.value, - 'test business', None, parties) - token = 'token' - with patch.object(dissolution_notification, 'get_recipient_from_auth', return_value='recipient@email.com'): - with patch.object(dissolution_notification, 'get_user_email_from_auth', return_value='user@email.com'): - with requests_mock.Mocker() as m: - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{identifier}' - f'/filings/{filing.id}/documents/dissolution', - content=b'pdf_content_1', - status_code=200, - ) - output = dissolution_notification.process( - {'filingId': filing.id, 'type': 'dissolution', 'option': 'COMPLETED'}, token) - - attachments = output['content']['attachments'] - assert len(attachments) == 1 - assert attachments[0]['fileName'] == 'Statement of Dissolution.pdf' - assert base64.b64decode(attachments[0]['fileBytes']).decode('utf-8') == 'pdf_content_1' - - -def test_dissolution_attachments_completed_bc(session, config): - """COMPLETED BC (non-admin): Certificate of Dissolution.""" - identifier = 'BC1234567' - filing = prep_dissolution_filing( - session, identifier, '1', 'COMPLETED', Business.LegalTypes.COMP.value, 'test business', None) - token = 'token' - with patch.object(dissolution_notification, 'get_recipient_from_auth', return_value='recipient@email.com'): - with patch.object(dissolution_notification, 'get_user_email_from_auth', return_value='user@email.com'): - with requests_mock.Mocker() as m: - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{identifier}' - f'/filings/{filing.id}/documents/certificateOfDissolution', - content=b'pdf_content_1', - status_code=200, - ) - output = dissolution_notification.process( - {'filingId': filing.id, 'type': 'dissolution', 'option': 'COMPLETED'}, token) - - attachments = output['content']['attachments'] - assert len(attachments) == 1 - assert attachments[0]['fileName'] == 'Certificate of Dissolution.pdf' - assert base64.b64decode(attachments[0]['fileBytes']).decode('utf-8') == 'pdf_content_1' - - -def test_dissolution_attachments_completed_coop(session, config): - """COMPLETED COOP: Certificate of Dissolution + Certified Affidavit + Certified Special Resolution.""" - identifier = 'CP1234567' - filing = prep_dissolution_filing( - session, identifier, '1', 'COMPLETED', Business.LegalTypes.COOP.value, 'test business', None) - token = 'token' - with patch.object(dissolution_notification, 'get_recipient_from_auth', return_value='recipient@email.com'): - with patch.object(dissolution_notification, 'get_user_email_from_auth', return_value='user@email.com'): - with requests_mock.Mocker() as m: - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{identifier}' - f'/filings/{filing.id}/documents/certificateOfDissolution', - content=b'pdf_content_1', - status_code=200, - ) - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{identifier}' - f'/filings/{filing.id}/documents/affidavit', - content=b'pdf_content_2', - status_code=200, - ) - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{identifier}' - f'/filings/{filing.id}/documents/specialResolution', - content=b'pdf_content_3', - status_code=200, - ) - output = dissolution_notification.process( - {'filingId': filing.id, 'type': 'dissolution', 'option': 'COMPLETED'}, token) - - attachments = output['content']['attachments'] - assert len(attachments) == 3 - assert attachments[0]['fileName'] == 'Certificate of Dissolution.pdf' - assert base64.b64decode(attachments[0]['fileBytes']).decode('utf-8') == 'pdf_content_1' - assert attachments[1]['fileName'] == 'Certified Affidavit.pdf' - assert base64.b64decode(attachments[1]['fileBytes']).decode('utf-8') == 'pdf_content_2' - assert attachments[2]['fileName'] == 'Certified Special Resolution.pdf' - assert base64.b64decode(attachments[2]['fileBytes']).decode('utf-8') == 'pdf_content_3' diff --git a/queue_services/business-emailer/tests/unit/email_processors/test_email_processors_init.py b/queue_services/business-emailer/tests/unit/email_processors/test_email_processors_init.py index 7ab0bf6afc..3767ed8519 100644 --- a/queue_services/business-emailer/tests/unit/email_processors/test_email_processors_init.py +++ b/queue_services/business-emailer/tests/unit/email_processors/test_email_processors_init.py @@ -107,12 +107,16 @@ def test_substitute_template_parts_md_replaces_footer_marker(app): def test_substitute_template_parts_md_replaces_all_md_parts(app): - """Assert that substitute_template_parts with file_type='md' replaces all five markdown common parts.""" + """Assert that substitute_template_parts with file_type='md' replaces all markdown common parts.""" template = ( "[[attachments.md]]\n" "[[business-number.md]]\n" "[[business-registry-footer.md]]\n" "[[business-tombstone.md]]\n" + "[[business-tombstone-out-filing.md]]\n" + "[[consent.md]]\n" + "[[consent-next-steps.md]]\n" + "[[out-details.md]]\n" "[[what-happens-next.md]]" ) with app.app_context(): @@ -122,6 +126,10 @@ def test_substitute_template_parts_md_replaces_all_md_parts(app): '[[business-number.md]]', '[[business-registry-footer.md]]', '[[business-tombstone.md]]', + '[[business-tombstone-out-filing.md]]', + '[[consent.md]]', + '[[consent-next-steps.md]]', + '[[out-details.md]]', '[[what-happens-next.md]]', ]: assert marker not in result @@ -314,6 +322,18 @@ def test_get_recipients_mras_option_excludes_party_emails(app, filing_type): assert 'comp@example.com' not in result +def test_get_recipients_dissolution_includes_party_emails(app): + """Assert that dissolution recipients include the custodian party email.""" + filing_json = _make_filing_json( + 'dissolution', + parties=[{'officer': {'email': 'custodian@example.com'}, 'roles': [{'roleType': 'Custodian'}]}] + ) + with app.app_context(): + result = get_recipients('COMPLETED', filing_json, None, 'dissolution') + assert CONTACT_POINT in result + assert 'custodian@example.com' in result + + def test_get_recipients_missing_contact_point_coalesced_to_empty_string(app): """Assert that a missing contactPoint email is coalesced to an empty string (not None).""" filing_json = _make_filing_json( @@ -326,23 +346,8 @@ def test_get_recipients_missing_contact_point_coalesced_to_empty_string(app): assert result == '' -def test_get_recipients_coop_identifier_skips_auth_call(app): - """Assert that CP (coop) identifiers return empty string without calling get_recipient_from_auth.""" - filing_json = { - 'filing': { - 'header': {'name': 'annualReport'}, - 'business': {'identifier': 'CP1234567'}, - } - } - with app.app_context(): - with patch('business_emailer.email_processors.get_recipient_from_auth') as mock_auth: - result = get_recipients('COMPLETED', filing_json, token='token', filing_type='annualReport') - assert result == '' - mock_auth.assert_not_called() - - -def test_get_recipients_non_coop_identifier_calls_get_recipient_from_auth(app): - """Assert that non-coop identifiers call get_recipient_from_auth for the business email.""" +def test_get_recipients_calls_get_recipient_from_auth(app): + """Assert that get_recipients calls get_recipient_from_auth for the business email.""" filing_json = { 'filing': { 'header': {'name': 'annualReport'}, diff --git a/queue_services/business-emailer/tests/unit/email_processors/test_filing_notification.py b/queue_services/business-emailer/tests/unit/email_processors/test_filing_notification.py index 4315a4b0b1..eb27ee3f5e 100644 --- a/queue_services/business-emailer/tests/unit/email_processors/test_filing_notification.py +++ b/queue_services/business-emailer/tests/unit/email_processors/test_filing_notification.py @@ -20,8 +20,9 @@ from business_emailer.email_processors import filing_notification from tests.unit import (CONTACT_POINT, LEGAL_NAME, PARTY_EMAIL_1, PARTY_EMAIL_2, - prep_bootstrap_filing, prep_change_of_registration_filing, prep_incorp_filing, - prep_maintenance_filing, prep_registration_filing) + prep_bootstrap_filing, prep_change_of_registration_filing, prep_correction_filing, + prep_incorp_filing, prep_maintenance_filing, + prep_registration_filing, prep_special_resolution_filing) from tests.unit.helpers import make_future_effective, make_non_future_effective @@ -47,6 +48,12 @@ def mock_user_email(mocker): return mocker.patch.object(filing_notification, 'get_user_email_from_auth', return_value='user@email.com') +@pytest.fixture +def mock_auth_recipient(mocker): + """Patch get_recipient_from_auth to return the business contact email.""" + return mocker.patch.object(filing_notification, 'get_recipient_from_auth', return_value='auth@email.com') + + def process_filing(filing, filing_type, option, token='token'): """Run the filing_notification processor for the given filing.""" return filing_notification.process( @@ -75,8 +82,9 @@ def mock_filing_docs(m, config, identifier, filing, doc_contents, receipt=b'rece status_code=200, ) if receipt is not None: - m.post( - f'{config.get("PAY_API_URL")}/{filing.payment_token}/receipts', + m.get( + f'{config.get("LEGAL_API_URL")}/businesses/{identifier}' + f'/filings/{filing.id}/documents/receipt', content=receipt, status_code=201, ) @@ -238,18 +246,20 @@ def test_bootstrap_cp_filing_attachments(session, config): @pytest.mark.parametrize('filing_type', [ 'amalgamationApplication', 'continuationIn', + 'dissolution', 'incorporationApplication', 'registration' ]) def test_paid_non_future_effective_returns_none(app, session, filing_type): """Assert that a PAID non-future-effective filing returns None (no email is sent).""" - if filing_type != 'registration': - filing = prep_bootstrap_filing(session, filing_type, 'BC1234567', 'BC', 'PAID', LEGAL_NAME) - make_non_future_effective(filing) - else: + if filing_type == 'registration': filing = prep_registration_filing( - session, 'FM1234567', '1', 'PAID', Business.LegalTypes.SOLE_PROP.value, 'test business') - + session, 'FM1234567', 'PAID', Business.LegalTypes.SOLE_PROP.value, 'test business') + elif filing_type == 'dissolution': + filing = prep_maintenance_filing(session, 'BC1234567', '1', 'PAID', 'dissolution', 'voluntary') + else: + filing = prep_bootstrap_filing(session, filing_type, 'BC1234567', 'BC', 'PAID', LEGAL_NAME) + make_non_future_effective(filing) result = process_filing(filing, filing_type, 'PAID') assert result is None @@ -281,6 +291,9 @@ def test_bootstrap_body_details(app, mock_pdfs, session, legal_type, status, fil assert '**Business Name:** Not Available' in body assert '**Incorporation Number:** Not Available' in body assert 'What happens next' in body + if filing_type == 'amalgamationApplication': + # what-happens-next uses the short name for amalgamations + assert 'Once the amalgamation is effective on' in body else: assert '**Business Name:** 1234567 B.C. Ltd.' in body assert '**Incorporation Number:** BC1234567' in body @@ -323,157 +336,538 @@ def test_business_number_rendering(app, session, mock_pdfs, filing_type, legal_t # --------------------------------------------------------------------------- -# tests for non firm maintenance filings (changeOfAddress, changeOfDirectors, alteration, annualReport) +# tests for maintenance filings (changeOfAddress, changeOfDirectors, alteration, annualReport, dissolution, etc.) # --------------------------------------------------------------------------- -@pytest.mark.parametrize(['status', 'filing_type', 'submitter_role'], [ - ('PAID', 'changeOfAddress', None), - ('PAID', 'alteration', None), - ('COMPLETED', 'annualReport', None), - ('COMPLETED', 'changeOfAddress', None), - ('COMPLETED', 'changeOfDirectors', None), - ('COMPLETED', 'alteration', None), - ('COMPLETED', 'alteration', 'staff') +@pytest.mark.parametrize(['status', 'filing_type', 'filing_sub_type', 'submitter_role', 'legal_type', 'identifier'], [ + ('PAID', 'changeOfAddress', None, None, None, 'BC1234567'), + ('PAID', 'alteration', None, None, None, 'BC1234567'), + ('COMPLETED', 'annualReport', None, None, None, 'BC1234567'), + ('COMPLETED', 'changeOfAddress', None, None, None, 'BC1234567'), + ('COMPLETED', 'changeOfDirectors', None, None, None, 'BC1234567'), + ('COMPLETED', 'alteration', None, None, None, 'BC1234567'), + ('COMPLETED', 'alteration', None, 'staff', None, 'BC1234567'), + ('COMPLETED', 'dissolution', 'voluntary', None, Business.LegalTypes.COMP.value, 'BC1234567'), + ('COMPLETED', 'dissolution', 'voluntary', 'staff', None, 'BC1234567'), + ('COMPLETED', 'dissolution', 'voluntary', None, None, 'CP1234567'), + ('COMPLETED', 'dissolution', 'voluntary', None, Business.LegalTypes.SOLE_PROP.value, 'FM1234567'), + ('COMPLETED', 'dissolution', 'voluntary', None, Business.LegalTypes.PARTNERSHIP.value, 'FM1234567'), + ('COMPLETED', 'restoration', 'fullRestoration', None, None, 'BC1234567'), + ('COMPLETED', 'restoration', 'limitedRestoration', None, None, 'BC1234567'), + ('COMPLETED', 'restoration', 'limitedRestorationExtension', None, None, 'BC1234567'), + ('COMPLETED', 'restoration', 'limitedRestorationToFull', None, None, 'BC1234567'), + ('COMPLETED', 'specialResolution', None, None, None, 'CP1234567'), + ('COMPLETED', 'specialResolution', None, 'staff', None, 'CP1234567'), + ('COMPLETED', 'consentContinuationOut', None, None, None, 'BC1234567'), + ('COMPLETED', 'consentContinuationOut', None, 'staff', None, 'BC1234567'), + ('COMPLETED', 'continuationOut', None, None, None, 'BC1234567'), + ('COMPLETED', 'continuationOut', None, 'staff', None, 'BC1234567'), + ('COMPLETED', 'changeOfLiquidators', 'intentToLiquidate', None, None, 'BC1234567'), + ('COMPLETED', 'changeOfLiquidators', 'appointLiquidator', None, None, 'BC1234567'), + ('COMPLETED', 'changeOfLiquidators', 'ceaseLiquidator', None, None, 'BC1234567'), + ('COMPLETED', 'changeOfLiquidators', 'changeAddressLiquidator', None, None, 'BC1234567'), + ('COMPLETED', 'changeOfLiquidators', 'liquidationReport', None, None, 'BC1234567'), + ('COMPLETED', 'changeOfReceivers', 'appointReceiver', None, None, 'BC1234567'), + ('COMPLETED', 'changeOfReceivers', 'ceaseReceiver', None, None, 'BC1234567'), + ('COMPLETED', 'changeOfReceivers', 'changeAddressReceiver', None, None, 'BC1234567'), ]) -def test_maintenance_notification(app, session, status, filing_type, submitter_role, - mock_pdfs, mock_recipients, mock_user_email): +def test_maintenance_notification(app, session, mock_pdfs, mock_recipients, mock_user_email, mock_auth_recipient, + status, filing_type, filing_sub_type, submitter_role, legal_type, identifier): """Assert that maintenance filings produce an email with the correct recipients and attachments.""" # setup filing + business for email - filing = prep_maintenance_filing(session, 'BC1234567', '1', status, filing_type, submitter_role=submitter_role) + parties = firm_parties() if identifier.startswith('FM') else None + filing = prep_maintenance_filing(session, identifier, '1', status, filing_type, filing_sub_type, submitter_role, + legal_type=legal_type, parties=parties) if status == 'PAID': make_future_effective(filing) token = 'token' # test processor email = process_filing(filing, filing_type, status) - if filing_type == 'alteration': + if filing_type in ['alteration', 'changeOfLiquidators', 'changeOfReceivers', 'consentContinuationOut', 'continuationOut', 'dissolution']: if submitter_role: assert f'{submitter_role}@email.com' in email['recipients'] else: assert 'user@email.com' in email['recipients'] + expected_legal_type = legal_type or ( + Business.LegalTypes.COOP.value if identifier.startswith('CP') else Business.LegalTypes.BCOMP.value) + expected_legal_name = 'JANE A DOE' if identifier.startswith('FM') else LEGAL_NAME assert 'test@test.com' in email['recipients'] assert email['content']['body'] assert email['content']['attachments'] == [] assert mock_pdfs.call_args[0][0] == token - assert mock_pdfs.call_args[0][1]['identifier'] == 'BC1234567' - assert mock_pdfs.call_args[0][1]['legalType'] == Business.LegalTypes.BCOMP.value - assert mock_pdfs.call_args[0][1]['legalName'] == 'test business' + assert mock_pdfs.call_args[0][1]['identifier'] == identifier + assert mock_pdfs.call_args[0][1]['legalType'] == expected_legal_type + assert mock_pdfs.call_args[0][1]['legalName'] == expected_legal_name assert mock_pdfs.call_args[0][2] == filing assert mock_recipients.call_args[0][0] == status assert mock_recipients.call_args[0][1] == filing.filing_json assert mock_recipients.call_args[0][2] == token - -def test_filing_attachments_change_of_address_paid(session, config, mock_recipients): - """changeOfAddress PAID: filing PDF + receipt.""" - identifier = 'BC1234567' - filing = prep_maintenance_filing(session, identifier, '1', 'PAID', 'changeOfAddress') - make_future_effective(filing) - with requests_mock.Mocker() as m: - mock_filing_docs(m, config, identifier, filing, - {'changeOfAddress': b'pdf_content_1'}, receipt=b'pdf_content_2') - output = process_filing(filing, 'changeOfAddress', 'PAID') - - attachments = output['content']['attachments'] - assert len(attachments) == 2 - assert_attachment(attachments[0], 'Address Change.pdf', 'pdf_content_1') - assert_attachment(attachments[1], 'Receipt.pdf', 'pdf_content_2') + if filing_type == 'dissolution': + # dissolution also notifies the business contact email from auth + assert 'auth@email.com' in email['recipients'] + # party emails are pulled by get_recipients with the dissolution filing type + assert mock_recipients.call_args[0][3] == 'dissolution' + assert email['content']['subject'] == f'{LEGAL_NAME} - Successful Dissolution' + else: + assert mock_recipients.call_args[0][3] is None + assert not mock_auth_recipient.called -def test_filing_attachments_annual_report_completed(session, config, mock_recipients): - """annualReport COMPLETED: filing PDF (prefixed with the AR year) + receipt.""" +@pytest.mark.parametrize('filing_type, filing_sub_type, legal_type, status, has_name_change, has_rule_change, expected_attachments', [ + ('alteration', None, None, 'PAID', False, False, [ + {'fileName': 'Alteration.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '2'}, + ]), + ('alteration', None, None, 'PAID', True, False, [ + {'fileName': 'Alteration.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '2'}, + ]), + ('alteration', None, None, 'COMPLETED', False, False, [ + {'fileName': 'Alteration.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Notice of Articles.pdf', 'content': 'pdf_content_noa', 'order': '2'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '3'}, + ]), + ('alteration', None, None, 'COMPLETED', True, False, [ + {'fileName': 'Alteration.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Notice of Articles.pdf', 'content': 'pdf_content_noa', 'order': '2'}, + {'fileName': 'Certificate of Name Change.pdf', 'content': 'pdf_content_con', 'order': '3'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '4'}, + ]), + ('annualReport', None, None, 'COMPLETED', False, False, [ + {'fileName': '2018 Annual Report.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '2'}, + ]), + ('changeOfAddress', None, None, 'PAID', False, False, [ + {'fileName': 'Address Change.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '2'}, + ]), + ('changeOfAddress', None, None, 'COMPLETED', False, False, [ + {'fileName': 'Address Change.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Notice of Articles.pdf', 'content': 'pdf_content_noa', 'order': '2'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '3'}, + ]), + ('changeOfDirectors', None, None, 'COMPLETED', False, False, [ + {'fileName': 'Director Change.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Notice of Articles.pdf', 'content': 'pdf_content_noa', 'order': '2'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '3'}, + ]), + ('restoration', 'fullRestoration', None, 'COMPLETED', False, False, [ + {'fileName': 'Full Restoration Application.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Notice of Articles.pdf', 'content': 'pdf_content_noa', 'order': '2'}, + {'fileName': 'Certificate of Restoration.pdf', 'content': 'pdf_content_cor', 'order': '3'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '4'}, + ]), + ('restoration', 'limitedRestoration', None, 'COMPLETED', False, False, [ + {'fileName': 'Limited Restoration Application.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Notice of Articles.pdf', 'content': 'pdf_content_noa', 'order': '2'}, + {'fileName': 'Certificate of Restoration.pdf', 'content': 'pdf_content_cor', 'order': '3'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '4'}, + ]), + ('restoration', 'limitedRestorationExtension', None, 'COMPLETED', False, False, [ + {'fileName': 'Limited Restoration Extension Application.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Notice of Articles.pdf', 'content': 'pdf_content_noa', 'order': '2'}, + {'fileName': 'Certificate of Restoration.pdf', 'content': 'pdf_content_cor', 'order': '3'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '4'}, + ]), + ('restoration', 'limitedRestorationToFull', None, 'COMPLETED', False, False, [ + {'fileName': 'Conversion to Full Restoration Application.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Notice of Articles.pdf', 'content': 'pdf_content_noa', 'order': '2'}, + {'fileName': 'Certificate of Restoration.pdf', 'content': 'pdf_content_cor', 'order': '3'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '4'}, + ]), + ('specialResolution', None, None, 'COMPLETED', False, False, [ + {'fileName': 'Special Resolution.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Special Resolution Application.pdf', 'content': 'pdf_content_sra', 'order': '2'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '3'}, + ]), + ('specialResolution', None, None, 'COMPLETED', True, False, [ + {'fileName': 'Special Resolution.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Special Resolution Application.pdf', 'content': 'pdf_content_sra', 'order': '2'}, + {'fileName': 'Certificate of Name Change.pdf', 'content': 'pdf_content_con', 'order': '3'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '4'}, + ]), + ('specialResolution', None, None, 'COMPLETED', False, True, [ + {'fileName': 'Special Resolution.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Special Resolution Application.pdf', 'content': 'pdf_content_sra', 'order': '2'}, + {'fileName': 'Certified Rules.pdf', 'content': 'pdf_content_cr', 'order': '3'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '4'}, + ]), + ('specialResolution', None, None, 'COMPLETED', True, True, [ + {'fileName': 'Special Resolution.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Special Resolution Application.pdf', 'content': 'pdf_content_sra', 'order': '2'}, + {'fileName': 'Certificate of Name Change.pdf', 'content': 'pdf_content_con', 'order': '3'}, + {'fileName': 'Certified Rules.pdf', 'content': 'pdf_content_cr', 'order': '4'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '5'}, + ]), + ('dissolution', 'voluntary', Business.LegalTypes.COMP.value, 'COMPLETED', False, False, [ + {'fileName': 'Voluntary Dissolution Application.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Certificate of Dissolution.pdf', 'content': 'pdf_content_cod', 'order': '2'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '3'}, + ]), + ('dissolution', 'voluntary', Business.LegalTypes.COOP.value, 'COMPLETED', False, False, [ + {'fileName': 'Voluntary Dissolution Application.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Certificate of Dissolution.pdf', 'content': 'pdf_content_cod', 'order': '2'}, + {'fileName': 'Certified Affidavit.pdf', 'content': 'pdf_content_affidavit', 'order': '3'}, + {'fileName': 'Certified Special Resolution.pdf', 'content': 'pdf_content_sr', 'order': '4'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '5'}, + ]), + ('dissolution', 'voluntary', Business.LegalTypes.SOLE_PROP.value, 'COMPLETED', False, False, [ + {'fileName': 'Statement of Dissolution.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '2'}, + ]), + ('dissolution', 'administrative', Business.LegalTypes.COMP.value, 'COMPLETED', False, False, [ + {'fileName': 'Dissolution Application.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '2'}, + ]), + ('consentContinuationOut', None, None, 'COMPLETED', False, False, [ + {'fileName': 'Continue Out Application.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Letter of Consent.pdf', 'content': 'pdf_content_loc', 'order': '2'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '3'}, + ]), + ('continuationOut', None, None, 'COMPLETED', False, False, [ + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '1'}, + ]), + ('changeOfLiquidators', 'intentToLiquidate', None, 'COMPLETED', False, False, [ + {'fileName': 'Statement of Intent to Liquidate.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '2'}, + ]), + ('changeOfLiquidators', 'appointLiquidator', None, 'COMPLETED', False, False, [ + {'fileName': 'Notice to Appoint Liquidators.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '2'}, + ]), + ('changeOfLiquidators', 'ceaseLiquidator', None, 'COMPLETED', False, False, [ + {'fileName': 'Notice to Cease Liquidators.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '2'}, + ]), + ('changeOfLiquidators', 'changeAddressLiquidator', None, 'COMPLETED', False, False, [ + {'fileName': 'Liquidators (or Records) Change of Address.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '2'}, + ]), + ('changeOfLiquidators', 'liquidationReport', None, 'COMPLETED', False, False, [ + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '1'}, + ]), + ('changeOfReceivers', 'appointReceiver', None, 'COMPLETED', False, False, [ + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '1'}, + ]), + ('changeOfReceivers', 'ceaseReceiver', None, 'COMPLETED', False, False, [ + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '1'}, + ]), + ('changeOfReceivers', 'changeAddressReceiver', None, 'COMPLETED', False, False, [ + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '1'}, + ]), +], ids=[ + 'alteration - PAID no name change', + 'alteration - PAID name change included', + 'alteration - COMPLETED no name change', + 'alteration - COMPLETED name change included', + 'annualReport', + 'changeOfAddress - PAID', + 'changeOfAddress - COMPLETED', + 'changeOfDirectors', + 'restoration - fullRestoration', + 'restoration - limitedRestoration', + 'restoration - limitedRestorationExtension', + 'restoration - limitedRestorationToFull', + 'specialResolution - No name or rule changes', + 'specialResolution - Name change included', + 'specialResolution - Rule change included', + 'specialResolution - Name and Rule change included', + 'dissolution - voluntary corp', + 'dissolution - voluntary coop', + 'dissolution - voluntary firm', + 'dissolution - administrative suppresses certificate', + 'consentContinuationOut - application + letter of consent + receipt', + 'continuationOut - receipt only', + 'changeOfLiquidators - intentToLiquidate', + 'changeOfLiquidators - appointLiquidator', + 'changeOfLiquidators - ceaseLiquidator', + 'changeOfLiquidators - changeAddressLiquidator', + 'changeOfLiquidators - liquidationReport (receipt only)', + 'changeOfReceivers - appointReceiver (receipt only)', + 'changeOfReceivers - ceaseReceiver (receipt only)', + 'changeOfReceivers - changeAddressReceiver (receipt only)' +]) +def test_maintenance_filing_attachments(session, config, mock_recipients, mock_user_email, mock_auth_recipient, + filing_type, filing_sub_type, legal_type, status, has_name_change, has_rule_change, expected_attachments): + """Assert maintenance filings add the correct attachments.""" + # Setup identifier = 'BC1234567' - filing = prep_maintenance_filing(session, identifier, '1', 'COMPLETED', 'annualReport') - with requests_mock.Mocker() as m: - mock_filing_docs(m, config, identifier, filing, - {'annualReport': b'pdf_content_1'}, receipt=b'pdf_content_2') - output = process_filing(filing, 'annualReport', 'COMPLETED') - - attachments = output['content']['attachments'] - # annualReport PAID: filing application PDF + receipt - assert len(attachments) == 2 - # _add_filing_document_pdf prefixes the AR filing PDF with the annualReportDate year (2018-04-08 -> 2018) - assert_attachment(attachments[0], '2018 Annual Report.pdf', 'pdf_content_1') - assert_attachment(attachments[1], 'Receipt.pdf', 'pdf_content_2') - + if legal_type == Business.LegalTypes.COOP.value: + identifier = 'CP1234567' + elif legal_type in [Business.LegalTypes.SOLE_PROP.value, Business.LegalTypes.PARTNERSHIP.value]: + identifier = 'FM1234567' + filing = prep_maintenance_filing(session, identifier, '1', status, filing_type, filing_sub_type, legal_type=legal_type) + if filing_type == 'specialResolution': + # Only COOP + identifier = 'CP1234567' + filing = prep_special_resolution_filing(session, has_name_change=has_name_change, has_rule_change=has_rule_change) + elif filing_type == 'alteration' and has_name_change: + # force the name-change branch via meta_data + filing._meta_data = {'alteration': {'toLegalName': 'new name'}} + filing.save() + if status == 'PAID': + make_future_effective(filing) -def test_filing_attachments_alteration_completed_name_change(session, config, mock_recipients, mock_user_email): - """alteration COMPLETED (BC, with name change): NOA + Certificate of Name Change.""" - identifier = 'BC1234567' - filing = prep_maintenance_filing(session, identifier, '1', 'COMPLETED', 'alteration') - # force the name-change branch via meta_data - filing._meta_data = {'alteration': {'toLegalName': 'new name'}} - filing.save() + # Test with requests_mock.Mocker() as m: mock_filing_docs(m, config, identifier, filing, { - 'alteration': b'pdf_content_1', - 'noticeOfArticles': b'pdf_content_2', - 'certificateOfNameChange': b'pdf_content_3', - }, receipt=b'pdf_content_4') - output = process_filing(filing, 'alteration', 'COMPLETED') + # 'specialResolution' first so it is overridden by the filing pdf when it is the filing type + 'specialResolution': b'pdf_content_sr', + filing_type: b'pdf_content_filing', + 'specialResolutionApplication': b'pdf_content_sra', + 'noticeOfArticles': b'pdf_content_noa', + 'certificateOfNameChange': b'pdf_content_con', + 'certifiedRules': b'pdf_content_cr', + 'certificateOfRestoration': b'pdf_content_cor', + 'certificateOfDissolution': b'pdf_content_cod', + 'affidavit': b'pdf_content_affidavit', + 'letterOfConsent': b'pdf_content_loc', + }, receipt=b'pdf_content_receipt') + output = process_filing(filing, filing_type, status) attachments = output['content']['attachments'] - assert len(attachments) == 4 - assert_attachment(attachments[0], 'Alteration.pdf', 'pdf_content_1') - assert_attachment(attachments[1], 'Notice of Articles.pdf', 'pdf_content_2') - assert_attachment(attachments[2], 'Certificate of Name Change.pdf', 'pdf_content_3') - assert_attachment(attachments[3], 'Receipt.pdf', 'pdf_content_4') + assert len(attachments) == len(expected_attachments) + for attachment, expected in zip(attachments, expected_attachments): + assert_attachment(attachment, expected['fileName'], expected['content'], expected['order']) -@pytest.mark.parametrize(['filing_type', 'status', 'expected_header', 'expected_subject'], [ +@pytest.mark.parametrize(['filing_type', 'filing_sub_type', 'status', 'expected_header', 'expected_subject', 'expected_body_snippets'], [ ( 'alteration', + None, 'PAID', 'Your alteration has been filed', 'test business - Alteration Filed', + ['Effective Date and Time:', 'Once the alteration is effective on'], ), ( 'changeOfAddress', + None, 'PAID', 'Your address change has been filed', 'test business - Address Change Filed', + ['Effective Date and Time:', 'Once the address change is effective on'], + ), + ( + 'dissolution', + 'voluntary', + 'PAID', + 'Your voluntary dissolution application has been filed', + 'test business - Voluntary Dissolution Application Filed', + ['Effective Date and Time:', 'Once the dissolution is effective on', 'Certificate of Dissolution'], ), ( 'alteration', + None, 'COMPLETED', 'You have successfully completed your alteration with the BC Business Registry', 'test business - Successful Alteration', + [] ), ( 'annualReport', + None, 'COMPLETED', 'You have successfully completed your 2018 annual report with the BC Business Registry', 'test business - Successful Annual Report', + [] ), ( 'changeOfAddress', + None, 'COMPLETED', 'You have successfully completed your address change with the BC Business Registry', 'test business - Successful Address Change', + [] ), ( 'changeOfDirectors', + None, 'COMPLETED', 'You have successfully completed your director change with the BC Business Registry', 'test business - Successful Director Change', + [] + ), + ( + 'restoration', + 'fullRestoration', + 'COMPLETED', + 'You have successfully restored your business with the BC Business Registry', + 'test business - Successful Restoration', + [] + ), + ( + 'restoration', + 'limitedRestoration', + 'COMPLETED', + 'You have successfully restored your business with the BC Business Registry', + 'test business - Successful Restoration', + [] + ), + ( + 'restoration', + 'limitedRestorationExtension', + 'COMPLETED', + 'You have successfully extended your period of restoration with the BC Business Registry', + 'test business - Successful Extension of Limited Restoration', + [] + ), + ( + 'restoration', + 'limitedRestorationToFull', + 'COMPLETED', + 'You have successfully restored your business with the BC Business Registry', + 'test business - Successful Conversion to Full Restoration', + [] + ), + ( + 'dissolution', + 'voluntary', + 'COMPLETED', + 'You have successfully completed your dissolution with the BC Business Registry', + 'test business - Successful Dissolution', + [] + ), + ( + 'consentContinuationOut', + None, + 'COMPLETED', + 'Your request for consent to continue out of B.C. has been granted for 6 months', + 'test business - Consent to Continue Out Granted', + [ + '**Effective Until:** October 30, 2025', + '**New Jurisdiction:** Alberta, Canada', + 'granted a 6 month consent to continue', + 'This consent expires on October 30, 2025', + 'Once you have completed your continuation into the jurisdiction of Alberta, Canada', + ], + ), + ( + 'continuationOut', + None, + 'COMPLETED', + 'You have successfully continued out of B.C.', + 'test business - Successful Continuation Out', + [ + '**New Jurisdiction:** Alberta, Canada', + '**Continue Out Effective Date:** April 29, 2025', + 'made historical in British Columbia as of April 29, 2025', + 'under the name NEW TEST BUSINESS', + ], + ), + ( + 'changeOfLiquidators', + 'intentToLiquidate', + 'COMPLETED', + 'You have successfully filed your statement of intent to liquidate with the BC Business Registry', + 'test business - Successful Intent to Liquidate', + [] + ), + ( + 'changeOfLiquidators', + 'appointLiquidator', + 'COMPLETED', + 'You have successfully filed your notice of liquidator appointment with the BC Business Registry', + 'test business - Successful Liquidator Appointment', + [] + ), + ( + 'changeOfLiquidators', + 'ceaseLiquidator', + 'COMPLETED', + 'You have successfully filed your notice of liquidator cessation with the BC Business Registry', + 'test business - Successful Liquidator Cessation', + [] + ), + ( + 'changeOfLiquidators', + 'changeAddressLiquidator', + 'COMPLETED', + 'You have successfully filed your liquidator (or records) address change with the BC Business Registry', + 'test business - Successful Liquidator (or Records) Address Change', + [] + ), + ( + 'changeOfLiquidators', + 'liquidationReport', + 'COMPLETED', + 'You have successfully filed your liquidation report with the BC Business Registry', + 'test business - Successful Liquidation Report', + [] + ), + ( + 'changeOfReceivers', + 'appointReceiver', + 'COMPLETED', + 'Your receiver/receiver manager information has been successfully updated', + 'test business - Confirmation of Receiver Change', + [] + ), + ( + 'changeOfReceivers', + 'ceaseReceiver', + 'COMPLETED', + 'Your receiver/receiver manager information has been successfully updated', + 'test business - Confirmation of Receiver Change', + [] + ), + ( + 'changeOfReceivers', + 'changeAddressReceiver', + 'COMPLETED', + 'Your receiver/receiver manager information has been successfully updated', + 'test business - Confirmation of Receiver Change', + [] ), ]) -def test_maintenance_filing_fe_renders_body_and_subject(app, session, filing_type, status, expected_header, - expected_subject, mock_pdfs, mock_recipients, mock_user_email): - """Assert alteration and address change future effective emails render the expected body and subject.""" - filing = prep_maintenance_filing(session, 'BC1234567', '1', status, filing_type) +def test_maintenance_filing_renders_body_and_subject(app, session, + mock_pdfs, mock_recipients, mock_user_email, mock_auth_recipient, + filing_type, filing_sub_type, status, + expected_header, expected_subject, expected_body_snippets): + """Assert maintenance filing emails render the expected body and subject.""" + filing = prep_maintenance_filing(session, 'BC1234567', '1', status, filing_type, filing_sub_type) if status == 'PAID': make_future_effective(filing) filing.save() email = process_filing(filing, filing_type, status) assert email is not None + assert email['content']['subject'] == expected_subject body = email['content']['body'] assert expected_header in body - assert not ".html]]" in body - assert not ".md]]" in body - assert email['content']['subject'] == expected_subject + for snippet in expected_body_snippets: + assert snippet in body + assert ".html]]" not in body + assert ".md]]" not in body + + +@pytest.mark.parametrize(['status', 'tax_id', 'shows_bn'], [ + ('COMPLETED', None, False), + ('COMPLETED', '123456789BC0001', True), + ('PAID', None, False), +]) +def test_dissolution_business_number_line(app, session, mock_pdfs, mock_recipients, mock_user_email, + mock_auth_recipient, status, tax_id, shows_bn): + """Assert dissolution emails only show the business number line when a bn exists (#32963).""" + filing = prep_maintenance_filing(session, 'BC1234567', '1', status, 'dissolution', 'voluntary') + if status == 'PAID': + make_future_effective(filing) + business = Business.find_by_identifier('BC1234567') + business.tax_id = tax_id + business.save() + filing.save() + + email = process_filing(filing, 'dissolution', status) + + assert email is not None + body = email['content']['body'] + if shows_bn: + assert '**Business Number:** 123456789 BC0001' in body + else: + assert '**Business Number:**' not in body # --------------------------------------------------------------------------- @@ -492,7 +886,7 @@ def test_firm_filing_via_filing_notification(app, session, status, filing_type, """Assert that FIRM registration and changeOfRegistration produce correct emails via filing_notification.""" legal_name = 'test business' if filing_type == 'registration': - filing = prep_registration_filing(session, 'FM1234567', '1', status, legal_type, legal_name, firm_parties()) + filing = prep_registration_filing(session, 'FM1234567', status, legal_type, legal_name, firm_parties()) else: filing = prep_change_of_registration_filing( session, 'FM1234567', '1', legal_type, legal_name, submitter_role, firm_parties()) @@ -534,7 +928,7 @@ def test_filing_attachments_registration_completed(session, config, mock_recipie """registration COMPLETED: Statement of Registration filing PDF + receipt.""" identifier = 'FM1234567' filing = prep_registration_filing( - session, identifier, '1', 'COMPLETED', Business.LegalTypes.SOLE_PROP.value, 'test business', firm_parties()) + session, identifier, 'COMPLETED', Business.LegalTypes.SOLE_PROP.value, 'test business', firm_parties()) with requests_mock.Mocker() as m: mock_filing_docs(m, config, identifier, filing, {'registration': b'pdf_content_1'}, receipt=b'pdf_content_2') @@ -578,12 +972,163 @@ def test_firm_filing_subject(app, session, filing_type, expected_subject_suffix, legal_name = 'test business' legal_type = Business.LegalTypes.SOLE_PROP.value if filing_type == 'registration': - filing = prep_registration_filing(session, 'FM1234567', '1', 'COMPLETED', legal_type, legal_name, - firm_parties()) + filing = prep_registration_filing(session, 'FM1234567', 'COMPLETED', legal_type, legal_name, firm_parties()) else: filing = prep_change_of_registration_filing( session, 'FM1234567', '1', legal_type, legal_name, None, firm_parties()) email = process_filing(filing, filing_type, 'COMPLETED') assert email is not None - assert email['content']['subject'] == f'JANE A DOE - {expected_subject_suffix}' + assert email['content']['subject'] == f'test business - {expected_subject_suffix}' + + +# --------------------------------------------------------------------------- +# Corrections via filing_notification +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize(['orig_filing_type', 'legal_type', 'identifier'], [ + ('incorporationApplication', Business.LegalTypes.COMP.value, 'BC1234567'), + ('registration', Business.LegalTypes.PARTNERSHIP.value, 'FM1234567'), + ('registration', Business.LegalTypes.SOLE_PROP.value, 'FM1234567'), + ('specialResolution', Business.LegalTypes.COOP.value, 'CP1234567'), +]) +def test_correction_filing_via_filing_notification(app, session, mock_pdfs, + orig_filing_type, legal_type, identifier): + """Assert that Corrections send emails via filing_notification.""" + if orig_filing_type == 'specialResolution': + original_filing = prep_special_resolution_filing(session, identifier) + else: + original_filing = prep_bootstrap_filing(session, orig_filing_type, identifier, legal_type, 'COMPLETED') + business = Business.find_by_identifier(identifier) + corrected_filing = prep_correction_filing(session, business, original_filing.id, orig_filing_type, 'COMPLETED') + + email = process_filing(corrected_filing, 'correction', 'COMPLETED') + + assert email is not None + body = email['content']['body'] + assert body + assert email['content']['attachments'] == [] + assert mock_pdfs.call_args[0][1]['identifier'] == identifier + assert mock_pdfs.call_args[0][2] == corrected_filing + assert CONTACT_POINT in email['recipients'] + if orig_filing_type == 'registration': + assert PARTY_EMAIL_1 in email['recipients'] + + +@pytest.mark.parametrize('orig_filing_type, legal_type, identifier, has_name_change, has_rule_change, expected_attachments', [ + ('incorporationApplication', 'BC', 'BC1234567', False, False, [ + {'fileName': 'Register Correction Application.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Notice of Articles.pdf', 'content': 'pdf_content_noa', 'order': '2'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '3'}, + ]), + ('registration', 'SP', 'FM1234567', False, False, [ + {'fileName': 'Register Correction Application.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Corrected Registration Statement.pdf', 'content': 'pdf_content_crs', 'order': '2'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '3'}, + ]), + ('registration', 'GP', 'FM1234567', False, False, [ + {'fileName': 'Register Correction Application.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Corrected Registration Statement.pdf', 'content': 'pdf_content_crs', 'order': '2'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '3'}, + ]), + ('specialResolution', 'CP', 'CP1234567', False, False, [ + {'fileName': 'Register Correction Application.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '2'}, + ]), + ('specialResolution', 'CP', 'CP1234567', True, False, [ + {'fileName': 'Register Correction Application.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Certificate of Name Correction.pdf', 'content': 'pdf_content_con', 'order': '2'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '3'}, + ]), + ('specialResolution', 'CP', 'CP1234567', False, True, [ + {'fileName': 'Register Correction Application.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Certified Rules.pdf', 'content': 'pdf_content_cr', 'order': '2'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '3'}, + ]), + ('specialResolution', 'CP', 'CP1234567', True, True, [ + {'fileName': 'Register Correction Application.pdf', 'content': 'pdf_content_filing', 'order': '1'}, + {'fileName': 'Certificate of Name Correction.pdf', 'content': 'pdf_content_con', 'order': '2'}, + {'fileName': 'Certified Rules.pdf', 'content': 'pdf_content_cr', 'order': '3'}, + {'fileName': 'Receipt.pdf', 'content': 'pdf_content_receipt', 'order': '4'}, + ]), +]) +def test_correction_filing_attachments(session, config, mock_recipients, mock_user_email, + orig_filing_type, legal_type, identifier, has_name_change, has_rule_change, expected_attachments): + """Assert correction filings add the correct attachments.""" + # Setup + if orig_filing_type == 'specialResolution': + original_filing = prep_special_resolution_filing(session, identifier, has_name_change=has_name_change, has_rule_change=has_rule_change) + else: + original_filing = prep_bootstrap_filing(session, orig_filing_type, identifier, legal_type, 'COMPLETED') + business = Business.find_by_identifier(identifier) + corrected_filing = prep_correction_filing( + session, business, original_filing.id, orig_filing_type, 'COMPLETED', + has_name_change=has_name_change, has_rule_change=has_rule_change) + + # Test + with requests_mock.Mocker() as m: + mock_filing_docs(m, config, identifier, corrected_filing, { + 'correction': b'pdf_content_filing', + 'noticeOfArticles': b'pdf_content_noa', + 'correctedRegistrationStatement': b'pdf_content_crs', + 'certificateOfNameCorrection': b'pdf_content_con', + 'certifiedRules': b'pdf_content_cr', + }, receipt=b'pdf_content_receipt') + output = process_filing(corrected_filing, 'correction', 'COMPLETED') + + attachments = output['content']['attachments'] + assert len(attachments) == len(expected_attachments) + for attachment, expected in zip(attachments, expected_attachments): + assert_attachment(attachment, expected['fileName'], expected['content'], expected['order']) + + +@pytest.mark.parametrize('orig_filing_type, legal_type, identifier, expected_header, expected_subject', [ + ('incorporationApplication', 'BC', 'BC1234567', 'You have successfully completed your correction with the BC Business Registry', 'test business - Successful Correction'), + ('registration', 'SP', 'FM1234567', 'You have successfully completed your correction with the BC Business Registry', 'test business - Successful Correction'), + ('registration', 'GP', 'FM1234567', 'You have successfully completed your correction with the BC Business Registry', 'test business - Successful Correction'), + ('specialResolution', 'CP', 'CP1234567', 'You have successfully completed your correction with the BC Business Registry', 'test business - Successful Correction'), +]) +def test_correction_filing_header_and_subject(session, config, mock_pdfs, mock_recipients, mock_user_email, + orig_filing_type, legal_type, identifier, expected_header, expected_subject): + """Assert correction filings add the correct header and subject.""" + # Setup + if orig_filing_type == 'specialResolution': + original_filing = prep_special_resolution_filing(session, identifier) + else: + original_filing = prep_bootstrap_filing(session, orig_filing_type, identifier, legal_type, 'COMPLETED', LEGAL_NAME, parties=firm_parties()) + business = Business.find_by_identifier(identifier) + corrected_filing = prep_correction_filing(session, business, original_filing.id, orig_filing_type, 'COMPLETED') + + # Test + email = process_filing(corrected_filing, 'correction', 'COMPLETED') + + assert email is not None + body = email['content']['body'] + assert expected_header in body + assert not ".html]]" in body + assert not ".md]]" in body + assert email['content']['subject'] == expected_subject + +# --------------------------------------------------------------------------- +# Dissolution-specific behaviour (skipped sub types) +# --------------------------------------------------------------------------- + +def test_dissolution_delay_returns_none(app, session): + """Assert that a delay of dissolution filing does not send the dissolution email.""" + filing = prep_maintenance_filing(session, 'BC1234567', '1', 'COMPLETED', 'dissolution', 'delay') + + result = process_filing(filing, 'dissolution', 'COMPLETED') + + assert result is None + + +def test_dissolution_involuntary_returns_none(app, session): + """Assert that an involuntary dissolution filing does not send the dissolution email. + + Involuntary dissolution is notified via the stage 1 furnishing email instead. + """ + filing = prep_maintenance_filing(session, 'BC1234567', '1', 'COMPLETED', 'dissolution', 'involuntary') + + result = process_filing(filing, 'dissolution', 'COMPLETED') + + assert result is None diff --git a/queue_services/business-emailer/tests/unit/email_processors/test_intent_to_liquidate_notification.py b/queue_services/business-emailer/tests/unit/email_processors/test_intent_to_liquidate_notification.py deleted file mode 100644 index 5c717c13d5..0000000000 --- a/queue_services/business-emailer/tests/unit/email_processors/test_intent_to_liquidate_notification.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright © 2025 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""The Unit Tests for the Intent to Liquidate email processor.""" -import base64 -from unittest.mock import patch - -import pytest -import requests_mock -from business_model.models import Business - -from business_emailer.email_processors import intent_to_liquidate_notification -from tests.unit import prep_intent_to_liquidate_filing - - -@pytest.mark.skip('This email processor is invalid and needs to be updated') -@pytest.mark.parametrize('status,legal_type,submitter_role', [ - ('COMPLETED', Business.LegalTypes.COMP.value, 'staff'), - ('COMPLETED', Business.LegalTypes.BCOMP.value, 'staff'), - ('COMPLETED', Business.LegalTypes.BC_CCC.value, 'staff'), - ('COMPLETED', Business.LegalTypes.BC_ULC_COMPANY.value, 'staff'), - ('COMPLETED', Business.LegalTypes.CCC_CONTINUE_IN.value, 'staff'), - ('COMPLETED', Business.LegalTypes.BCOMP_CONTINUE_IN.value, 'staff'), - ('COMPLETED', Business.LegalTypes.CONTINUE_IN.value, 'staff'), - ('COMPLETED', Business.LegalTypes.ULC_CONTINUE_IN.value, 'staff') -]) -def test_intent_to_liquidate_notification(app, session, status, legal_type, submitter_role): - """Assert that the intent_to_liquidate email processor for corps works as expected.""" - # setup filing + business for email - legal_name = 'test business' - filing = prep_intent_to_liquidate_filing(session, 'BC1234567', '1', legal_type, legal_name, submitter_role) - token = 'token' - # test processor - with patch.object(intent_to_liquidate_notification, '_get_pdfs', return_value=[]) as mock_get_pdfs: - with patch.object(intent_to_liquidate_notification, 'get_recipient_from_auth', - return_value='recipient@email.com'): - email = intent_to_liquidate_notification.process( - {'filingId': filing.id, 'type': 'intentToLiquidate', 'option': status}, token) - assert email['content']['subject'] == \ - legal_name + ' - Statement of Intent to Liquidate' - - if submitter_role: - assert f'{submitter_role}@email.com' in email['recipients'] - assert 'recipient@email.com' in email['recipients'] - assert email['content']['body'] - assert email['content']['attachments'] == [] - assert mock_get_pdfs.call_args[0][0] == token - assert mock_get_pdfs.call_args[0][1]['identifier'] == 'BC1234567' - assert mock_get_pdfs.call_args[0][1]['legalName'] == legal_name - assert mock_get_pdfs.call_args[0][1]['legalType'] == legal_type - assert mock_get_pdfs.call_args[0][2] == filing - - -@pytest.mark.skip('This email processor is invalid and needs to be updated') -def test_intent_to_liquidate_attachments(session, config): - """Assert _get_pdfs assembles the Statement of Intent to Liquidate and receipt.""" - identifier = 'BC1234567' - filing = prep_intent_to_liquidate_filing( - session, identifier, '1', Business.LegalTypes.COMP.value, 'test business', 'staff') - token = 'token' - with patch.object(intent_to_liquidate_notification, 'get_recipient_from_auth', - return_value='recipient@email.com'): - with requests_mock.Mocker() as m: - m.get( - f'{config.get("LEGAL_API_URL")}/businesses/{identifier}' - f'/filings/{filing.id}/documents/intentToLiquidate', - content=b'pdf_content_1', - status_code=200, - ) - m.post( - f'{config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - content=b'pdf_content_2', - status_code=201, - ) - output = intent_to_liquidate_notification.process( - {'filingId': filing.id, 'type': 'intentToLiquidate', 'option': 'COMPLETED'}, token) - - attachments = output['content']['attachments'] - assert len(attachments) == 2 - assert attachments[0]['fileName'] == 'Statement of Intent to Liquidate.pdf' - assert base64.b64decode(attachments[0]['fileBytes']).decode('utf-8') == 'pdf_content_1' - assert attachments[1]['fileName'] == 'Receipt.pdf' - assert base64.b64decode(attachments[1]['fileBytes']).decode('utf-8') == 'pdf_content_2' diff --git a/queue_services/business-emailer/tests/unit/email_processors/test_involuntary_dissolution_stage_1_notification.py b/queue_services/business-emailer/tests/unit/email_processors/test_involuntary_dissolution_stage_1_notification.py index f2569d1d87..63274cc6f6 100644 --- a/queue_services/business-emailer/tests/unit/email_processors/test_involuntary_dissolution_stage_1_notification.py +++ b/queue_services/business-emailer/tests/unit/email_processors/test_involuntary_dissolution_stage_1_notification.py @@ -17,26 +17,42 @@ import pytest import requests -from simple_cloudevent import SimpleCloudEvent from business_model.models import Furnishing from business_emailer.email_processors import involuntary_dissolution_stage_1_notification from tests.unit import create_business, create_furnishing # noqa: I003 +def create_test_business(identifier, legal_type): + """Return a test business, bypassing the identifier setter for expro 'A' identifiers it rejects.""" + business = create_business('BC1234567', legal_type, 'Test Business') + if identifier != 'BC1234567': + business._identifier = identifier + business.save() + return business + + @pytest.mark.parametrize( - 'test_name, legal_type, furnishing_name', [ - ('TEST_BC_NO_AR', 'BC', Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_AR), - # next one is failing, needs more investigation why ??!? - ('TEST_XPRO_NO_AR', 'A', Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_AR_XPRO), - ('TEST_BC_NO_TR', 'BC', Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_TR), + 'test_name, business_identifier, legal_type, furnishing_name, action, reason_title, attachment_name', [ + ('TEST_BC_NO_AR', 'BC1234567', 'BC', Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_AR, + 'dissolved', 'overdue annual reports', + 'Notice of Commencement of Dissolution'), + ('TEST_XPRO_NO_AR', 'A1234567', 'A', Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_AR_XPRO, + 'cancelled', 'overdue annual reports', + 'Notice of Commencement of Cancellation'), + ('TEST_BC_NO_TR', 'BC1234567', 'BC', Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_TR, + 'dissolved', 'failure to file a post restoration transition application', + 'Notice of Commencement of Dissolution'), + ('TEST_XPRO_NO_TR', 'A1234567', 'A', Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_TR_XPRO, + 'cancelled', 'failure to file a post restoration transition application', + 'Notice of Commencement of Cancellation'), ] ) -def test_involuntary_dissolution_stage_1_notification(app, session, test_name, legal_type, furnishing_name): +def test_involuntary_dissolution_stage_1_notification(app, session, test_name, business_identifier, legal_type, + furnishing_name, action, reason_title, attachment_name): """Assert that the test_involuntary_dissolution_stage_1_notification can be processed.""" token = 'token' - business_identifier = 'BC1234567' - business = create_business(business_identifier, legal_type, 'Test Business') + business = create_test_business(business_identifier, legal_type) furnishing = create_furnishing(session, business=business, furnishing_name=furnishing_name) message_payload = { 'furnishing': { @@ -55,15 +71,129 @@ def test_involuntary_dissolution_stage_1_notification(app, session, test_name, l with patch.object(requests, 'get', return_value=mock_response): email = involuntary_dissolution_stage_1_notification.process(message_payload, token) - assert email['content']['subject'] == f'Attention {business_identifier} - Test Business' + assert email['content']['subject'] == \ + f'Test Business - URGENT - Your business is in the process of being {action}' assert email['recipients'] == 'test@test.com' body = email['content']['body'] assert body - if test_name == 'TEST_XPRO_NO_AR': - assert 'Notice of Commencement of Cancellation' in body + assert f'Your business is in the process of being {action} for {reason_title}' in body + assert '**Business Name:** Test Business' in body + if legal_type == 'A': + assert f'**Registration Number:** {business_identifier}' in body else: - assert 'Notice of Commencement of Dissolution' in body + assert f'**Incorporation Number:** {business_identifier}' in body + assert '## Attention' in body + assert f'Your business is in the process of being {action} because' in body + assert '## Next Steps' in body + assert f'you can request a delay of {"cancellation" if action == "cancelled" else "dissolution"}' in body + assert '## Attachments' in body + assert attachment_name in body + # no MRAS registrations mocked, so no extraprovincial sentence + assert 'Our records indicate' not in body + assert '.md]]' not in body assert email['content']['attachments'] assert mock_get_pdfs.call_args[0][0] == token assert mock_get_pdfs.call_args[0][1] == business assert mock_get_pdfs.call_args[0][2] == furnishing + + +@pytest.mark.parametrize( + 'test_name, jurisdictions, expected_display', [ + ('AB', [('AB', 'Alberta')], 'Alberta'), + ('MB', [('MB', 'Manitoba')], 'Manitoba'), + ('SK', [('SK', 'Saskatchewan')], 'Saskatchewan'), + ('AB_MB', [('AB', 'Alberta'), ('MB', 'Manitoba')], 'Alberta and Manitoba'), + ('AB_SK', [('AB', 'Alberta'), ('SK', 'Saskatchewan')], 'Alberta and Saskatchewan'), + ('MB_SK', [('MB', 'Manitoba'), ('SK', 'Saskatchewan')], 'Manitoba and Saskatchewan'), + ('AB_MB_SK', [('AB', 'Alberta'), ('MB', 'Manitoba'), ('SK', 'Saskatchewan')], + 'Alberta, Manitoba, and Saskatchewan'), + ] +) +def test_involuntary_dissolution_stage_1_notification_extra_provincials(app, session, test_name, + jurisdictions, expected_display): + """Assert that NWPTA registrations render the foreign jurisdiction sentence in the Attention block.""" + token = 'token' + business_identifier = 'BC1234567' + business = create_business(business_identifier, 'BC', 'Test Business') + furnishing = create_furnishing(session, business=business, + furnishing_name=Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_AR) + message_payload = { + 'furnishing': { + 'type': 'INVOLUNTARY_DISSOLUTION', + 'furnishingId': furnishing.id, + 'furnishingName': furnishing.furnishing_name + } + } + + # non-NWPTA jurisdictions must be filtered out of the display + mras_jurisdictions = [{'id': j_id, 'name': name} for j_id, name in jurisdictions] + mras_jurisdictions.append({'id': 'BC', 'name': 'British Columbia'}) + + with patch.object(involuntary_dissolution_stage_1_notification, '_get_pdfs', return_value=[]): + with patch.object(involuntary_dissolution_stage_1_notification, 'get_jurisdictions', + return_value={'jurisdictions': mras_jurisdictions}): + email = involuntary_dissolution_stage_1_notification.process(message_payload, token) + + body = email['content']['body'] + assert f'Our records indicate your business is registered in {expected_display} ' \ + 'as an extraprovincial company.' in body + assert f'its registration as an extraprovincial company in {expected_display} ' \ + 'will automatically be cancelled as well' in body + assert 'British Columbia' not in body + + +@pytest.mark.parametrize( + 'test_name, tax_id, expected_display', [ + ('BN15', '123456789BC0001', '123456789 BC0001'), + ('BN9_OMITTED', '123456789', None), + ('NO_BN_OMITTED', None, None), + ] +) +def test_involuntary_dissolution_stage_1_notification_business_number(app, session, test_name, + tax_id, expected_display): + """Assert that the business number is only displayed when a bn15 is saved, formatted for ux.""" + token = 'token' + business = create_business('BC1234567', 'BC', 'Test Business') + business.tax_id = tax_id + business.save() + furnishing = create_furnishing(session, business=business, + furnishing_name=Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_AR) + message_payload = { + 'furnishing': { + 'type': 'INVOLUNTARY_DISSOLUTION', + 'furnishingId': furnishing.id, + 'furnishingName': furnishing.furnishing_name + } + } + + with patch.object(involuntary_dissolution_stage_1_notification, '_get_pdfs', return_value=[]): + with patch.object(involuntary_dissolution_stage_1_notification, 'get_jurisdictions', return_value=None): + email = involuntary_dissolution_stage_1_notification.process(message_payload, token) + + body = email['content']['body'] + if expected_display: + assert f'**Business Number:** {expected_display}' in body + else: + assert '**Business Number:**' not in body + + +def test_involuntary_dissolution_stage_1_notification_xpro_skips_mras(app, session): + """Assert that XPRO furnishings do not look up MRAS jurisdictions.""" + token = 'token' + business = create_test_business('A1234567', 'A') + furnishing = create_furnishing(session, business=business, + furnishing_name=Furnishing.FurnishingName.DISSOLUTION_COMMENCEMENT_NO_AR_XPRO) + message_payload = { + 'furnishing': { + 'type': 'INVOLUNTARY_DISSOLUTION', + 'furnishingId': furnishing.id, + 'furnishingName': furnishing.furnishing_name + } + } + + with patch.object(involuntary_dissolution_stage_1_notification, '_get_pdfs', return_value=[]): + with patch.object(involuntary_dissolution_stage_1_notification, 'get_jurisdictions') as mock_get_jurisdictions: + email = involuntary_dissolution_stage_1_notification.process(message_payload, token) + + mock_get_jurisdictions.assert_not_called() + assert 'Our records indicate' not in email['content']['body'] diff --git a/queue_services/business-emailer/tests/unit/email_processors/test_restoration_notification.py b/queue_services/business-emailer/tests/unit/email_processors/test_restoration_notification.py deleted file mode 100644 index 04d1800d42..0000000000 --- a/queue_services/business-emailer/tests/unit/email_processors/test_restoration_notification.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright © 2020 Province of British Columbia -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""The Unit Tests for the Restoration email processor.""" - -import base64 -from unittest.mock import patch - -import pytest -import requests_mock - -from business_emailer.email_processors import restoration_notification -from tests.unit import prep_restoration_filing - - -LEGAL_NAME = 'test business' -BUS_ID = 'BC1234567' -TOKEN = 'token' -EXPECTED_EMAIL = 'joe@email.com' - - -def test_complete_full_restoration_notification_includes_notice_of_articles_and_incorporation_cert(session, config): - """Test completed full restoration notification.""" - # setup filing + business for email - status = 'COMPLETED' - filing = prep_restoration_filing(BUS_ID, '1', 'BC', LEGAL_NAME) - with requests_mock.Mocker() as m: - m.get(f'{config.get("LEGAL_API_URL")}/businesses/{BUS_ID}/filings/{filing.id}/documents/noticeOfArticles', - content=b'pdf_content_1', status_code=200) - m.get(f'{config.get("LEGAL_API_URL")}/businesses/{BUS_ID}/filings/{filing.id}' - '/documents/certificateOfRestoration', - content=b'pdf_content_2') - output = restoration_notification.process({ - 'filingId': filing.id, - 'type': 'restoration', - 'option': status - }, TOKEN) - assert 'content' in output - assert 'attachments' in output['content'] - assert len(output['content']['attachments']) == 2 - assert output['content']['attachments'][0]['fileName'] == 'Notice of Articles.pdf' - assert base64.b64decode(output['content']['attachments'][0]['fileBytes']).decode('utf-8') == 'pdf_content_1' - assert output['content']['attachments'][1]['fileName'] == 'Certificate of Restoration.pdf' - assert base64.b64decode(output['content']['attachments'][1]['fileBytes']).decode('utf-8') == 'pdf_content_2' - - -def test_paid_restoration_notification_includes_receipt_and_restoration_application_attachments(session, config): - """Test PAID full restoration notification.""" - # setup filing + business for email - status = 'PAID' - filing = prep_restoration_filing(BUS_ID, '1', 'BC', LEGAL_NAME) - with requests_mock.Mocker() as m: - m.post(f'{config.get("PAY_API_URL")}/{filing.payment_token}/receipts', - content=b'pdf_content_1', status_code=201) - m.get(f'{config.get("LEGAL_API_URL")}/businesses/{BUS_ID}/filings/{filing.id}/documents/restoration', - content=b'pdf_content_2', status_code=200) - output = restoration_notification.process({ - 'filingId': filing.id, - 'type': 'restoration', - 'option': status - }, TOKEN) - assert 'content' in output - assert 'attachments' in output['content'] - assert len(output['content']['attachments']) == 2 - assert output['content']['attachments'][0]['fileName'] == 'Restoration Application.pdf' - assert base64.b64decode(output['content']['attachments'][0]['fileBytes']).decode('utf-8') == 'pdf_content_2' - assert output['content']['attachments'][1]['fileName'] == 'Receipt.pdf' - assert base64.b64decode(output['content']['attachments'][1]['fileBytes']).decode('utf-8') == 'pdf_content_1' - - -def test_completed_full_restoration_notification(session, config): - """Test completed full restoration notification.""" - # setup filing + business for email - status = 'COMPLETED' - filing = prep_restoration_filing(BUS_ID, '1', 'BC', LEGAL_NAME) - # test processor - with patch.object(restoration_notification, '_get_completed_pdfs', return_value=[]): - email_dict = restoration_notification.process({ - 'filingId': filing.id, - 'type': 'restoration', - 'option': status - }, TOKEN) - email = email_dict['content']['body'] - assert email_dict['content']['subject'] == 'test business - Restoration Documents from the Business Registry' - assert EXPECTED_EMAIL in email_dict['recipients'] - assert 'You have successfully restored your business with the BC Business Registry' in email - - -def test_completed_extended_restoration_notification(session, config): - """Test completed extended restoration notification includes specific wording.""" - # setup filing + business for email - status = 'COMPLETED' - filing = prep_restoration_filing(BUS_ID, '1', 'BC', LEGAL_NAME, 'limitedRestorationExtension') - with patch.object(restoration_notification, '_get_completed_pdfs', return_value=[]): - email_dict = restoration_notification.process({ - 'filingId': filing.id, - 'type': 'restoration', - 'option': status - }, TOKEN) - email = email_dict['content']['body'] - assert 'You have successfully extended the period of restoration with the BC Business' in email - - -@pytest.mark.parametrize( - 'restoration_type, attachment_name', - [ - ('fullRestoration', 'Full Restoration Application'), - ('limitedRestoration', 'Limited Restoration Application'), - ('limitedRestorationExtension', 'Limited Restoration Extension Application'), - ('limitedRestorationToFull', 'Conversion to Full Restoration Application'), - ] -) -def test_paid_full_restoration_notification(session, restoration_type, attachment_name): - """Test PAID restoration notification.""" - # setup filing + business for email - status = 'PAID' - filing = prep_restoration_filing('BC1234567', '1', 'BC', LEGAL_NAME, restoration_type) - # test processor - with patch.object(restoration_notification, '_get_paid_pdfs', return_value=[]): - email_dict = restoration_notification.process({ - 'filingId': filing.id, - 'type': 'restoration', - 'option': status - }, TOKEN) - email = email_dict['content']['body'] - assert EXPECTED_EMAIL in email_dict['recipients'] - assert email_dict['content']['subject'] == 'test business - Confirmation of Filing from the Business Registry' - assert EXPECTED_EMAIL in email_dict['recipients'] - assert 'You have successfully filed your restoration with the BC Business Registry' in email - assert email_dict['content']['attachments'] == [] - assert attachment_name in email diff --git a/queue_services/business-emailer/tests/unit/meta/test_filing.py b/queue_services/business-emailer/tests/unit/meta/test_filing.py index 7a5d31cb83..59cbf248b9 100644 --- a/queue_services/business-emailer/tests/unit/meta/test_filing.py +++ b/queue_services/business-emailer/tests/unit/meta/test_filing.py @@ -137,6 +137,27 @@ def test_alter_outputs_dissolution_noop_for_non_dissolution(): assert 'certificateOfDissolution' in result +def test_alter_outputs_dissolution_firm_administrative_no_cert_no_raise(): + # Firms (SP/GP) have no certificateOfDissolution in their outputs, so admin + # dissolution suppression must be a no-op, not a KeyError. Regression for + # bcgov/entity#33806 (admin dissolution 400'd the ledger for firms). + filing = _filing(filing_type='dissolution', filing_sub_type='administrative', + json_legal_type=Business.LegalTypes.SOLE_PROP) + outputs = set() + result = FilingMeta.alter_outputs_dissolution(filing, outputs) + assert result == set() + + +def test_alter_outputs_dissolution_coop_voluntary_missing_docs_no_raise(): + # certifiedRules/certifiedMemorandum may be absent from outputs; suppression + # must discard silently rather than raising KeyError. + filing = _filing(filing_type='dissolution', filing_sub_type='voluntary', + json_legal_type=Business.LegalTypes.COOP) + outputs = {'certificateOfDissolution'} + result = FilingMeta.alter_outputs_dissolution(filing, outputs) + assert result == {'certificateOfDissolution'} + + # --------------------------------------------------------------------------- # # alter_outputs_special_resolution # # --------------------------------------------------------------------------- # @@ -170,6 +191,17 @@ def test_alter_outputs_special_resolution_noop_for_other_types(): assert 'certifiedMemorandum' in result +def test_alter_outputs_special_resolution_missing_docs_no_raise(): + # certifiedMemorandum/certifiedRules may be absent from outputs; suppression + # must discard silently rather than raising KeyError (same class of bug as + # bcgov/entity#33806). + filing = _filing(filing_type='specialResolution', + meta_data={'legalFilings': [], 'alteration': {}}) + outputs = set() + result = FilingMeta.alter_outputs_special_resolution(filing, outputs) + assert result == set() + + # --------------------------------------------------------------------------- # # alter_outputs — orchestrator # # --------------------------------------------------------------------------- # diff --git a/queue_services/business-emailer/tests/unit/test_worker.py b/queue_services/business-emailer/tests/unit/test_worker.py index e41aea0fed..2dd3ee6886 100644 --- a/queue_services/business-emailer/tests/unit/test_worker.py +++ b/queue_services/business-emailer/tests/unit/test_worker.py @@ -24,12 +24,10 @@ from business_emailer.email_processors import ( ar_reminder_notification, - continuation_in_notification, - correction_notification, + continuation_authorization_notification, filing_notification, name_request, nr_notification, - special_resolution_notification, ) from business_emailer.exceptions import EmailException, QueueException from business_emailer.resources import business_emailer as worker @@ -37,13 +35,14 @@ from tests import MockResponse from tests.unit import ( CONTACT_POINT, + LEGAL_NAME, create_business, create_furnishing, prep_bootstrap_filing, - prep_cp_special_resolution_correction_filing, - prep_cp_special_resolution_filing, + prep_correction_filing, prep_incorp_filing, prep_maintenance_filing, + prep_special_resolution_filing ) from tests.unit.helpers import make_future_effective, make_non_future_effective @@ -126,19 +125,13 @@ def test_process_incorp_email_paid_non_future_no_email_sent(app, session, mocker assert not mock_send_email.call_args -@pytest.mark.parametrize(['status', 'filing_type'], [ - ('PAID', 'changeOfAddress'), - ('COMPLETED', 'alteration'), - ('COMPLETED', 'annualReport'), - ('COMPLETED', 'changeOfAddress'), - ('COMPLETED', 'changeOfDirectors') -]) -def test_maintenance_notification(app, session, status, filing_type): - """Assert that the legal name is changed.""" +def test_correction_notification(app, session): + """Assert that valid correction filing cases send an email.""" # setup filing + business for email - filing = prep_maintenance_filing(session, 'BC1234567', '1', status, filing_type) - if status == 'PAID': - make_future_effective(filing) + identifier = 'BC1234567' + original_filing = prep_bootstrap_filing(session, 'incorporationApplication', identifier, 'BC', 'COMPLETED', LEGAL_NAME) + business = Business.find_by_identifier(identifier) + corrected_filing = prep_correction_filing(session, business, original_filing.id, 'incorporationApplication', 'COMPLETED') token = 'token' # test worker with patch.object(AccountService, 'get_bearer_token', return_value=token): @@ -149,14 +142,63 @@ def test_maintenance_notification(app, session, status, filing_type): with patch.object(worker, 'send_email', return_value='success') as mock_send_email: worker.process_email( SimpleCloudEvent( - data={'email': {'filingId': filing.id, 'type': f'{filing_type}', 'option': status}} + data={'email': {'filingId': corrected_filing.id, 'type': 'correction', 'option': 'COMPLETED'}} ) ) assert mock_get_pdfs.call_args[0][0] == token + assert mock_get_pdfs.call_args[0][1]['identifier'] == identifier + business: Business = Business.find_by_identifier(identifier) + assert mock_get_pdfs.call_args[0][1]['legalType'] == business.legal_type + assert mock_get_pdfs.call_args[0][1]['legalName'] == 'test business' + + assert mock_get_pdfs.call_args[0][2] == corrected_filing + assert mock_get_recipients.call_args[0][0] == 'COMPLETED' + assert mock_get_recipients.call_args[0][1] == corrected_filing.filing_json + assert mock_get_recipients.call_args[0][2] == token - assert mock_get_pdfs.call_args[0][1]['identifier'] == 'BC1234567' - assert mock_get_pdfs.call_args[0][1]['legalType'] == Business.LegalTypes.BCOMP.value + assert mock_send_email.call_args[0][0]['content']['subject'] + assert 'test@test.com' in mock_send_email.call_args[0][0]['recipients'] + assert mock_send_email.call_args[0][0]['content']['body'] + assert mock_send_email.call_args[0][0]['content']['attachments'] == [] + assert mock_send_email.call_args[0][1] == token + + +@pytest.mark.parametrize(['status', 'filing_type', 'filing_sub_type', 'identifier', 'submitter_role'], [ + ('PAID', 'changeOfAddress', None, 'BC1234567', None), + ('COMPLETED', 'alteration', None, 'BC1234567', None), + ('COMPLETED', 'annualReport', None, 'BC1234567', None), + ('COMPLETED', 'changeOfAddress', None, 'BC1234567', None), + ('COMPLETED', 'dissolution', 'voluntary', 'BC1234567', None), + ('COMPLETED', 'dissolution', 'voluntary', 'CP1234567', None), + ('COMPLETED', 'specialResolution', None, 'CP1234567', None), + ('COMPLETED', 'specialResolution', None, 'CP1234567', 'staff'), +]) +def test_maintenance_notification(app, session, status, filing_type, filing_sub_type, identifier, submitter_role): + """Assert that valid maintenance filing cases send an email.""" + # setup filing + business for email + filing = prep_maintenance_filing(session, identifier, '1', status, filing_type, filing_sub_type, submitter_role) + if status == 'PAID': + make_future_effective(filing) + token = 'token' + # test worker + with patch.object(AccountService, 'get_bearer_token', return_value=token): + with patch.object(filing_notification, 'get_user_email_from_auth', return_value='user@email.com'): + with patch.object(filing_notification, 'get_recipient_from_auth', return_value='auth@email.com'): + with patch.object(filing_notification, 'get_pdfs', return_value=[]) as mock_get_pdfs: + with patch.object(filing_notification, 'get_recipients', return_value='test@test.com') \ + as mock_get_recipients: + with patch.object(worker, 'send_email', return_value='success') as mock_send_email: + worker.process_email( + SimpleCloudEvent( + data={'email': {'filingId': filing.id, 'type': f'{filing_type}', 'option': status}} + ) + ) + + assert mock_get_pdfs.call_args[0][0] == token + assert mock_get_pdfs.call_args[0][1]['identifier'] == identifier + business: Business = Business.find_by_identifier(identifier) + assert mock_get_pdfs.call_args[0][1]['legalType'] == business.legal_type assert mock_get_pdfs.call_args[0][1]['legalName'] == 'test business' assert mock_get_pdfs.call_args[0][2] == filing @@ -166,22 +208,28 @@ def test_maintenance_notification(app, session, status, filing_type): assert mock_send_email.call_args[0][0]['content']['subject'] assert 'test@test.com' in mock_send_email.call_args[0][0]['recipients'] + if submitter_role: + assert 'staff@email.com' in mock_send_email.call_args[0][0]['recipients'] assert mock_send_email.call_args[0][0]['content']['body'] assert mock_send_email.call_args[0][0]['content']['attachments'] == [] assert mock_send_email.call_args[0][1] == token -@pytest.mark.parametrize(['status', 'filing_type', 'identifier'], [ - ('PAID', 'annualReport', 'BC1234567'), - ('PAID', 'changeOfAddress', 'CP1234567'), - ('PAID', 'changeOfDirectors', 'CP1234567'), - ('COMPLETED', 'changeOfAddress', 'CP1234567'), - ('COMPLETED', 'changeOfDirectors', 'CP1234567') +@pytest.mark.parametrize(['status', 'filing_type', 'filing_sub_type', 'identifier'], [ + ('PAID', 'annualReport', None, 'BC1234567'), + ('PAID', 'changeOfAddress', None, 'CP1234567'), + ('PAID', 'changeOfDirectors', None, 'CP1234567'), + ('PAID', 'specialResolution', None, 'BC1234567'), + ('PAID', 'consentContinuationOut', None, 'BC1234567'), + ('COMPLETED', 'annualReport', None, 'CP1234567'), + ('COMPLETED', 'changeOfAddress', None, 'CP1234567'), + ('COMPLETED', 'changeOfDirectors', None, 'CP1234567'), + ('COMPLETED', 'dissolution', 'delay', 'BC1234567') ]) -def test_skips_notification(app, session, status, filing_type, identifier): - """Assert that the legal name is changed.""" +def test_skips_notification(app, session, status, filing_type, filing_sub_type, identifier): + """Assert that the emailer skips sending an email for invalid cases.""" # setup filing + business for email - filing = prep_maintenance_filing(session, identifier, '1', status, filing_type) + filing = prep_maintenance_filing(session, identifier, '1', status, filing_type, filing_sub_type) token = 'token' # test processor with patch.object(AccountService, 'get_bearer_token', return_value=token): @@ -219,83 +267,6 @@ def test_process_mras_email(app, session): assert mock_send_email.call_args[0][1] == token -@pytest.mark.parametrize(['option', 'submitter_role'], [ - ('PAID', 'staff'), - ('COMPLETED', None), -]) -def test_process_special_resolution_email(app, session, option, submitter_role): - """Assert that an special resolution email msg is processed correctly.""" - filing = prep_cp_special_resolution_filing('CP1234567', '1', 'CP', 'TEST', submitter_role=submitter_role) - token = '1' - get_pdf_function = 'get_paid_pdfs' if option == 'PAID' else 'get_completed_pdfs' - # test worker - with patch.object(AccountService, 'get_bearer_token', return_value=token): - with patch.object(special_resolution_notification, get_pdf_function, return_value=[]) as mock_get_pdfs: - with patch.object(special_resolution_notification, 'get_recipient_from_auth', - return_value='recipient@email.com'): - with patch.object(special_resolution_notification, 'get_user_email_from_auth', - return_value='user@email.com'): - with patch.object(worker, 'send_email', return_value='success') as mock_send_email: - worker.process_email( - SimpleCloudEvent( - data={'email': {'filingId': filing.id, 'type': 'specialResolution', 'option': option}} - ) - ) - - assert mock_get_pdfs.call_args[0][0] == token - assert mock_get_pdfs.call_args[0][1]['identifier'] == 'CP1234567' - assert mock_get_pdfs.call_args[0][2] == filing - - if option == 'PAID': - assert mock_send_email.call_args[0][0]['content']['subject'] == \ - 'TEST - Confirmation of Special Resolution from the Business Registry' - else: - assert mock_send_email.call_args[0][0]['content']['subject'] == \ - 'TEST - Special Resolution Documents from the Business Registry' - assert 'recipient@email.com' in mock_send_email.call_args[0][0]['recipients'] - if submitter_role: - assert f'{submitter_role}@email.com' in mock_send_email.call_args[0][0]['recipients'] - else: - assert 'user@email.com' in mock_send_email.call_args[0][0]['recipients'] - assert mock_send_email.call_args[0][0]['content']['body'] - assert mock_send_email.call_args[0][0]['content']['attachments'] == [] - assert mock_send_email.call_args[0][1] == token - - -@pytest.mark.parametrize('option', [ - ('PAID'), - ('COMPLETED'), -]) -def test_process_correction_cp_sr_email(app, session, option): - """Assert that a correction email msg is processed correctly.""" - identifier = 'CP1234567' - original_filing = prep_cp_special_resolution_filing(identifier, '1', 'CP', 'TEST', submitter_role=None) - token = '1' - business = Business.find_by_identifier(identifier) - filing = prep_cp_special_resolution_correction_filing(session, business, original_filing.id, - '1', option, 'specialResolution') - # test worker - with patch.object(AccountService, 'get_bearer_token', return_value=token): - with patch.object(correction_notification, '_get_pdfs', return_value=[]): - with patch.object(worker, 'send_email', return_value='success') as mock_send_email: - worker.process_email( - SimpleCloudEvent( - data={'email': {'filingId': filing.id, 'type': 'correction', 'option': option}} - ) - ) - - if option == 'PAID': - assert mock_send_email.call_args[0][0]['content']['subject'] == \ - 'TEST - Confirmation of correction' - else: - assert mock_send_email.call_args[0][0]['content']['subject'] == \ - 'TEST - Correction Documents from the Business Registry' - assert 'cp_sr@test.com' in mock_send_email.call_args[0][0]['recipients'] - assert mock_send_email.call_args[0][0]['content']['body'] - assert mock_send_email.call_args[0][0]['content']['attachments'] == [] - assert mock_send_email.call_args[0][1] == token - - def test_process_ar_reminder_email(app, session): """Assert that the ar reminder notification can be processed.""" # setup filing + business for email @@ -589,7 +560,8 @@ def test_involuntary_dissolution_stage_1_notification(app, db, session, mocker, if furnishing_name == 'INVALID_NAME': assert call_args is None else: - assert call_args[0][0]['content']['subject'] == f'Attention {business_identifier} - Test Business' + assert call_args[0][0]['content']['subject'] == \ + 'Test Business - URGENT - Your business is in the process of being dissolved' assert call_args[0][0]['recipients'] == 'test@test.com' assert call_args[0][0]['content']['body'] @@ -602,11 +574,11 @@ def test_involuntary_dissolution_stage_1_notification(app, db, session, mocker, @pytest.mark.parametrize(['option', 'expected_processor'], [ (Filing.Status.PAID.value, filing_notification), (Filing.Status.COMPLETED.value, filing_notification), - (ReviewStatus.APPROVED.name, continuation_in_notification), - (ReviewStatus.AWAITING_REVIEW.name, continuation_in_notification), - (ReviewStatus.CHANGE_REQUESTED.name, continuation_in_notification), - (ReviewStatus.REJECTED.name, continuation_in_notification), - (ReviewStatus.RESUBMITTED.name, continuation_in_notification), + (ReviewStatus.APPROVED.name, continuation_authorization_notification), + (ReviewStatus.AWAITING_REVIEW.name, continuation_authorization_notification), + (ReviewStatus.CHANGE_REQUESTED.name, continuation_authorization_notification), + (ReviewStatus.REJECTED.name, continuation_authorization_notification), + (ReviewStatus.RESUBMITTED.name, continuation_authorization_notification), ]) def test_continuation_in_notification(app, db, session, mocker, option, expected_processor): """Assert that the worker uses the correct continuation in processor for each option.""" diff --git a/queue_services/business-emailer/tests/unit/test_worker_dispatch.py b/queue_services/business-emailer/tests/unit/test_worker_dispatch.py index bafb30c3cf..a3d5e8339c 100644 --- a/queue_services/business-emailer/tests/unit/test_worker_dispatch.py +++ b/queue_services/business-emailer/tests/unit/test_worker_dispatch.py @@ -25,24 +25,14 @@ agm_extension_notification, agm_location_change_notification, amalgamation_out_notification, - appoint_receiver_notification, ar_reminder_notification, bn_notification, - cease_receiver_notification, consent_amalgamation_out_notification, - consent_continuation_out_notification, - continuation_out_notification, - correction_notification, - dissolution_notification, filing_notification, - intent_to_liquidate_notification, - intent_to_liquidate_notification as _intent_to_liquidate, # noqa: F401 (keep alias import-safe) mras_notification, name_request, notice_of_withdrawal_notification, nr_notification, - restoration_notification, - special_resolution_notification, ) from business_emailer.resources import business_emailer as worker from business_emailer.services import flags @@ -225,36 +215,6 @@ def test_agm_extension_completed_dispatches(app, session, mocker, mock_send_emai mock_send_email.assert_called_once_with(STUB_EMAIL, TOKEN) -def test_dissolution_dispatches(app, session, mocker, mock_send_email): - mock_process = mocker.patch.object(dissolution_notification, "process", return_value=STUB_EMAIL) - email = {"type": "dissolution", "option": COMPLETED} - - worker.process_email(_ce({"email": email})) - - mock_process.assert_called_once_with(email, TOKEN) - mock_send_email.assert_called_once_with(STUB_EMAIL, TOKEN) - - -def test_restoration_dispatches(app, session, mocker, mock_send_email): - mock_process = mocker.patch.object(restoration_notification, "process", return_value=STUB_EMAIL) - email = {"type": "restoration", "option": COMPLETED} - - worker.process_email(_ce({"email": email})) - - mock_process.assert_called_once_with(email, TOKEN) - mock_send_email.assert_called_once_with(STUB_EMAIL, TOKEN) - - -def test_correction_dispatches(app, session, mocker, mock_send_email): - mock_process = mocker.patch.object(correction_notification, "process", return_value=STUB_EMAIL) - email = {"type": "correction", "option": "PAID"} - - worker.process_email(_ce({"email": email})) - - mock_process.assert_called_once_with(email, TOKEN) - mock_send_email.assert_called_once_with(STUB_EMAIL, TOKEN) - - def test_consent_amalgamation_out_dispatches(app, session, mocker, mock_send_email): mock_process = mocker.patch.object(consent_amalgamation_out_notification, "process", return_value=STUB_EMAIL) email = {"type": "consentAmalgamationOut", "option": COMPLETED} @@ -275,56 +235,6 @@ def test_amalgamation_out_dispatches(app, session, mocker, mock_send_email): mock_send_email.assert_called_once_with(STUB_EMAIL, TOKEN) -def test_consent_continuation_out_completed_dispatches(app, session, mocker, mock_send_email): - mock_process = mocker.patch.object(consent_continuation_out_notification, "process", return_value=STUB_EMAIL) - email = {"type": "consentContinuationOut", "option": COMPLETED} - - worker.process_email(_ce({"email": email})) - - mock_process.assert_called_once_with(email, TOKEN) - mock_send_email.assert_called_once_with(STUB_EMAIL, TOKEN) - - -def test_consent_continuation_out_non_completed_skipped(app, session, mocker, mock_send_email): - mock_process = mocker.patch.object(consent_continuation_out_notification, "process", return_value=STUB_EMAIL) - email = {"type": "consentContinuationOut", "option": "PAID"} - - worker.process_email(_ce({"email": email})) - - mock_process.assert_not_called() - mock_send_email.assert_not_called() - - -def test_continuation_out_completed_dispatches(app, session, mocker, mock_send_email): - mock_process = mocker.patch.object(continuation_out_notification, "process", return_value=STUB_EMAIL) - email = {"type": "continuationOut", "option": COMPLETED} - - worker.process_email(_ce({"email": email})) - - mock_process.assert_called_once_with(email, TOKEN) - mock_send_email.assert_called_once_with(STUB_EMAIL, TOKEN) - - -def test_special_resolution_dispatches(app, session, mocker, mock_send_email): - mock_process = mocker.patch.object(special_resolution_notification, "process", return_value=STUB_EMAIL) - email = {"type": "specialResolution", "option": "PAID"} - - worker.process_email(_ce({"email": email})) - - mock_process.assert_called_once_with(email, TOKEN) - mock_send_email.assert_called_once_with(STUB_EMAIL, TOKEN) - - -def test_intent_to_liquidate_dispatches(app, session, mocker, mock_send_email): - mock_process = mocker.patch.object(intent_to_liquidate_notification, "process", return_value=STUB_EMAIL) - email = {"type": "intentToLiquidate", "option": COMPLETED} - - worker.process_email(_ce({"email": email})) - - mock_process.assert_called_once_with(email, TOKEN) - mock_send_email.assert_called_once_with(STUB_EMAIL, TOKEN) - - def test_notice_of_withdrawal_completed_dispatches(app, session, mocker, mock_send_email): mock_process = mocker.patch.object(notice_of_withdrawal_notification, "process", return_value=STUB_EMAIL) email = {"type": "noticeOfWithdrawal", "option": COMPLETED} @@ -335,40 +245,28 @@ def test_notice_of_withdrawal_completed_dispatches(app, session, mocker, mock_se mock_send_email.assert_called_once_with(STUB_EMAIL, TOKEN) -def test_appoint_receiver_completed_dispatches(app, session, mocker, mock_send_email): - mock_process = mocker.patch.object(appoint_receiver_notification, "process", return_value=STUB_EMAIL) - email = {"type": "appointReceiver", "option": COMPLETED} - - worker.process_email(_ce({"email": email})) - - mock_process.assert_called_once_with(email, TOKEN) - mock_send_email.assert_called_once_with(STUB_EMAIL, TOKEN) - - -def test_cease_receiver_completed_dispatches(app, session, mocker, mock_send_email): - mock_process = mocker.patch.object(cease_receiver_notification, "process", return_value=STUB_EMAIL) - email = {"type": "ceaseReceiver", "option": COMPLETED} - - worker.process_email(_ce({"email": email})) - - mock_process.assert_called_once_with(email, TOKEN) - mock_send_email.assert_called_once_with(STUB_EMAIL, TOKEN) - - # --------------------------------------------------------------------------- # # filing_notification branch # # --------------------------------------------------------------------------- # @pytest.mark.parametrize('filing_type', [ - "amalgamationApplication", "alteration", + "amalgamationApplication", "annualReport", "changeOfAddress", "changeOfDirectors", + "changeOfLiquidators", + "changeOfReceivers", "changeOfRegistration", + "consentContinuationOut", "continuationIn", + "continuationOut", + "correction", + "dissolution", "incorporationApplication", "registration", + "restoration", + "specialResolution", ]) def test_filing_notification_dispatches(app, session, mocker, mock_send_email, filing_type): """Assert that the filing_notification branch works as expected.""" diff --git a/queue_services/business-filer/devops/vaults.gcp.env b/queue_services/business-filer/devops/vaults.gcp.env index 82ba9a05ee..2361bc9d14 100644 --- a/queue_services/business-filer/devops/vaults.gcp.env +++ b/queue_services/business-filer/devops/vaults.gcp.env @@ -38,6 +38,7 @@ BUSINESS_EVENTS_TOPIC="op://gcp-queue/$APP_ENV/topics/BUSINESS_EVENTS_TOPIC" BUSINESS_MAILER_TOPIC="op://gcp-queue/$APP_ENV/topics/BUSINESS_EMAILER_TOPIC" BUSINESS_PAY_TOPIC="op://gcp-queue/$APP_ENV/topics/BUSINESS_PAY_TOPIC" DOC_CREATE_REC_TOPIC="op://gcp-queue/$APP_ENV/topics/DOC_CREATE_REC_TOPIC" +DOC_UPDATE_REC_TOPIC="op://gcp-queue/$APP_ENV/topics/DOC_UPDATE_REC_TOPIC" NAMEX_PAY_TOPIC="op://gcp-queue/$APP_ENV/topics/NAMEX_PAY_TOPIC" VPC_CONNECTOR="op://CD/$APP_ENV/base/VPC_CONNECTOR" diff --git a/queue_services/business-filer/poetry.lock b/queue_services/business-filer/poetry.lock index 8b2132cd1d..96b23d677d 100644 --- a/queue_services/business-filer/poetry.lock +++ b/queue_services/business-filer/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "aiofiles" @@ -294,7 +294,7 @@ files = [ [[package]] name = "business-model" -version = "3.3.32" +version = "3.4.6" description = "" optional = false python-versions = ">=3.13,<3.14" @@ -314,14 +314,14 @@ pg8000 = ">=1.31.2,<2.0.0" pycountry = ">=24.6.1,<25.0.0" pydantic = ">=2.10.6,<3.0.0" pytz = ">=2025.1,<2026.0" -registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.73"} +registry-schemas = {git = "https://github.com/bcgov/business-schemas.git", rev = "2.18.77"} sql-versioning = {git = "https://github.com/bcgov/lear.git", rev = "main", subdirectory = "python/common/sql-versioning-alt"} [package.source] type = "git" url = "https://github.com/bcgov/lear.git" reference = "main" -resolved_reference = "295ecb7a24620142f1a18af02fb191df4cafcc85" +resolved_reference = "095adf04125d51df16c614cafc08ac2aae83861e" subdirectory = "python/common/business-registry-model" [[package]] @@ -2791,7 +2791,7 @@ rpds-py = ">=0.7.0" [[package]] name = "registry_schemas" -version = "2.18.73" +version = "2.18.70" description = "A short description of the project" optional = false python-versions = ">=3.6" @@ -2809,8 +2809,8 @@ strict-rfc3339 = "*" [package.source] type = "git" url = "https://github.com/bcgov/business-schemas.git" -reference = "2.18.73" -resolved_reference = "e6999a46b6e38c52cc79f0e5a3c9fcbd0c473434" +reference = "2.18.77" +resolved_reference = "d63ceaca445890a93f62540d9b14496d1715f3d1" [[package]] name = "requests" diff --git a/queue_services/business-filer/pyproject.toml b/queue_services/business-filer/pyproject.toml index 626f7e1dc7..976b7e405b 100644 --- a/queue_services/business-filer/pyproject.toml +++ b/queue_services/business-filer/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "business-filer" -version = "3.0.22" +version = "3.0.26" description = "Business Registry Filer Service" authors = [ {name = "thor",email = "1042854+thorwolpert@users.noreply.github.com"} diff --git a/queue_services/business-filer/src/business_filer/config.py b/queue_services/business-filer/src/business_filer/config.py index 92c50bbae1..a4ba1974a6 100644 --- a/queue_services/business-filer/src/business_filer/config.py +++ b/queue_services/business-filer/src/business_filer/config.py @@ -92,6 +92,7 @@ class _Config: # pylint: disable=too-few-public-methods BUSINESS_MAILER_TOPIC = os.getenv("BUSINESS_MAILER_TOPIC", "business-mailer-dev") BUSINESS_PAY_TOPIC = os.getenv("BUSINESS_PAY_TOPIC", "business-pay-dev") DOC_CREATE_REC_TOPIC = os.getenv("DOC_CREATE_REC_TOPIC") + DOC_UPDATE_REC_TOPIC = os.getenv("DOC_UPDATE_REC_TOPIC") NAMEX_PAY_TOPIC = os.getenv("NAMEX_PAY_TOPIC", "namex-pay-dev") SUB_AUDIENCE = os.getenv("SUB_AUDIENCE", "") SUB_SERVICE_ACCOUNT = os.getenv("SUB_SERVICE_ACCOUNT", "") @@ -132,6 +133,7 @@ class TestConfig(_Config): # pylint: disable=too-few-public-methods # Faked out publishing DOC_CREATE_REC_TOPIC = os.getenv("TEST_DOC_CREATE_REC_TOPIC", "fake-doc-create-rec-topic") + DOC_UPDATE_REC_TOPIC = os.getenv("TEST_DOC_UPDATE_REC_TOPIC", "fake-doc-update-rec-topic") class ProdConfig(_Config): # pylint: disable=too-few-public-methods diff --git a/queue_services/business-filer/src/business_filer/filing_processors/alteration.py b/queue_services/business-filer/src/business_filer/filing_processors/alteration.py index d7ee95361a..44eb71d593 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/alteration.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/alteration.py @@ -91,7 +91,7 @@ def process( # update court order, if any is present with suppress(IndexError, KeyError, TypeError): court_order_json = dpath.get(filing, "/alteration/courtOrder") - filings.update_filing_court_order(filing_submission, court_order_json) + filings.create_court_order(filing_submission, court_order_json, filing_meta) # update name translations, if any with suppress(IndexError, KeyError, TypeError): diff --git a/queue_services/business-filer/src/business_filer/filing_processors/amalgamation_application.py b/queue_services/business-filer/src/business_filer/filing_processors/amalgamation_application.py index da91db0294..2b4e0cb4f0 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/amalgamation_application.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/amalgamation_application.py @@ -129,7 +129,7 @@ def process(business: Business, # pylint: disable=too-many-branches, too-many-l aliases.update_aliases(business, name_translations) if court_order := amalgamation_filing.get("courtOrder"): - filings.update_filing_court_order(filing_rec, court_order) + filings.create_court_order(filing_rec, court_order, filing_meta, business) # Update the filing json with identifier and founding date. amalgamation_json = copy.deepcopy(filing_rec.filing_json) diff --git a/queue_services/business-filer/src/business_filer/filing_processors/amalgamation_out.py b/queue_services/business-filer/src/business_filer/filing_processors/amalgamation_out.py index 520ea7e3c3..9b3d9f8122 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/amalgamation_out.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/amalgamation_out.py @@ -27,7 +27,7 @@ def process(business: Business, amalgamation_out_filing: Filing, filing: dict, f # update amalgamation out, if any is present with suppress(IndexError, KeyError, TypeError): court_order_json = dpath.get(filing, "/amalgamationOut/courtOrder") - filings.update_filing_court_order(amalgamation_out_filing, court_order_json) + filings.create_court_order(amalgamation_out_filing, court_order_json, filing_meta) amalgamation_out_json = filing["amalgamationOut"] diff --git a/queue_services/business-filer/src/business_filer/filing_processors/change_of_liquidators.py b/queue_services/business-filer/src/business_filer/filing_processors/change_of_liquidators.py index de9e63c69f..f60f6a034a 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/change_of_liquidators.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/change_of_liquidators.py @@ -40,7 +40,7 @@ from business_filer.common.services import InvoluntaryDissolutionService from business_filer.filing_meta import FilingMeta -from business_filer.filing_processors.filing_components.filings import update_filing_court_order +from business_filer.filing_processors.filing_components.filings import create_court_order from business_filer.filing_processors.filing_components.offices import update_or_create_offices from business_filer.filing_processors.filing_components.relationships import ( cease_relationships, @@ -96,4 +96,4 @@ def process(business: Business, filing_rec: Filing, filing_meta: FilingMeta): # update court order, if any is present if court_order := filing_json["filing"]["changeOfLiquidators"].get("courtOrder"): - update_filing_court_order(filing_rec, court_order) + create_court_order(filing_rec, court_order, filing_meta) diff --git a/queue_services/business-filer/src/business_filer/filing_processors/change_of_receivers.py b/queue_services/business-filer/src/business_filer/filing_processors/change_of_receivers.py index c8a5824559..2766e43f4a 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/change_of_receivers.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/change_of_receivers.py @@ -37,7 +37,7 @@ from business_model.models import Business, Filing, PartyRole from business_filer.filing_meta import FilingMeta -from business_filer.filing_processors.filing_components.filings import update_filing_court_order +from business_filer.filing_processors.filing_components.filings import create_court_order from business_filer.filing_processors.filing_components.relationships import ( cease_relationships, create_relationships, @@ -67,4 +67,4 @@ def process(business: Business, filing_rec: Filing, filing_meta: FilingMeta): # update court order, if any is present if court_order := filing_json["filing"]["changeOfReceivers"].get("courtOrder"): - update_filing_court_order(filing_rec, court_order) + create_court_order(filing_rec, court_order, filing_meta) diff --git a/queue_services/business-filer/src/business_filer/filing_processors/change_of_registration.py b/queue_services/business-filer/src/business_filer/filing_processors/change_of_registration.py index a5d020dc23..15fe1eea1d 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/change_of_registration.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/change_of_registration.py @@ -91,7 +91,7 @@ def process(business: Business, change_filing_rec: Filing, change_filing: dict, # update court order, if any is present with suppress(IndexError, KeyError, TypeError): court_order_json = dpath.get(change_filing, "/changeOfRegistration/courtOrder") - filings.update_filing_court_order(change_filing_rec, court_order_json) + filings.create_court_order(change_filing_rec, court_order_json, filing_meta) def update_parties(business: Business, parties: dict, change_filing_rec: Filing): diff --git a/queue_services/business-filer/src/business_filer/filing_processors/consent_amalgamation_out.py b/queue_services/business-filer/src/business_filer/filing_processors/consent_amalgamation_out.py index 8197be4d5a..c8b231c447 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/consent_amalgamation_out.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/consent_amalgamation_out.py @@ -27,7 +27,7 @@ def process(business: Business, cco_filing: Filing, filing: dict, filing_meta: F # update consent amalgamation out, if any is present with suppress(IndexError, KeyError, TypeError): consent_amalgamation_out_json = dpath.get(filing, "/consentAmalgamationOut/courtOrder") - filings.update_filing_court_order(cco_filing, consent_amalgamation_out_json) + filings.create_court_order(cco_filing, consent_amalgamation_out_json, filing_meta) foreign_jurisdiction = filing["consentAmalgamationOut"]["foreignJurisdiction"] consent_amalgamation_out = ConsentContinuationOut() diff --git a/queue_services/business-filer/src/business_filer/filing_processors/consent_continuation_out.py b/queue_services/business-filer/src/business_filer/filing_processors/consent_continuation_out.py index a10c868b63..ec7a1ec248 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/consent_continuation_out.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/consent_continuation_out.py @@ -48,7 +48,7 @@ def process(business: Business, cco_filing: Filing, filing: dict, filing_meta: F # update consent continuation out, if any is present with suppress(IndexError, KeyError, TypeError): consent_continuation_out_json = dpath.get(filing, "/consentContinuationOut/courtOrder") - filings.update_filing_court_order(cco_filing, consent_continuation_out_json) + filings.create_court_order(cco_filing, consent_continuation_out_json, filing_meta) cco_json = filing["consentContinuationOut"] foreign_jurisdiction = cco_json["foreignJurisdiction"] diff --git a/queue_services/business-filer/src/business_filer/filing_processors/continuation_in.py b/queue_services/business-filer/src/business_filer/filing_processors/continuation_in.py index 4d7a2b7d63..6ee033548c 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/continuation_in.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/continuation_in.py @@ -148,7 +148,7 @@ def process(business: Business, # noqa: PLR0912 aliases.update_aliases(business, name_translations) if court_order := continuation_in.get("courtOrder"): - filings.update_filing_court_order(filing_rec, court_order) + filings.create_court_order(filing_rec, court_order, filing_meta, business) if not filing_rec.colin_event_ids: # Update the filing json with identifier and founding date. diff --git a/queue_services/business-filer/src/business_filer/filing_processors/continuation_out.py b/queue_services/business-filer/src/business_filer/filing_processors/continuation_out.py index 03b56ad515..1f185cdcae 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/continuation_out.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/continuation_out.py @@ -82,7 +82,7 @@ def process(business: Business, continuation_out_filing: Filing, filing: dict, f # update continuation out, if any is present with suppress(IndexError, KeyError, TypeError): court_order_json = dpath.get(filing, "/continuationOut/courtOrder") - filings.update_filing_court_order(continuation_out_filing, court_order_json) + filings.create_court_order(continuation_out_filing, court_order_json, filing_meta) continuation_out_json = filing["continuationOut"] diff --git a/queue_services/business-filer/src/business_filer/filing_processors/court_order.py b/queue_services/business-filer/src/business_filer/filing_processors/court_order.py index 7bb572292c..673edee828 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/court_order.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/court_order.py @@ -32,27 +32,45 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. """File processing rules and actions for the court order filing.""" -from contextlib import suppress -from datetime import datetime - from business_model.models import Business, Document, DocumentType, Filing from business_filer.filing_meta import FilingMeta +from business_filer.filing_processors.filing_components import filings def process(business: Business, court_order_filing: Filing, filing: dict, filing_meta: FilingMeta): """Render the court order filing into the business model objects.""" - court_order_filing.court_order_file_number = filing["courtOrder"].get("fileNumber") - court_order_filing.court_order_effect_of_order = filing["courtOrder"].get("effectOfOrder") - court_order_filing.order_details = filing["courtOrder"].get("orderDetails") - - with suppress(IndexError, KeyError, TypeError, ValueError): - court_order_filing.court_order_date = datetime.fromisoformat(filing["courtOrder"].get("orderDate")) + court_order_data = filing["courtOrder"] + filings.create_court_order(court_order_filing, court_order_data, filing_meta) - if file_key := filing["courtOrder"].get("fileKey"): + file_list = [] + if file_key := court_order_data.get("fileKey"): + # This if can be removed once court order filings start using files document = Document() document.type = DocumentType.COURT_ORDER.value document.file_key = file_key document.business_id = business.id document.filing_id = court_order_filing.id business.documents.append(document) + file_list.append({ + "fileKey": document.file_key, + "fileName": f'Court Order {court_order_data.get("fileNumber")}.pdf' + }) + elif files := court_order_data.get("files", []): + for file in files: + document = Document() + document.type = DocumentType.COURT_ORDER.value + document.file_key = file.get("fileKey") + document.file_name = file.get("fileName") + file_list.append({ + "fileKey": document.file_key, + "fileName": document.file_name + }) + document.filing_id = court_order_filing.id + business.documents.append(document) + + if file_list: + filing_meta.court_order = { + **filing_meta.court_order, + "files": file_list + } diff --git a/queue_services/business-filer/src/business_filer/filing_processors/dissolution.py b/queue_services/business-filer/src/business_filer/filing_processors/dissolution.py index 6e4f968ffa..ce3ba5e350 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/dissolution.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/dissolution.py @@ -87,12 +87,12 @@ def process(business: Business, filing: dict, filing_rec: Filing, filing_meta: F if parties := dissolution_filing.get("parties"): update_parties(business, parties, filing_rec, False) - filing_rec.order_details = dissolution_filing.get("details") + filing_rec.details = dissolution_filing.get("details") # update court order, if any is present with suppress(IndexError, KeyError, TypeError): court_order_json = dpath.get(dissolution_filing, "/courtOrder") - filings.update_filing_court_order(filing_rec, court_order_json) + filings.create_court_order(filing_rec, court_order_json, filing_meta) if business.legal_type == Business.LegalTypes.COOP: _update_cooperative(dissolution_filing, business, filing_rec, dissolution_type) diff --git a/queue_services/business-filer/src/business_filer/filing_processors/filing_components/correction.py b/queue_services/business-filer/src/business_filer/filing_processors/filing_components/correction.py index 468be1c2b5..944e7c9d35 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/filing_components/correction.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/filing_components/correction.py @@ -58,6 +58,7 @@ create_relationships, update_relationship_addresses, update_relationship_entity_info, + update_relationships_appointment_date, ) CEASE_ROLE_MAPPING = { @@ -132,6 +133,8 @@ def correct_business_data(business: Business, # noqa: PLR0915 PartyRole.RoleTypes.RECEIVER.value ], filing_meta.application_date) + + update_relationships_appointment_date(relationships, business, [PartyRole.RoleTypes.DIRECTOR.value]) update_relationship_addresses(relationships, business) update_relationship_entity_info(relationships, business) _set_lear_only(correction_filing, correction_filing_rec, relationships, business) @@ -139,7 +142,7 @@ def correct_business_data(business: Business, # noqa: PLR0915 # update court order, if any is present with suppress(IndexError, KeyError, TypeError): court_order_json = dpath.get(correction_filing, "/correction/courtOrder") - filings.update_filing_court_order(correction_filing_rec, court_order_json) + filings.create_court_order(correction_filing_rec, court_order_json, filing_meta) # update business start date, if any is present with suppress(IndexError, KeyError, TypeError): diff --git a/queue_services/business-filer/src/business_filer/filing_processors/filing_components/filings.py b/queue_services/business-filer/src/business_filer/filing_processors/filing_components/filings.py index ca8707e82b..20cda39f86 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/filing_components/filings.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/filing_components/filings.py @@ -34,22 +34,47 @@ """Manages the names of a Business.""" from contextlib import suppress -from business_model.models import Filing +from business_model.models import Business, CourtOrder, Filing from flask_babel import _ as babel from business_filer.common.datetime import datetime +from business_filer.common.filing import FilingTypes +from business_filer.filing_meta import FilingMeta -def update_filing_court_order(filing_submission: Filing, court_order: dict) -> dict | None: - """Update the court_order info for a Filing.""" - if not Filing: - return {"error": babel("Filing required before alternate names can be set.")} +def create_court_order(filing_submission: Filing, + court_order: dict, + filing_meta: FilingMeta, + new_business: Business = None) -> dict | None: + """Create a court order.""" + if not filing_submission: + return {"error": babel("Filing is required before a court order can be created.")} - filing_submission.court_order_file_number = court_order.get("fileNumber") - filing_submission.court_order_effect_of_order = court_order.get("effectOfOrder") - filing_submission.order_details = court_order.get("orderDetails") + if not (file_number := court_order.get("fileNumber")): + return + court_order_obj = CourtOrder( + file_number=file_number, + effect_of_order=court_order.get("effectOfOrder"), + order_details=court_order.get("orderDetails"), + ) with suppress(IndexError, KeyError, TypeError, ValueError): - filing_submission.court_order_date = datetime.fromisoformat(court_order.get("orderDate")) + court_order_obj.order_date = datetime.fromisoformat(court_order.get("orderDate")) + + filing_meta.court_order = {"fileNumber": file_number} + if court_order_obj.effect_of_order: + filing_meta.court_order["effectOfOrder"] = court_order_obj.effect_of_order + if court_order.get("orderDate"): + filing_meta.court_order["orderDate"] = court_order.get("orderDate") + if filing_submission.filing_type == FilingTypes.COURTORDER.value: + # Only add order details for court orders + filing_meta.court_order["orderDetails"] = court_order_obj.order_details + + if new_business: + court_order_obj.filing_id = filing_submission.id + new_business.court_orders.append(court_order_obj) + else: + court_order_obj.business_id = filing_submission.business_id + filing_submission.court_orders.append(court_order_obj) return None diff --git a/queue_services/business-filer/src/business_filer/filing_processors/filing_components/relationships.py b/queue_services/business-filer/src/business_filer/filing_processors/filing_components/relationships.py index 17e1610c17..9286efa8d5 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/filing_components/relationships.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/filing_components/relationships.py @@ -213,6 +213,37 @@ def has_ceased_role(roles: list[dict], role: str): party_role.cessation_date = date_time +def update_relationships_appointment_date( + relationships: list[dict], + business: Business, + allowed_roles: list[str]): + """Update the appointment date for the given parties for a business. + + Assumption: The structure has already been validated, upon submission. + """ + def _get_info_by_role(roles: list[dict], role: str): + return next( + role_info for role_info in roles + if RELATIONSHIP_ROLE_CONVERTER.get(role_info.get("roleType", "").lower(), "") == role and + role_info.get("appointmentDate") + ) + + existing_relationships = [ + relationship + for relationship in relationships + if relationship.get("entity", {}).get("identifier") is not None + ] + for relationship in existing_relationships: + party_id = _str_to_int(relationship["entity"]["identifier"]) + party_roles: list[PartyRole] = PartyRole.get_party_roles_by_party_id(business.id, party_id) + + for party_role in party_roles: + if (party_role.role in allowed_roles + and (role_info := _get_info_by_role(relationship.get("roles", []), party_role.role)) + ): + party_role.appointment_date = role_info.get("appointmentDate") + + def update_relationship_addresses(relationships: list[dict], business: Business) -> None: """Update the relationship addresses for existing party relationships. diff --git a/queue_services/business-filer/src/business_filer/filing_processors/incorporation_filing.py b/queue_services/business-filer/src/business_filer/filing_processors/incorporation_filing.py index f4386bc566..dbc79f4022 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/incorporation_filing.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/incorporation_filing.py @@ -97,7 +97,7 @@ def process(business: Business, # noqa: PLR0912 aliases.update_aliases(business, name_translations) if court_order := incorp_filing.get("courtOrder"): - filings.update_filing_court_order(filing_rec, court_order) + filings.create_court_order(filing_rec, court_order, filing_meta, business) if not filing_rec.colin_event_ids: # Update the filing json with identifier and founding date. diff --git a/queue_services/business-filer/src/business_filer/filing_processors/notice_of_withdrawal.py b/queue_services/business-filer/src/business_filer/filing_processors/notice_of_withdrawal.py index 7d08b2614a..531ccea4cf 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/notice_of_withdrawal.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/notice_of_withdrawal.py @@ -30,7 +30,7 @@ def process( current_app.logger.debug("start notice_of_withdrawal filing process, noticeOfWithdrawal: %s", now_filing) if court_order := now_filing.get("courtOrder"): - filings.update_filing_court_order(filing_submission, court_order) + filings.create_court_order(filing_submission, court_order, filing_meta) withdrawn_filing_id = now_filing.get("filingId") withdrawn_filing = Filing.find_by_id(withdrawn_filing_id) diff --git a/queue_services/business-filer/src/business_filer/filing_processors/put_back_off.py b/queue_services/business-filer/src/business_filer/filing_processors/put_back_off.py index 59060683db..765150668e 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/put_back_off.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/put_back_off.py @@ -38,9 +38,9 @@ def process(business: Business, filing: dict, filing_rec: Filing, filing_meta: F # update court order, if any is present with suppress(IndexError, KeyError, TypeError): court_order_json = dpath.get(put_back_off_filing, "/courtOrder") - filings.update_filing_court_order(filing_rec, court_order_json) + filings.create_court_order(filing_rec, court_order_json, filing_meta) - filing_rec.order_details = put_back_off_filing.get("details") + filing_rec.details = put_back_off_filing.get("details") if business.restoration_expiry_date: filing_meta.put_back_off = { diff --git a/queue_services/business-filer/src/business_filer/filing_processors/put_back_on.py b/queue_services/business-filer/src/business_filer/filing_processors/put_back_on.py index d0afbdef39..36ba55176a 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/put_back_on.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/put_back_on.py @@ -56,9 +56,9 @@ def process(business: Business, filing: dict, filing_rec: Filing, filing_meta: F # update court order, if any is present with suppress(IndexError, KeyError, TypeError): court_order_json = dpath.get(put_back_on_filing, "/courtOrder") - filings.update_filing_court_order(filing_rec, court_order_json) + filings.create_court_order(filing_rec, court_order_json, filing_meta) - filing_rec.order_details = put_back_on_filing.get("details") + filing_rec.details = put_back_on_filing.get("details") restoration.cease_custodian(business) diff --git a/queue_services/business-filer/src/business_filer/filing_processors/registrars_notation.py b/queue_services/business-filer/src/business_filer/filing_processors/registrars_notation.py index ea2b1bd2e5..15c79de0f4 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/registrars_notation.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/registrars_notation.py @@ -32,20 +32,20 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. """File processing rules and actions for the registrars notation filing.""" -from contextlib import suppress -from datetime import datetime - from business_model.models import Filing from business_filer.filing_meta import FilingMeta +from business_filer.filing_processors.filing_components import filings def process(registrars_notation_filing: Filing, filing: dict, filing_meta: FilingMeta): """Render the registrars notation filing into the business model objects.""" - registrars_notation_filing.court_order_file_number = filing["registrarsNotation"].get("fileNumber") - registrars_notation_filing.court_order_effect_of_order = filing["registrarsNotation"].get("effectOfOrder") - registrars_notation_filing.order_details = filing["registrarsNotation"]["orderDetails"] - - with suppress(IndexError, KeyError, TypeError, ValueError): - registrars_notation_filing.court_order_date = datetime.fromisoformat( - filing["registrarsNotation"].get("orderDate")) + filings.create_court_order( + registrars_notation_filing, + { + "fileNumber": filing["registrarsNotation"].get("fileNumber"), + "effectOfOrder": filing["registrarsNotation"].get("effectOfOrder") + }, + filing_meta + ) + registrars_notation_filing.details = filing["registrarsNotation"]["orderDetails"] diff --git a/queue_services/business-filer/src/business_filer/filing_processors/registrars_order.py b/queue_services/business-filer/src/business_filer/filing_processors/registrars_order.py index 31bb3aae07..c37a1d63bd 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/registrars_order.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/registrars_order.py @@ -32,20 +32,20 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. """File processing rules and actions for the registrars order filing.""" -from contextlib import suppress -from datetime import datetime - from business_model.models import Filing from business_filer.filing_meta import FilingMeta +from business_filer.filing_processors.filing_components import filings def process(registrars_order_filing: Filing, filing: dict, filing_meta: FilingMeta): """Render the registrars order filing into the business model objects.""" - registrars_order_filing.court_order_file_number = filing["registrarsOrder"].get("fileNumber") - registrars_order_filing.court_order_effect_of_order = filing["registrarsOrder"].get("effectOfOrder") - registrars_order_filing.order_details = filing["registrarsOrder"]["orderDetails"] - - with suppress(IndexError, KeyError, TypeError, ValueError): - registrars_order_filing.court_order_date = datetime.fromisoformat( - filing["registrarsOrder"].get("orderDate")) + filings.create_court_order( + registrars_order_filing, + { + "fileNumber": filing["registrarsOrder"].get("fileNumber"), + "effectOfOrder": filing["registrarsOrder"].get("effectOfOrder") + }, + filing_meta + ) + registrars_order_filing.details = filing["registrarsOrder"]["orderDetails"] diff --git a/queue_services/business-filer/src/business_filer/filing_processors/registration.py b/queue_services/business-filer/src/business_filer/filing_processors/registration.py index cdb5fbb531..45a2dbaf65 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/registration.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/registration.py @@ -94,7 +94,7 @@ def process(business: Business, # pylint: disable=too-many-branches # update court order, if any is present if court_order := registration_filing.get("courtOrder"): - filings.update_filing_court_order(filing_rec, court_order) + filings.create_court_order(filing_rec, court_order, filing_meta, business) # Update the filing json with identifier and founding date. registration_json = copy.deepcopy(filing_rec.filing_json) diff --git a/queue_services/business-filer/src/business_filer/filing_processors/restoration.py b/queue_services/business-filer/src/business_filer/filing_processors/restoration.py index 032812f8aa..38491f5b63 100644 --- a/queue_services/business-filer/src/business_filer/filing_processors/restoration.py +++ b/queue_services/business-filer/src/business_filer/filing_processors/restoration.py @@ -98,7 +98,7 @@ def process(business: Business, filing: dict, filing_rec: Filing, filing_meta: F if filing_rec.approval_type == "courtOrder": with suppress(IndexError, KeyError, TypeError): court_order_json = dpath.get(restoration_filing, "/courtOrder") - filings.update_filing_court_order(filing_rec, court_order_json) + filings.create_court_order(filing_rec, court_order_json, filing_meta) elif filing_rec.approval_type == "registrar": application_date = restoration_filing.get("applicationDate") notice_date = restoration_filing.get("noticeDate") diff --git a/queue_services/business-filer/src/business_filer/services/filer.py b/queue_services/business-filer/src/business_filer/services/filer.py index 3a16572a08..fc3ffb8d10 100644 --- a/queue_services/business-filer/src/business_filer/services/filer.py +++ b/queue_services/business-filer/src/business_filer/services/filer.py @@ -326,6 +326,15 @@ def process_filing(filing_message: FilingMessage): # noqa: PLR0915, PLR0912 if not Flags.is_on("enable-sandbox"): PublishEvent.publish_email_message(current_app, business, filing_submission, filing_submission.status) + try: + # Update the DRS record(s) for any client-submitted documents on this filing with + # the filing id, filing date, and business identifier + PublishEvent.publish_drs_update_message(current_app, business, filing_submission) + except Exception as err: + # log error for ops, but don't prevent filing from completing + current_app.logger.warning(err.with_traceback(None)) + current_app.logger.warning(f"Failed to publish DRS update for {filing_submission.id}.") + if filing_type in [ FilingTypes.CHANGEOFLIQUIDATORS, FilingTypes.CHANGEOFRECEIVERS diff --git a/queue_services/business-filer/src/business_filer/services/publish_event.py b/queue_services/business-filer/src/business_filer/services/publish_event.py index b95b1ba804..ca1a2e4244 100644 --- a/queue_services/business-filer/src/business_filer/services/publish_event.py +++ b/queue_services/business-filer/src/business_filer/services/publish_event.py @@ -1,8 +1,9 @@ +import re import uuid from datetime import UTC, datetime # if TYPE_CHECKING: -from business_model.models import Business, Filing +from business_model.models import Business, Document, Filing from flask import Flask from business_filer.common.filing import FilingTypes @@ -10,6 +11,8 @@ from business_filer.services import Flags, gcp_queue from gcp_queue import SimpleCloudEvent, to_queue_message +# DRS keys are formatted as "{documentClass}-{documentServiceId}", e.g. "COOP-DS0000101951". +_DRS_KEY_PATTERN = re.compile(r"^[A-Z]+-DS\d+$") class PublishEvent: """Service to publish specific events onto the GCP Queue.""" @@ -78,6 +81,39 @@ def publish_drs_create_message(app: Flask, business: Business, filing: Filing): except Exception as err: # pylint: disable=broad-except; raise PublishException(err) from err + @staticmethod + def publish_drs_update_message(app: Flask, business: Business, filing: Filing): + """Publish a drs update record message for each DRS document uploaded with the filing. + + Updates the document record(s) with filing/business information once a filing completes, + so the document shows up correctly in the ledger and is searchable in the DRS UI. + """ + try: + subject = app.config.get("DOC_UPDATE_REC_TOPIC") + documents = Document.query.filter_by(filing_id=filing.id).all() + for document in documents: + if not PublishEvent._is_drs_document(document.file_key): + continue + data = { + "accountId": "business-api", + "fileKey": document.file_key, + "businessIdentifier": business.identifier if business else None, + "filingDate": filing.completion_date.isoformat() if filing.completion_date else None, + "filingId": filing.id + } + data = {k: v for k, v in data.items() if v is not None} + + ce = PublishEvent._create_cloud_event(app, business, filing, subject, data) + gcp_queue.publish(subject, to_queue_message(ce)) + + except Exception as err: # pylint: disable=broad-except; + raise PublishException(err) from err + + @staticmethod + def _is_drs_document(file_key: str) -> bool: + """Return True if the file_key is a DRS key.""" + return bool(file_key) and bool(_DRS_KEY_PATTERN.match(file_key)) + @staticmethod def publish_mras_email(app: Flask, business: Business, filing: Filing): """Publish MRAS email message onto the NATS emailer subject.""" diff --git a/queue_services/business-filer/tests/unit/filing_processors/filing_components/test_filing.py b/queue_services/business-filer/tests/unit/filing_processors/filing_components/test_filing.py index 2e6818a381..f45efa3702 100644 --- a/queue_services/business-filer/tests/unit/filing_processors/filing_components/test_filing.py +++ b/queue_services/business-filer/tests/unit/filing_processors/filing_components/test_filing.py @@ -32,38 +32,48 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. """The Unit Tests for the Name Request filing component.""" +from business_filer.filing_meta import FilingMeta import copy +import random from datetime import datetime, timezone from typing import Final from registry_schemas.example_data import ALTERATION_FILING_TEMPLATE from business_filer.filing_processors.filing_components import filings -from tests.unit import create_filing +from tests.unit import create_business, create_filing -def test_update_filing_court_order(app, session): +def test_create_court_order(app, session): """Assert that the new aliases are created.""" # setup file_number: Final = '#1234-5678/90' order_date: Final = '2021-01-30T09:56:01+08:00' effect_of_order: Final = 'hasPlan' + order_details: Final = 'Some order details' + identifier = f'BC{random.randint(1000000, 9999999)}' + business = create_business(identifier, legal_type='BC') filing = copy.deepcopy(ALTERATION_FILING_TEMPLATE) - alteration_filing = create_filing(token='123', json_filing=filing) - court_order_json= {'courtOrder': + alteration_filing = create_filing(token='123', json_filing=filing, business_id=business.id) + court_order_json = {'courtOrder': { 'fileNumber': file_number, 'orderDate': order_date, - 'effectOfOrder': effect_of_order + 'effectOfOrder': effect_of_order, + 'orderDetails': order_details } } # test - filings.update_filing_court_order(alteration_filing, court_order_json['courtOrder']) + effective_date = datetime.now(timezone.utc) + filing_meta = FilingMeta(application_date=effective_date) + filings.create_court_order(alteration_filing, court_order_json['courtOrder'], filing_meta) # validate - assert file_number == alteration_filing.court_order_file_number - assert datetime.fromisoformat(order_date) == alteration_filing.court_order_date - assert effect_of_order == alteration_filing.court_order_effect_of_order - + court_order = alteration_filing.court_orders[0] + assert file_number == court_order.file_number + assert datetime.fromisoformat(order_date) == court_order.order_date + assert effect_of_order == court_order.effect_of_order + assert filing_meta.court_order['fileNumber'] == file_number + assert filing_meta.court_order['effectOfOrder'] == effect_of_order diff --git a/queue_services/business-filer/tests/unit/filing_processors/test_alteration.py b/queue_services/business-filer/tests/unit/filing_processors/test_alteration.py index 7a06a366d3..99ae4a244a 100644 --- a/queue_services/business-filer/tests/unit/filing_processors/test_alteration.py +++ b/queue_services/business-filer/tests/unit/filing_processors/test_alteration.py @@ -40,7 +40,6 @@ import pytest from business_model.models import Business, Filing, Document from business_model.models.document import DocumentType -# from legal_api.services.minio import MinioService from registry_schemas.example_data import ( ALTERATION, ALTERATION_FILING_TEMPLATE, diff --git a/queue_services/business-filer/tests/unit/filing_processors/test_dissolution.py b/queue_services/business-filer/tests/unit/filing_processors/test_dissolution.py index dce013bdb0..ebffe1bf25 100644 --- a/queue_services/business-filer/tests/unit/filing_processors/test_dissolution.py +++ b/queue_services/business-filer/tests/unit/filing_processors/test_dissolution.py @@ -41,7 +41,6 @@ from business_model.models import BatchProcessing, Batch, Business, Office, OfficeType, Party, PartyRole, Filing, Amalgamation, AmalgamatingBusiness from business_model.models.document import DocumentType -# from legal_api.services.minio import MinioService from business_filer.common.legislation_datetime import LegislationDatetime from registry_schemas.example_data import DISSOLUTION, FILING_HEADER @@ -242,10 +241,6 @@ def test_administrative_dissolution(app, session, legal_type, identifier, dissol assert documents[0].type == DocumentType.AFFIDAVIT.value affidavit_key = filing_json['filing']['dissolution']['affidavitFileKey'] assert documents[0].file_key == affidavit_key - # assert MinioService.get_file(documents[0].file_key) - # affidavit_obj = MinioService.get_file(affidavit_key) - # assert affidavit_obj - # assert_pdf_contains_text('Filed on ', affidavit_obj.read()) assert filing_meta.dissolution['dissolutionType'] == dissolution_type @@ -255,7 +250,7 @@ def test_administrative_dissolution(app, session, legal_type, identifier, dissol assert filing_meta.dissolution['dissolutionDate'] == dissolution_date_str final_filing = Filing.find_by_id(filing_id) - assert filing_json['filing']['dissolution']['details'] == final_filing.order_details + assert filing_json['filing']['dissolution']['details'] == final_filing.details @pytest.mark.parametrize('dissolution_type', [ diff --git a/queue_services/business-filer/tests/unit/filing_processors/test_incorporation_filing.py b/queue_services/business-filer/tests/unit/filing_processors/test_incorporation_filing.py index 60bd86c934..4c856c492d 100644 --- a/queue_services/business-filer/tests/unit/filing_processors/test_incorporation_filing.py +++ b/queue_services/business-filer/tests/unit/filing_processors/test_incorporation_filing.py @@ -42,7 +42,6 @@ from business_model.models import Business, Filing from business_model.models.colin_event_id import ColinEventId from business_model.models.document import DocumentType -# from legal_api.services import MinioService from business_filer.services import AccountService from registry_schemas.example_data import INCORPORATION_FILING_TEMPLATE @@ -111,9 +110,14 @@ def test_incorporation_filing_process_with_nr(app, session, legal_type, filing, assert len(filing_rec.filing_party_roles.all()) == 2 assert len(business.share_classes.all()) == 2 assert len(business.offices.all()) == 2 # One office is created in create_business method. - if legal_type == 'CC': - assert filing_rec.court_order_file_number == '12356' - assert filing_rec.court_order_effect_of_order == 'planOfArrangement' + if legal_type == 'CC': + court_order_obj = business.court_orders[0] + court_order = filing['filing']['incorporationApplication']['courtOrder'] + assert court_order['fileNumber'] == court_order_obj.file_number + assert court_order['effectOfOrder'] == court_order_obj.effect_of_order + assert filing_meta.court_order['fileNumber'] == court_order['fileNumber'] + assert filing_meta.court_order['effectOfOrder'] == court_order['effectOfOrder'] + if legal_type == 'CP': assert len(filing_rec.filing_party_roles.all()) == 1 assert len(business.offices.all()) == 1 @@ -161,8 +165,11 @@ def test_incorporation_filing_process_no_nr(app, session, legal_type, filing, le assert len(business.offices.all()) == 2 # One office is created in create_business method. assert len(business.party_roles.all()) == 1 assert len(filing_rec.filing_party_roles.all()) == 2 - assert filing_rec.court_order_file_number == '12356' - assert filing_rec.court_order_effect_of_order == 'planOfArrangement' + + court_order_obj = business.court_orders[0] + court_order = filing['filing']['incorporationApplication']['courtOrder'] + assert court_order['fileNumber'] == court_order_obj.file_number + assert court_order['effectOfOrder'] == court_order_obj.effect_of_order # Parties parties = filing_rec.filing_json['filing']['incorporationApplication']['parties'] diff --git a/queue_services/business-filer/tests/unit/filing_processors/test_registration.py b/queue_services/business-filer/tests/unit/filing_processors/test_registration.py index fac1d83155..ff21fbdf91 100644 --- a/queue_services/business-filer/tests/unit/filing_processors/test_registration.py +++ b/queue_services/business-filer/tests/unit/filing_processors/test_registration.py @@ -108,6 +108,14 @@ def test_registration_process(app, session, legal_type, filing): business, filing_rec, filing_meta = registration.process(None, filing, filing_rec, filing_meta) # Assertions + + court_order_obj = business.court_orders[0] + court_order = filing['filing']['registration']['courtOrder'] + assert court_order['fileNumber'] == court_order_obj.file_number + assert court_order['effectOfOrder'] == court_order_obj.effect_of_order + assert filing_meta.court_order['fileNumber'] == court_order['fileNumber'] + assert filing_meta.court_order['effectOfOrder'] == court_order['effectOfOrder'] + assert business.identifier.startswith('FM') assert business.founding_date == effective_date assert business.start_date == datetime.fromisoformat(f'{now}T08:00:00+00:00') @@ -155,10 +163,9 @@ def test_registration_affiliation(app, session, legal_type, filing, party_type, filing['filing']['registration']['parties'][0]['officer']['lastName'] = last_name filing['filing']['registration']['parties'][0]['officer']['middleName'] = middle_name - create_filing('123', filing) + filing_rec = create_filing('123', filing) effective_date = datetime.now(timezone.utc) - filing_rec = Filing(effective_date=effective_date, filing_json=filing) filing_meta = FilingMeta(application_date=effective_date) naics_response = { diff --git a/queue_services/business-filer/tests/unit/test_filer/test_alteration.py b/queue_services/business-filer/tests/unit/test_filer/test_alteration.py index 5be0a6fc29..f776e1549a 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_alteration.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_alteration.py @@ -40,7 +40,6 @@ import pytest from business_model.models import Business, Filing, Document from business_model.models.document import DocumentType -# from legal_api.services.minio import MinioService from registry_schemas.example_data import ( ALTERATION, ALTERATION_FILING_TEMPLATE, @@ -227,9 +226,13 @@ def tests_filer_alteration_court_order(app, session, mocker): # Check outcome final_filing = Filing.find_by_id(filing_id) - assert file_number == final_filing.court_order_file_number - assert datetime.fromisoformat(order_date) == final_filing.court_order_date - assert effect_of_order == final_filing.court_order_effect_of_order + + court_order_obj = final_filing.court_orders[0] + assert file_number == court_order_obj.file_number + assert datetime.fromisoformat(order_date) == court_order_obj.order_date + assert effect_of_order == court_order_obj.effect_of_order + assert final_filing.meta_data.get('courtOrder')['fileNumber'] == file_number + assert final_filing.meta_data.get('courtOrder')['effectOfOrder'] == effect_of_order @pytest.mark.parametrize('new_association_type', [ diff --git a/queue_services/business-filer/tests/unit/test_filer/test_amalgamation_application.py b/queue_services/business-filer/tests/unit/test_filer/test_amalgamation_application.py index 2d1ef50cbe..b27da57ace 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_amalgamation_application.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_amalgamation_application.py @@ -83,6 +83,15 @@ def test_regular_amalgamation_application_process(app, session, set_publish_mock filing_rec = Filing.find_by_id(filing_rec.id) business = Business.find_by_identifier(next_corp_num) + court_order_obj = filing_rec.court_orders[0] + court_order = filing['filing'][filing_type]['courtOrder'] + assert business.id == court_order_obj.business_id + assert court_order['orderDetails'] == court_order_obj.order_details + assert court_order['fileNumber'] == court_order_obj.file_number + assert court_order['effectOfOrder'] == court_order_obj.effect_of_order + assert filing_rec.meta_data.get('courtOrder')['fileNumber'] == court_order_obj.file_number + assert filing_rec.meta_data.get('courtOrder')['effectOfOrder'] == court_order_obj.effect_of_order + assert filing_rec.business_id == business.id assert filing_rec.status == Filing.Status.COMPLETED.value assert business.identifier diff --git a/queue_services/business-filer/tests/unit/test_filer/test_amalgamation_out.py b/queue_services/business-filer/tests/unit/test_filer/test_amalgamation_out.py index f83ee2d104..06ad837813 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_amalgamation_out.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_amalgamation_out.py @@ -50,8 +50,12 @@ def tests_filer_amalgamation_out(app, session): amalgamation_out_date_str = filing_json['filing']['amalgamationOut']['amalgamationOutDate'] amalgamation_out_date = LegislationDatetime.as_utc_timezone_from_legislation_date_str(amalgamation_out_date_str) - assert filing_json['filing']['amalgamationOut']['courtOrder']['fileNumber'] == final_filing.court_order_file_number - assert filing_json['filing']['amalgamationOut']['courtOrder']['effectOfOrder'] == final_filing.court_order_effect_of_order + court_order_obj = final_filing.court_orders[0] + court_order = filing_json['filing']['amalgamationOut']['courtOrder'] + assert court_order['fileNumber'] == court_order_obj.file_number + assert court_order['effectOfOrder'] == court_order_obj.effect_of_order + assert filing_meta.court_order['fileNumber'] == court_order['fileNumber'] + assert filing_meta.court_order['effectOfOrder'] == court_order['effectOfOrder'] assert business.state == Business.State.HISTORICAL assert business.state_filing_id == final_filing.id diff --git a/queue_services/business-filer/tests/unit/test_filer/test_change_of_registration.py b/queue_services/business-filer/tests/unit/test_filer/test_change_of_registration.py index 3bd2edd041..122c8b74fb 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_change_of_registration.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_change_of_registration.py @@ -249,9 +249,12 @@ def tests_filer_change_of_registration_court_order(app, session, mocker, test_na # Check outcome final_filing = Filing.find_by_id(filing_id) - assert file_number == final_filing.court_order_file_number - assert datetime.fromisoformat(order_date) == final_filing.court_order_date - assert effect_of_order == final_filing.court_order_effect_of_order + court_order_obj = final_filing.court_orders[0] + assert file_number == court_order_obj.file_number + assert datetime.fromisoformat(order_date) == court_order_obj.order_date + assert effect_of_order == court_order_obj.effect_of_order + assert final_filing.meta_data.get('courtOrder')['fileNumber'] == file_number + assert final_filing.meta_data.get('courtOrder')['effectOfOrder'] == effect_of_order def tests_filer_proprietor_name_and_address_change(app, session, mocker): diff --git a/queue_services/business-filer/tests/unit/test_filer/test_consent_amalgamation_out.py b/queue_services/business-filer/tests/unit/test_filer/test_consent_amalgamation_out.py index 999deec538..1e052a2fee 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_consent_amalgamation_out.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_consent_amalgamation_out.py @@ -71,10 +71,12 @@ def tests_filer_consent_amalgamation_out(app, session, mocker, test_name, effect # Check outcome final_filing = Filing.find_by_id(cco_filing.id) - assert filing_json['filing']['consentAmalgamationOut']['courtOrder']['fileNumber'] == \ - final_filing.court_order_file_number - assert filing_json['filing']['consentAmalgamationOut']['courtOrder']['effectOfOrder'] == \ - final_filing.court_order_effect_of_order + court_order_obj = final_filing.court_orders[0] + court_order = filing_json['filing']['consentAmalgamationOut']['courtOrder'] + assert court_order['fileNumber'] == court_order_obj.file_number + assert court_order['effectOfOrder'] == court_order_obj.effect_of_order + assert final_filing.meta_data.get('courtOrder')['fileNumber'] == court_order['fileNumber'] + assert final_filing.meta_data.get('courtOrder')['effectOfOrder'] == court_order['effectOfOrder'] expiry_date_utc = LegislationDatetime.as_utc_timezone(expiry_date) diff --git a/queue_services/business-filer/tests/unit/test_filer/test_consent_continuation_out.py b/queue_services/business-filer/tests/unit/test_filer/test_consent_continuation_out.py index bef79d82b5..d8c4881345 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_consent_continuation_out.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_consent_continuation_out.py @@ -91,10 +91,12 @@ def tests_filer_consent_continuation_out(app, session, mocker, test_name, effect # Check outcome final_filing = Filing.find_by_id(cco_filing.id) - assert filing_json['filing']['consentContinuationOut']['courtOrder']['fileNumber'] == \ - final_filing.court_order_file_number - assert filing_json['filing']['consentContinuationOut']['courtOrder']['effectOfOrder'] == \ - final_filing.court_order_effect_of_order + court_order_obj = final_filing.court_orders[0] + court_order = filing_json['filing']['consentContinuationOut']['courtOrder'] + assert court_order['fileNumber'] == court_order_obj.file_number + assert court_order['effectOfOrder'] == court_order_obj.effect_of_order + assert final_filing.meta_data.get('courtOrder')['fileNumber'] == court_order['fileNumber'] + assert final_filing.meta_data.get('courtOrder')['effectOfOrder'] == court_order['effectOfOrder'] expiry_date_utc = LegislationDatetime.as_utc_timezone(expiry_date) diff --git a/queue_services/business-filer/tests/unit/test_filer/test_continuation_in.py b/queue_services/business-filer/tests/unit/test_filer/test_continuation_in.py index fe0dc371f8..2f1c4aa394 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_continuation_in.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_continuation_in.py @@ -53,6 +53,9 @@ def test_continuation_in_process(app, session): filing_rec = Filing.find_by_id(filing_rec.id) business = Business.find_by_identifier(next_corp_num) + court_order_obj = filing_rec.court_orders.all() + assert len(court_order_obj) == 0 + assert filing_rec.business_id == business.id assert filing_rec.status == Filing.Status.COMPLETED.value assert business.identifier diff --git a/queue_services/business-filer/tests/unit/test_filer/test_continuation_out.py b/queue_services/business-filer/tests/unit/test_filer/test_continuation_out.py index 464f507ab2..4579bf85e2 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_continuation_out.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_continuation_out.py @@ -70,8 +70,12 @@ def tests_filer_continuation_out(app, session): continuation_out_date_str = filing_json['filing']['continuationOut']['continuationOutDate'] continuation_out_date = LegislationDatetime.as_utc_timezone_from_legislation_date_str(continuation_out_date_str) - assert filing_json['filing']['continuationOut']['courtOrder']['fileNumber'] == final_filing.court_order_file_number - assert filing_json['filing']['continuationOut']['courtOrder']['effectOfOrder'] == final_filing.court_order_effect_of_order + court_order_obj = final_filing.court_orders[0] + court_order = filing_json['filing']['continuationOut']['courtOrder'] + assert court_order['fileNumber'] == court_order_obj.file_number + assert court_order['effectOfOrder'] == court_order_obj.effect_of_order + assert filing_meta.court_order['fileNumber'] == court_order['fileNumber'] + assert filing_meta.court_order['effectOfOrder'] == court_order['effectOfOrder'] assert business.state == Business.State.HISTORICAL assert business.state_filing_id == final_filing.id diff --git a/queue_services/business-filer/tests/unit/test_filer/test_correction.py b/queue_services/business-filer/tests/unit/test_filer/test_correction.py index cc0b2c0ffc..e4b75f62b2 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_correction.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_correction.py @@ -58,6 +58,7 @@ CORRECTION_COD = copy.deepcopy(CORRECTION_COL) CORRECTION_COD['filing']['correction']['correctedFilingType'] = 'changeOfDirectors' CORRECTION_COD['filing']['correction']['relationships'][0]['roles'][0]['roleType'] = 'Director' +CORRECTION_COD['filing']['correction']['relationships'][0]['roles'][0]['appointmentDate'] = '2026-07-30' def _assert_common_data(business: Business, filing: Filing): @@ -155,3 +156,5 @@ def test_process_correction_filing_with_relationships(app, session, mocker, fili assert role.party.first_name == expected_given_name.upper() assert role.party.mailing_address.street == expected_mailing_street assert role.party.delivery_address.street == expected_delivery_street + if filing_name == 'changeOfDirectors': + assert role.appointment_date.date().isoformat() == correction_data['filing']['correction']['relationships'][0]['roles'][0]['appointmentDate'] diff --git a/queue_services/business-filer/tests/unit/test_filer/test_correction_bcia.py b/queue_services/business-filer/tests/unit/test_filer/test_correction_bcia.py index 0a7de2f821..4ec82f4e8f 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_correction_bcia.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_correction_bcia.py @@ -506,9 +506,12 @@ def tests_filer_correction_court_order(app, session, mocker, test_name, legal_ty # Check outcome final_filing = Filing.find_by_id(filing_id) - assert file_number == final_filing.court_order_file_number - assert datetime.fromisoformat(order_date) == final_filing.court_order_date - assert effect_of_order == final_filing.court_order_effect_of_order + court_order_obj = final_filing.court_orders[0] + assert file_number == court_order_obj.file_number + assert datetime.fromisoformat(order_date) == court_order_obj.order_date + assert effect_of_order == court_order_obj.effect_of_order + assert final_filing.meta_data.get('courtOrder')['fileNumber'] == file_number + assert final_filing.meta_data.get('courtOrder')['effectOfOrder'] == effect_of_order diff --git a/queue_services/business-filer/tests/unit/test_filer/test_correction_firms.py b/queue_services/business-filer/tests/unit/test_filer/test_correction_firms.py index b4b8743854..4df47fb7f8 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_correction_firms.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_correction_firms.py @@ -269,9 +269,12 @@ def tests_filer_correction_court_order(app, session, mocker, test_name, legal_ty # Check outcome final_filing = Filing.find_by_id(filing_id) - assert file_number == final_filing.court_order_file_number - assert datetime.fromisoformat(order_date) == final_filing.court_order_date - assert effect_of_order == final_filing.court_order_effect_of_order + court_order_obj = final_filing.court_orders[0] + assert file_number == court_order_obj.file_number + assert datetime.fromisoformat(order_date) == court_order_obj.order_date + assert effect_of_order == court_order_obj.effect_of_order + assert final_filing.meta_data.get('courtOrder')['fileNumber'] == file_number + assert final_filing.meta_data.get('courtOrder')['effectOfOrder'] == effect_of_order diff --git a/queue_services/business-filer/tests/unit/test_filer/test_correction_special_resolution.py b/queue_services/business-filer/tests/unit/test_filer/test_correction_special_resolution.py index f8460ffc06..32feb9c6a6 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_correction_special_resolution.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_correction_special_resolution.py @@ -42,14 +42,12 @@ from dateutil.parser import parse from business_model.models import Business, Document, Filing, PartyRole from business_model.models.document import DocumentType -# from legal_api.services.minio import MinioService from registry_schemas.example_data import CORRECTION_COA, CORRECTION_CP_SPECIAL_RESOLUTION,\ CP_SPECIAL_RESOLUTION_TEMPLATE, FILING_HEADER from business_filer.filing_meta import FilingMeta from business_filer.filing_processors import correction from business_filer.services.filer import process_filing from tests.unit import create_entity, create_filing -# from tests.utils import upload_file, assert_pdf_contains_text @pytest.mark.parametrize( @@ -64,14 +62,9 @@ def test_special_resolution_correction(app, session, mocker, test_name, correct_ """Test the special resolution correction functionality.""" return None # class MockFileResponse: - # """Mock the MinioService.""" - # def __init__(self, file_content): # self.data = io.BytesIO(file_content.encode('utf-8')) - # # Mock the MinioService's get_file method to return a dictionary with 'data' pointing - # # to an instance of MockFileResponse - # mocker.patch('legal_api.services.minio.MinioService.get_file', return_value=MockFileResponse('fake file content')) # mocker.patch('business_filer.services.publish_event.PublishEvent.publish_email_message', return_value=None) # mocker.patch('business_filer.services.publish_event.PublishEvent.publish_event', return_value=None) # mocker.patch('business_filer.filing_processors.filing_components.name_request.consume_nr', return_value=None) @@ -191,71 +184,3 @@ def test_special_resolution_correction(app, session, mocker, test_name, correct_ assert party.first_name == 'SARAH', 'First name should be corrected' assert party.last_name == 'DOE', 'Last name should be corrected' - -# def test_correction_coop_rules(app, session): -# """Assert that the coop rules and memorandum is altered.""" -# # Create business -# identifier = 'CP1234567' -# business = create_entity(identifier, 'CP', 'COOP INC.') -# business_id = business.id -# business.save() - -# # Create an initial special resolution filing -# sr_filing = copy.deepcopy(CP_SPECIAL_RESOLUTION_TEMPLATE) -# sr_payment_id = str(random.SystemRandom().getrandbits(0x58)) -# sr_filing_id = (create_filing(sr_payment_id, sr_filing, business_id=business_id)).id - -# # Mock the filing message -# sr_filing_msg = {'filing': {'id': sr_filing_id}} -# # Call the process_filing method for the original special resolution -# process_filing(sr_filing_msg, app) - -# correction_filing = copy.deepcopy(FILING_HEADER) -# correction_filing['filing']['header']['name'] = 'correction' -# correction_filing['filing']['business']['legalType'] = Business.LegalTypes.COOP.value -# correction_filing['filing']['correction'] = copy.deepcopy(CORRECTION_CP_SPECIAL_RESOLUTION) -# correction_filing['filing']['correction']['correctedFilingId'] = sr_filing_id -# rules_file_key_uploaded_by_user = upload_file('rules.pdf') -# correction_filing['filing']['correction']['rulesFileKey'] = rules_file_key_uploaded_by_user -# correction_filing['filing']['correction']['rulesFileName'] = 'rules.pdf' -# memorandum_file_key_uploaded_by_user = upload_file('memorandum.pdf') -# correction_filing['filing']['correction']['memorandumFileKey'] = memorandum_file_key_uploaded_by_user -# correction_filing['filing']['correction']['memorandumFileName'] = 'memorandum.pdf' - -# payment_id = str(random.SystemRandom().getrandbits(0x58)) - -# filing_submission = create_filing( -# payment_id, correction_filing, business_id=business.id, filing_date=datetime.datetime.now(datetime.timezone.utc) -# ) - -# filing_meta = FilingMeta() - -# # test -# correction.process(correction_filing=filing_submission, -# filing=correction_filing['filing'], -# filing_meta=filing_meta, -# business=business) - -# business.save() - -# rules_document = session.query(Document). \ -# filter(Document.filing_id == filing_submission.id). \ -# filter(Document.type == DocumentType.COOP_RULES.value). \ -# one_or_none() - -# assert rules_document.file_key == correction_filing['filing']['correction']['rulesFileKey'] -# assert MinioService.get_file(rules_document.file_key) -# rules_files_obj = MinioService.get_file(rules_file_key_uploaded_by_user) -# assert rules_files_obj -# assert_pdf_contains_text('Filed on ', rules_files_obj.read()) - -# memorandum_document = session.query(Document). \ -# filter(Document.filing_id == filing_submission.id). \ -# filter(Document.type == DocumentType.COOP_MEMORANDUM.value). \ -# one_or_none() - -# assert memorandum_document.file_key == correction_filing['filing']['correction']['memorandumFileKey'] -# assert MinioService.get_file(memorandum_document.file_key) -# memorandum_files_obj = MinioService.get_file(memorandum_file_key_uploaded_by_user) -# assert memorandum_files_obj -# assert_pdf_contains_text('Filed on ', memorandum_files_obj.read()) diff --git a/queue_services/business-filer/tests/unit/test_filer/test_court_order.py b/queue_services/business-filer/tests/unit/test_filer/test_court_order.py index e4bb9d6b79..3c1b9527ad 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_court_order.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_court_order.py @@ -32,6 +32,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. """The Unit Tests for the Court Order filing.""" +import uuid import copy import random @@ -61,11 +62,71 @@ def tests_filer_court_order(app, session): # Check outcome final_filing = Filing.find_by_id(filing_id) - assert filing['filing']['courtOrder']['fileNumber'] == final_filing.court_order_file_number - assert filing['filing']['courtOrder']['effectOfOrder'] == final_filing.court_order_effect_of_order - assert filing['filing']['courtOrder']['orderDetails'] == final_filing.order_details + court_order = final_filing.court_orders[0] + assert filing['filing']['courtOrder']['fileNumber'] == court_order.file_number + assert filing['filing']['courtOrder']['effectOfOrder'] == court_order.effect_of_order + assert filing['filing']['courtOrder']['orderDetails'] == court_order.order_details + assert final_filing.meta_data.get('courtOrder')['fileNumber'] == court_order.file_number + assert final_filing.meta_data.get('courtOrder')['effectOfOrder'] == court_order.effect_of_order + assert final_filing.meta_data.get('courtOrder')['orderDetails'] == court_order.order_details + assert final_filing.meta_data.get('courtOrder')['files'][0]['fileKey'] == filing['filing']['courtOrder']['fileKey'] + assert final_filing.meta_data.get('courtOrder')['files'][0]['fileName'] == f'Court Order {court_order.file_number}.pdf' court_order_file = final_filing.documents.one_or_none() assert court_order_file assert court_order_file.type == DocumentType.COURT_ORDER.value assert court_order_file.file_key == filing['filing']['courtOrder']['fileKey'] + + +def tests_filer_court_order_multiple_files(app, session): + """Assert that the court order object with multiple files is correctly populated to model objects.""" + identifier = f'BC{random.randint(1000000, 9999999)}' + business = create_business(identifier, legal_type='BC') + + filing = copy.deepcopy(COURT_ORDER_FILING_TEMPLATE) + filing['filing']['business']['identifier'] = identifier + file_key_1 = f"{uuid.uuid4()}.pdf" + file_key_2 = f"{uuid.uuid4()}.pdf" + file_name_1 = f'Court Order {filing['filing']['courtOrder']['fileNumber']}_01.pdf' + file_name_2 = f'Court Order {filing['filing']['courtOrder']['fileNumber']}_02.pdf' + filing['filing']['courtOrder']['files'] = [ + { + "fileKey": file_key_1, + "fileName": file_name_1 + }, + { + "fileKey": file_key_2, + "fileName": file_name_2 + } + ] + del filing['filing']['courtOrder']['fileKey'] + + + payment_id = str(random.SystemRandom().getrandbits(0x58)) + filing_id = (create_filing(payment_id, filing, business_id=business.id)).id + + filing_msg = FilingMessage(filing_identifier=filing_id) + + # Test + process_filing(filing_msg) + + # Check outcome + final_filing = Filing.find_by_id(filing_id) + court_order = final_filing.court_orders[0] + assert filing['filing']['courtOrder']['fileNumber'] == court_order.file_number + assert filing['filing']['courtOrder']['effectOfOrder'] == court_order.effect_of_order + assert filing['filing']['courtOrder']['orderDetails'] == court_order.order_details + assert final_filing.meta_data.get('courtOrder')['fileNumber'] == court_order.file_number + assert final_filing.meta_data.get('courtOrder')['effectOfOrder'] == court_order.effect_of_order + + court_order_files = final_filing.documents.all() + assert len(court_order_files) == 2 + for court_order_file in court_order_files: + assert court_order_file.type == DocumentType.COURT_ORDER.value + assert court_order_file.file_key in [file_key_1, file_key_2] + assert court_order_file.file_name in [file_name_1, file_name_2] + + assert len(final_filing.meta_data.get('courtOrder')['files']) == 2 + for court_order_file in final_filing.meta_data.get('courtOrder')['files']: + assert court_order_file['fileKey'] in [file_key_1, file_key_2] + assert court_order_file['fileName'] in [file_name_1, file_name_2] diff --git a/queue_services/business-filer/tests/unit/test_filer/test_notice_of_withdrawal.py b/queue_services/business-filer/tests/unit/test_filer/test_notice_of_withdrawal.py index e7d07ceae9..ab5babef6f 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_notice_of_withdrawal.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_notice_of_withdrawal.py @@ -72,7 +72,14 @@ def tests_filer_notice_of_withdrawal(session, test_name, filing_type, filing_tem final_withdrawn_filing = Filing.find_by_id(withdrawn_filing.id) final_now_filing = Filing.find_by_id(now_filing.id) - assert now_filing_json['filing']['noticeOfWithdrawal']['courtOrder']['orderDetails'] == final_now_filing.order_details + court_order_obj = final_now_filing.court_orders[0] + court_order = now_filing_json['filing']['noticeOfWithdrawal']['courtOrder'] + assert court_order['orderDetails'] == court_order_obj.order_details + assert court_order['fileNumber'] == court_order_obj.file_number + assert court_order['effectOfOrder'] == court_order_obj.effect_of_order + assert filing_meta.court_order['fileNumber'] == court_order['fileNumber'] + assert filing_meta.court_order['effectOfOrder'] == court_order['effectOfOrder'] + assert final_withdrawn_filing.status == Filing.Status.WITHDRAWN.value assert final_withdrawn_filing.withdrawal_pending == False assert final_withdrawn_filing.meta_data.get('withdrawnDate') diff --git a/queue_services/business-filer/tests/unit/test_filer/test_put_back_off.py b/queue_services/business-filer/tests/unit/test_filer/test_put_back_off.py index 3f4c6d61eb..daf3826ee9 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_put_back_off.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_put_back_off.py @@ -53,7 +53,7 @@ def tests_filer_put_back_off(session): assert business.state == Business.State.HISTORICAL assert business.state_filing_id == filing.id assert business.restoration_expiry_date is None - assert filing.order_details == final_filing.order_details + assert filing.details == final_filing.details assert filing_meta.put_back_off['reason'] == 'Limited Restoration Expired' assert filing_meta.put_back_off['expiryDate'] == LegislationDatetime.format_as_legislation_date(expiry) diff --git a/queue_services/business-filer/tests/unit/test_filer/test_registrars_notation.py b/queue_services/business-filer/tests/unit/test_filer/test_registrars_notation.py index 43bdcf6d8c..c816b96ccb 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_registrars_notation.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_registrars_notation.py @@ -61,6 +61,7 @@ def tests_filer_registrars_notation(app, session): # Check outcome final_filing = Filing.find_by_id(filing_id) - assert filing['filing']['registrarsNotation']['fileNumber'] == final_filing.court_order_file_number - assert filing['filing']['registrarsNotation']['effectOfOrder'] == final_filing.court_order_effect_of_order - assert filing['filing']['registrarsNotation']['orderDetails'] == final_filing.order_details + court_order = final_filing.court_orders[0] + assert filing['filing']['registrarsNotation']['fileNumber'] == court_order.file_number + assert filing['filing']['registrarsNotation']['effectOfOrder'] == court_order.effect_of_order + assert filing['filing']['registrarsNotation']['orderDetails'] == final_filing.details diff --git a/queue_services/business-filer/tests/unit/test_filer/test_registrars_order.py b/queue_services/business-filer/tests/unit/test_filer/test_registrars_order.py index 11d94ff34d..bebcc2ede9 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_registrars_order.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_registrars_order.py @@ -64,6 +64,7 @@ def tests_filer_registrars_order(app, session): # Check outcome final_filing = Filing.find_by_id(filing_id) - assert filing['filing']['registrarsOrder']['fileNumber'] == final_filing.court_order_file_number - assert filing['filing']['registrarsOrder']['effectOfOrder'] == final_filing.court_order_effect_of_order - assert filing['filing']['registrarsOrder']['orderDetails'] == final_filing.order_details + court_order = final_filing.court_orders[0] + assert filing['filing']['registrarsOrder']['fileNumber'] == court_order.file_number + assert filing['filing']['registrarsOrder']['effectOfOrder'] == court_order.effect_of_order + assert filing['filing']['registrarsOrder']['orderDetails'] == final_filing.details diff --git a/queue_services/business-filer/tests/unit/test_filer/test_restoration.py b/queue_services/business-filer/tests/unit/test_filer/test_restoration.py index c0c6066caa..3262e1dc75 100644 --- a/queue_services/business-filer/tests/unit/test_filer/test_restoration.py +++ b/queue_services/business-filer/tests/unit/test_filer/test_restoration.py @@ -207,9 +207,10 @@ def test_restoration_court_order(app, session, mocker, approval_type): final_filing = Filing.find_by_id(filing_id) assert filing['filing']['restoration']['approvalType'] == final_filing.approval_type if approval_type == 'courtOrder': - assert filing['filing']['restoration']['courtOrder']['fileNumber'] == final_filing.court_order_file_number + court_order_obj = final_filing.court_orders[0] + assert filing['filing']['restoration']['courtOrder']['fileNumber'] == court_order_obj.file_number else: - assert final_filing.court_order_file_number is None + assert final_filing.court_orders.one_or_none() is None @pytest.mark.parametrize('approval_type', [ diff --git a/queue_services/business-filer/tests/unit/test_publish/test_publish_drs_update.py b/queue_services/business-filer/tests/unit/test_publish/test_publish_drs_update.py new file mode 100644 index 0000000000..ddb9023de7 --- /dev/null +++ b/queue_services/business-filer/tests/unit/test_publish/test_publish_drs_update.py @@ -0,0 +1,190 @@ +# Copyright © 2026 Province of British Columbia +# +# Licensed under the BSD 3 Clause License, (the "License"); +# you may not use this file except in compliance with the License. +# The template for the license can be found here +# https://opensource.org/license/bsd-3-clause/ +# +# Redistribution and use in source and binary forms, +# with or without modification, are permitted provided that the +# following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +"""Unit tests for PublishEvent.publish_drs_update_message.""" +import json +from datetime import UTC, datetime +from unittest.mock import Mock, patch + +from business_model.models import Business, Document, Filing + +from business_filer.services import gcp_queue +from business_filer.services.publish_event import PublishEvent + +IDENTIFIER = 'BC1234567' +FILING_ID = 1438352 +COMPLETION_DATE = '2026-06-09T23:02:02+00:00' +FILE_KEY_1 = 'COOP-DS0000101951' +FILE_KEY_2 = 'BEN-DS0000101952' + +def _make_minimal_filing(): + filing = Filing() + filing.id = FILING_ID + filing._filing_type = 'continuationIn' + filing._completion_date = datetime.fromisoformat(COMPLETION_DATE) + return filing + + +def test_publish_drs_update_no_documents(app): + """Assert nothing is published when the filing has no documents.""" + business = Business(identifier=IDENTIFIER) + filing = _make_minimal_filing() + + mock_query = Mock() + mock_query.filter_by.return_value.all.return_value = [] + + with patch.object(Document, 'query', mock_query), \ + patch.object(gcp_queue, 'publish') as mock_publish: + + PublishEvent.publish_drs_update_message(app, business, filing) + + mock_publish.assert_not_called() + + +def test_publish_drs_update_drs_document(app): + """Assert a drs update message is published for a DRS document.""" + business = Business(identifier=IDENTIFIER) + filing = _make_minimal_filing() + + document = Document( + id=10, + filing_id=filing.id, + file_key=FILE_KEY_1 + ) + + mock_query = Mock() + mock_query.filter_by.return_value.all.return_value = [document] + + with patch.object(Document, 'query', mock_query), \ + patch.object(gcp_queue, 'publish') as mock_publish: + + PublishEvent.publish_drs_update_message(app, business, filing) + + mock_publish.assert_called_once() + subject, payload = mock_publish.call_args.args + assert subject == app.config['DOC_UPDATE_REC_TOPIC'] + payload_data = (json.loads(payload)).get('data') + assert payload_data == { + 'accountId': 'business-api', + 'fileKey': FILE_KEY_1, + 'businessIdentifier': IDENTIFIER, + 'filingDate': COMPLETION_DATE, + 'filingId': FILING_ID + } + + +def test_publish_drs_update_multiple_documents(app): + """Assert one message is published per DRS document, skipping legacy keys.""" + business = Business(identifier=IDENTIFIER) + filing = _make_minimal_filing() + + documents = [ + Document(id=10, filing_id=filing.id, file_key=FILE_KEY_1), + Document(id=11, filing_id=filing.id, file_key='550e8400-e29b-41d4-a716-446655440000'), + Document(id=12, filing_id=filing.id, file_key=FILE_KEY_2), + ] + + mock_query = Mock() + mock_query.filter_by.return_value.all.return_value = documents + + with patch.object(Document, 'query', mock_query), \ + patch.object(gcp_queue, 'publish') as mock_publish: + + PublishEvent.publish_drs_update_message(app, business, filing) + + assert mock_publish.call_count == 2 + file_keys = [ + (json.loads(call.args[1])).get('data', {}).get('fileKey') + for call in mock_publish.call_args_list + ] + assert file_keys == [FILE_KEY_1, FILE_KEY_2] + + +def test_publish_drs_update_skip_legacy_document(app): + """Assert legacy Minio documents are ignored.""" + business = Business(identifier=IDENTIFIER) + filing = _make_minimal_filing() + + document = Document( + id=10, + filing_id=filing.id, + file_key='550e8400-e29b-41d4-a716-446655440000' + ) + + mock_query = Mock() + mock_query.filter_by.return_value.all.return_value = [document] + + with patch.object(Document, 'query', mock_query), \ + patch.object(gcp_queue, 'publish') as mock_publish: + + PublishEvent.publish_drs_update_message(app, business, filing) + + mock_publish.assert_not_called() + + +def test_publish_drs_update_omits_empty_values(app): + """Assert filingDate is omitted when the filing has no completion date.""" + business = Business(identifier=IDENTIFIER) + filing = _make_minimal_filing() + filing._completion_date = None + + document = Document( + id=10, + filing_id=filing.id, + file_key=FILE_KEY_1 + ) + + mock_query = Mock() + mock_query.filter_by.return_value.all.return_value = [document] + + with patch.object(Document, 'query', mock_query), \ + patch.object(gcp_queue, 'publish') as mock_publish: + + PublishEvent.publish_drs_update_message(app, business, filing) + + mock_publish.assert_called_once() + payload_data = (json.loads(mock_publish.call_args.args[1])).get('data') + assert 'filingDate' not in payload_data + assert payload_data.get('filingId') == FILING_ID + + +def test_is_drs_document(): + """Assert DRS document detection.""" + assert PublishEvent._is_drs_document(FILE_KEY_1) + assert PublishEvent._is_drs_document(FILE_KEY_2) + assert PublishEvent._is_drs_document('COOP-DS0000123456') + assert PublishEvent._is_drs_document('BEN-DS123456') + + assert not PublishEvent._is_drs_document('550e8400-e29b-41d4-a716-446655440000') + assert not PublishEvent._is_drs_document('') + assert not PublishEvent._is_drs_document(None) diff --git a/support/notebooks/business-ops/correction-early-ar/add_registrars_notation_early_ar.ipynb b/support/notebooks/business-ops/correction-early-ar/add_registrars_notation_early_ar.ipynb new file mode 100644 index 0000000000..f0e6b62f32 --- /dev/null +++ b/support/notebooks/business-ops/correction-early-ar/add_registrars_notation_early_ar.ipynb @@ -0,0 +1,209 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "b2b96de1", + "metadata": {}, + "source": [ + "# Add Registrar's Notation for early AR filings (BAR AR Service)" + ] + }, + { + "cell_type": "markdown", + "id": "ab86d07d", + "metadata": {}, + "source": [ + " Purpose: Add Registrar's Notation filing to all businesses that used the BAR AR Service to file an Annual Report ahead of it being due.\n", + "\n", + "This is a one time (python) script to be run at a given date/time.
\n", + "Set the configuration (client_id, client_secret, url(s)) for a scpecific environment.
\n", + "Get access token for authorization.
" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5f18b034", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from dotenv import load_dotenv, find_dotenv\n", + "import psycopg2\n", + "\n", + "# this will load all the envars from a .env file located in the project root\n", + "load_dotenv(find_dotenv())\n", + "\n", + "%load_ext sql" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cdad9e35", + "metadata": {}, + "outputs": [], + "source": [ + "connect_to_db = 'postgresql://' + \\\n", + " os.getenv('ENTITY_DATABASE_USERNAME', '') + \":\" + os.getenv('ENTITY_DATABASE_PASSWORD', '') +'@' + \\\n", + " os.getenv('ENTITY_DATABASE_HOST', '') + ':' + os.getenv('ENTITY_DATABASE_PORT', '5432') + '/' + \\\n", + " os.getenv('ENTITY_DATABASE_NAME', '');\n", + "connect_to_db\n", + " \n", + "%sql $connect_to_db" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6eb7ffee", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql \n", + "select now() AT TIME ZONE 'PST' as current_date" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47248f21", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "import os\n", + "from datetime import datetime\n", + "\n", + "# token_url, client_id, client_secret, base_url - update based on environment\n", + "token_url = os.getenv('ACCOUNT_SVC_AUTH_URL')\n", + "client_id = os.getenv('ACCOUNT_SVC_CLIENT_ID')\n", + "client_secret = os.getenv('ACCOUNT_SVC_CLIENT_SECRET')\n", + "base_url = os.getenv('LEGAL_API_BASE_URL')\n", + "\n", + "header = {\n", + " \"Content-Type\": \"application/x-www-form-urlencoded\"\n", + "}\n", + "\n", + "data = 'grant_type=client_credentials'\n", + "\n", + "res = requests.post(token_url, data, auth=(client_id, client_secret), headers=header)\n", + "\n", + "# Check the status code of the response\n", + "if res.status_code == 200:\n", + " print(f\"Access token returned successfully : {base_url}\")\n", + " token = res.json()[\"access_token\"]\n", + "else:\n", + " print(f\"Failed to make POST request. Status code: {res.status_code}\")\n", + " print(res.text) # Print the error message if the request fails" + ] + }, + { + "cell_type": "markdown", + "id": "bbe4f3e6", + "metadata": {}, + "source": [ + "Call API (POST) endpoint to create Registrar's Notation filing for businesses." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97b473e4", + "metadata": {}, + "outputs": [], + "source": [ + "from urllib.parse import urljoin\n", + "from rn_early_ar_output_TEST import businesses\n", + "\n", + "current_date = datetime.now().date().isoformat()\n", + "headers = {\n", + " 'Content-Type': 'application/json',\n", + " 'Authorization': 'Bearer ' + token\n", + "}\n", + "\n", + "successful_identifiers = []\n", + "failed_identifiers = []\n", + "duplicate_identifiers = []\n", + "\n", + "# loop through list of businesses to create filing\n", + "for identifier in businesses:\n", + " business_details = %sql \\\n", + " SELECT b.state, b.legal_type, \\\n", + " (SELECT COUNT(1) \\\n", + " FROM filings f \\\n", + " WHERE f.business_id = b.id \\\n", + " AND f.filing_json->'filing'->'header'->>'name' = 'registrarsNotation' \\\n", + " AND f.filing_json->'filing'->'registrarsNotation'->>'orderDetails' = 'Early AR Filed') <> 0 AS has_early_ar_filed \\\n", + " FROM businesses b \\\n", + " WHERE b.identifier = :identifier\n", + "\n", + " state = None\n", + " legal_type = None\n", + " has_early_ar_filed = None\n", + "\n", + " if business_details:\n", + " row = business_details[0]\n", + " state = row._mapping['state']\n", + " legal_type = row._mapping['legal_type']\n", + " has_early_ar_filed = row._mapping['has_early_ar_filed']\n", + "\n", + " if has_early_ar_filed:\n", + " duplicate_identifiers.append(identifier)\n", + " continue\n", + "\n", + " filing_data = {\n", + " \"filing\": {\n", + " \"header\": {\n", + " \"name\": \"registrarsNotation\",\n", + " \"date\": current_date,\n", + " \"certifiedBy\": \"system\"\n", + " },\n", + " \"business\": {\n", + " \"identifier\": identifier,\n", + " \"legalType\": legal_type\n", + " },\n", + " \"registrarsNotation\": {\n", + " \"orderDetails\": \"Early AR Filed\"\n", + " }\n", + " }\n", + " }\n", + "\n", + " filing_url = urljoin(base_url, f\"/api/v2/businesses/{identifier}/filings\")\n", + " response = requests.post(filing_url, headers=headers, json=filing_data)\n", + "\n", + " # Check the status code of the response\n", + " if response.status_code == 201:\n", + " successful_identifiers.append(identifier)\n", + " else:\n", + " failed_identifiers.append(identifier)\n", + " print(f\"Failed to make POST request. Status code: {response.status_code} for {identifier}\")\n", + "\n", + "print('Successfully filed Registrar Notation for:', successful_identifiers)\n", + "print('Failed to file Registrar Notation for:', failed_identifiers)\n", + "print('Already had Early AR Filed Registrar Notation:', duplicate_identifiers)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv (3.9.18.final.0)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/support/notebooks/business-ops/correction-early-ar/rn_early_ar_output.py b/support/notebooks/business-ops/correction-early-ar/rn_early_ar_output.py new file mode 100644 index 0000000000..0d12cedfa5 --- /dev/null +++ b/support/notebooks/business-ops/correction-early-ar/rn_early_ar_output.py @@ -0,0 +1,3 @@ +businesses = [ + 'BC1XXXXXX' +] diff --git a/support/notebooks/business-ops/requirements.txt b/support/notebooks/business-ops/requirements.txt index 56a5f0bf4c..f2e92c1683 100644 --- a/support/notebooks/business-ops/requirements.txt +++ b/support/notebooks/business-ops/requirements.txt @@ -12,6 +12,7 @@ ipython-sql tlaplus_jupyter ipytest pytest +python-dotenv git+https://github.com/bcgov/business-schemas.git#egg=registry_schemas git+https://github.com/bcgov/lear.git#egg=legal_api&subdirectory=legal-api