Skip to content

Commit 7fa3b96

Browse files
BibianaCBibianaC
committed
additional_data: sources: refactor add licence info
1 parent fda34e0 commit 7fa3b96

9 files changed

Lines changed: 102 additions & 37 deletions

‎datastore/additional_data/generator.py‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,21 @@ def create(self, grant, source_file, additional_data_sources=DATA_SOURCES):
4848

4949
for additional_data_source in additional_data_sources:
5050
try:
51-
getattr(self, additional_data_source).update_additional_data(
52-
grant, source_file, additional_data
53-
)
51+
source_instance = getattr(self, additional_data_source)
52+
if additional_data_source == "grant_metadata":
53+
# Build sources dict excluding grant_metadata itself
54+
sources_dict = {
55+
key: getattr(self, key)
56+
for key in additional_data_sources
57+
if key != "grant_metadata"
58+
}
59+
source_instance.update_additional_data(
60+
grant, source_file, additional_data, sources=sources_dict
61+
)
62+
else:
63+
source_instance.update_additional_data(
64+
grant, source_file, additional_data
65+
)
5466
except AttributeError:
5567
raise Exception(
5668
f"Data source {additional_data_source} is not a known additional data source."

‎datastore/additional_data/sources/codelist_code.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,5 +139,3 @@ def update_additional_data(self, grant, source_file, additional_data):
139139
"recipientOrg_location_countryCode": recipientOrg_location_countryCode,
140140
"fundingOrg_location_countryCode": fundingOrg_location_countryCode,
141141
}
142-
143-
additional_data[f"{self.ADDITIONAL_DATA_KEY}_LICENCE"] = self.LICENCE

‎datastore/additional_data/sources/find_that_charity.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,6 @@ def __init__(self, *args, **kwargs):
8686
self._cache = {}
8787

8888
def update_additional_data(self, grant, source_file, additional_data):
89-
additional_data[f"{self.ADDITIONAL_DATA_KEY}_LICENCE"] = self.LICENCE
90-
9189
# We can't do anything if this grant doesn't have a recipientOrganization
9290
if not grant.get("recipientOrganization"):
9391
return

‎datastore/additional_data/sources/geo_lookup.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -148,5 +148,3 @@ def update_additional_data(self, grant, source_file, additional_data):
148148
area["source"] = "recipientOrganizationPostcode"
149149
area["sourceCode"] = lsoa
150150
additional_data["locationLookup"].append(area)
151-
152-
additional_data[f"{self.ADDITIONAL_DATA_KEY}_LICENCE"] = self.LICENCE

‎datastore/additional_data/sources/grant_metadata.py‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,12 @@ class GrantMetadataSource(object):
22
"""Adds metadata to a grant:
33
* metadata/source_license
44
* metadata/source_license_name
5+
* metadata/sources_metadata
56
"""
67

78
ADDITIONAL_DATA_KEY = "metadata"
89

9-
def update_additional_data(self, grant, source_file, additional_data):
10+
def update_additional_data(self, grant, source_file, additional_data, sources=None):
1011

1112
additional_data[self.ADDITIONAL_DATA_KEY] = {}
1213

@@ -20,3 +21,19 @@ def update_additional_data(self, grant, source_file, additional_data):
2021
additional_data[self.ADDITIONAL_DATA_KEY][
2122
"source_license"
2223
] = source_file.get("license")
24+
25+
# Aggregate licenses from all additional_data sources
26+
sources_metadata = {}
27+
if sources:
28+
for source_name, source_instance in sources.items():
29+
if hasattr(source_instance, "LICENCE") and hasattr(
30+
source_instance, "ADDITIONAL_DATA_KEY"
31+
):
32+
sources_metadata[source_instance.ADDITIONAL_DATA_KEY] = {
33+
"license": source_instance.LICENCE
34+
}
35+
36+
if sources_metadata:
37+
additional_data[self.ADDITIONAL_DATA_KEY][
38+
"sources_metadata"
39+
] = sources_metadata

‎datastore/additional_data/sources/nspl.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -208,5 +208,3 @@ def update_additional_data(self, grant, source_file, additional_data):
208208
self.update_location_data_code_names(location_data)
209209
additional_data["recipientOrganizationLocation"] = location_data
210210
break
211-
212-
additional_data[f"{self.ADDITIONAL_DATA_KEY}_LICENCE"] = self.LICENCE

‎datastore/tests/test_additional_data_codelist_code.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,7 @@ def test_code_list(self):
3737
"beneficiaryLocation_countryCode": "Australia",
3838
"fundingOrg_location_countryCode": "France",
3939
"recipientOrg_location_countryCode": "United Kingdom of Great Britain and Northern Ireland",
40-
},
41-
"codeListLookup_LICENCE": "https://creativecommons.org/licenses/by/4.0/",
40+
}
4241
}
4342

4443
source.update_additional_data(grant, source_file, additional_data_in)

‎datastore/tests/test_additional_data_license.py‎

Lines changed: 66 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -47,35 +47,82 @@ def test_publisher_license(self):
4747
class TestAdditionalDataSourceLicences(TestCase):
4848
fixtures = ["test_data.json"]
4949

50-
def _assert_licence(self, source, expected_licence):
50+
def test_source_licenses_aggregation(self):
51+
"""Test that GrantMetadataSource aggregates licenses from all sources"""
5152
grant = Grant.objects.first()
5253
additional_data = {}
5354

54-
source.update_additional_data(
55-
grant.data, grant.source_file.data, additional_data
56-
)
55+
# Create source instances
56+
sources = {
57+
"find_that_charity_source": FindThatCharitySource(),
58+
"geo_lookup": GeoLookupSource(),
59+
"nspl_source": NSPLSource(),
60+
"code_lists": CodeListSource(),
61+
}
5762

58-
licence_key = f"{source.ADDITIONAL_DATA_KEY}_LICENCE"
63+
grant_metadata_source = GrantMetadataSource()
64+
65+
# Call with sources parameter
66+
grant_metadata_source.update_additional_data(
67+
grant.data, grant.source_file.data, additional_data, sources=sources
68+
)
5969

70+
# Verify the structure
6071
self.assertIn(
61-
licence_key,
72+
"metadata",
6273
additional_data,
63-
f"{licence_key} was not added by {source.__class__.__name__}.",
74+
"metadata key was not added.",
6475
)
65-
self.assertEqual(
66-
additional_data[licence_key],
67-
expected_licence,
68-
f"{source.__class__.__name__} set the wrong licence value.",
76+
77+
self.assertIn(
78+
"sources_metadata",
79+
additional_data["metadata"],
80+
"sources_metadata was not aggregated by GrantMetadataSource.",
6981
)
7082

71-
def test_find_that_charity_licence(self):
72-
self._assert_licence(FindThatCharitySource(), OGL_V3)
83+
sources_metadata = additional_data["metadata"]["sources_metadata"]
7384

74-
def test_geo_lookup_licence(self):
75-
self._assert_licence(GeoLookupSource(), OGL_V3)
85+
# Verify each source's license is present
86+
self.assertIn(
87+
"recipientOrgInfos",
88+
sources_metadata,
89+
"FindThatCharitySource license not aggregated.",
90+
)
91+
self.assertEqual(
92+
sources_metadata["recipientOrgInfos"]["license"],
93+
OGL_V3,
94+
"FindThatCharitySource has wrong license.",
95+
)
7696

77-
def test_nspl_licence(self):
78-
self._assert_licence(NSPLSource(), OGL_V3)
97+
self.assertIn(
98+
"locationLookup",
99+
sources_metadata,
100+
"GeoLookupSource license not aggregated.",
101+
)
102+
self.assertEqual(
103+
sources_metadata["locationLookup"]["license"],
104+
OGL_V3,
105+
"GeoLookupSource has wrong license.",
106+
)
79107

80-
def test_codelist_lookup_licence(self):
81-
self._assert_licence(CodeListSource(), CC_BY_4)
108+
self.assertIn(
109+
"recipientOrganizationLocation",
110+
sources_metadata,
111+
"NSPLSource license not aggregated.",
112+
)
113+
self.assertEqual(
114+
sources_metadata["recipientOrganizationLocation"]["license"],
115+
OGL_V3,
116+
"NSPLSource has wrong license.",
117+
)
118+
119+
self.assertIn(
120+
"codeListLookup",
121+
sources_metadata,
122+
"CodeListSource license not aggregated.",
123+
)
124+
self.assertEqual(
125+
sources_metadata["codeListLookup"]["license"],
126+
CC_BY_4,
127+
"CodeListSource has wrong license.",
128+
)

‎datastore/tests/test_additional_data_nspl.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -200,8 +200,7 @@ def test_nspl_update_additional_data_with_not_existing_postcode(self):
200200
nspl.update_additional_data(grant.data, grant.source_file.data, additional_data)
201201

202202
self.assertNotIn("recipientOrganizationLocation", additional_data)
203-
self.assertEqual(len(additional_data), 1)
204-
self.assertIn("recipientOrganizationLocation_LICENCE", additional_data)
203+
self.assertEqual(len(additional_data), 0)
205204

206205
def test_nspl_update_additional_data_without_postcode(self):
207206
self.save_nspl_mock_data()
@@ -215,8 +214,7 @@ def test_nspl_update_additional_data_without_postcode(self):
215214
nspl.update_additional_data(grant.data, grant.source_file.data, additional_data)
216215

217216
self.assertNotIn("recipientOrganizationLocation", additional_data)
218-
self.assertEqual(len(additional_data), 1)
219-
self.assertIn("recipientOrganizationLocation_LICENCE", additional_data)
217+
self.assertEqual(len(additional_data), 0)
220218

221219
def test_nspl_update_additional_data_breaks_after_updating(self):
222220
# Once we have one recipient org location, the loop should break.

0 commit comments

Comments
 (0)