diff --git a/funidata_utils/data_scramblers/generic_sis_scramble.py b/funidata_utils/data_scramblers/generic_sis_scramble.py index 311b3c3..00e6cff 100644 --- a/funidata_utils/data_scramblers/generic_sis_scramble.py +++ b/funidata_utils/data_scramblers/generic_sis_scramble.py @@ -1,4 +1,5 @@ import sys +from functools import lru_cache if sys.version_info >= (3, 12): @@ -31,14 +32,27 @@ class SisMetadataScrambler(SingletonMetaScrambler): ) -class UnprocessedKeysDropperScrambler(SingletonMetaScrambler): +class UnprocessedKeysDropperScrambler: scrambling_keys = {} def __init__(self, loggable_keys: set[str]): self._loggable_keys = loggable_keys + @classmethod + @lru_cache + def keys_without_scrambling(cls): + return {k for k, v in getattr(cls, 'scrambling_keys', {}).items() if not v} + + @classmethod + @lru_cache + def dict_items_with_scrambling(cls): + return { + k: v + for k, v in getattr(cls, 'scrambling_keys', {}).items() + if v + }.items() + # Instance method override to provide access to self._loggable_keys, and also provide different scrambling logic from other classes - @override def scramble(self, entity: dict, processed_keys: set) -> dict: # noqa: instance method signature does not match cls method """ Pops any keys that are not present in processed_keys """ entity_keys = set(entity.keys()) diff --git a/funidata_utils/data_scramblers/kori_person.py b/funidata_utils/data_scramblers/kori_person.py index dadbd50..bc4eaaa 100644 --- a/funidata_utils/data_scramblers/kori_person.py +++ b/funidata_utils/data_scramblers/kori_person.py @@ -12,7 +12,7 @@ class KoriPersonScrambler(SingletonMetaScrambler): documentState=None, universityOrgIds=None, titles=[ - lambda x: {'fi': 'titteli', 'en': 'title', 'sv': 'titel'} + lambda x: [{'fi': 'titteli', 'en': 'title', 'sv': 'titel'}] ], firstName=[ ( diff --git a/funidata_utils/data_scramblers/study_right.py b/funidata_utils/data_scramblers/study_right.py index 85957ed..7f3a1be 100644 --- a/funidata_utils/data_scramblers/study_right.py +++ b/funidata_utils/data_scramblers/study_right.py @@ -36,14 +36,16 @@ class StudyRightScrambler(SingletonMetaScrambler): lambda x: None ], studyRightExtensions=[ - lambda x: [ - # Override grant + delete reasons with lorem ipsum, otherwise retain original extensions - original_extension | dict( - grantReason="Extension grant Raisin: Lorem Ipsum" if original_extension['grantReason'] else None, - deleteReason="Extension Delete Raisin: Lorem Ipsum" if original_extension['deleteReason'] else None - ) - for original_extension in x.get('studyRightExtensions', []) or [] - ] + lambda original_extension: update_inner_dictionary_key( + original_extension, + 'studyRightExtensions.grantReason', + 'studyRightExtensions.grantReason: Lorem Ipsum' + ), + lambda original_extension: update_inner_dictionary_key( + original_extension, + 'studyRightExtensions.deleteReason', + 'studyRightExtensions.deleteReason: Lorem Ipsum' + ) ], studyRightCancellation=[ ( @@ -55,22 +57,11 @@ class StudyRightScrambler(SingletonMetaScrambler): ), ], studyRightPassivations=[ - lambda x: [ - # Override grant + delete reasons with lorem ipsum, otherwise retain original extensions - lambda original_val: update_inner_dictionary_key( - original_passivation, - 'additionalInfo', - 'studyRightPassivations.additionalInfo: Lorem Ipsum' - ) - for original_passivation in x.get('studyRightPassivations', []) or [] - ] - ], - studyRightTermination=[ - lambda original_val: update_inner_dictionary_key( - original_val, - 'studyRightTermination.terminationReason', - 'studyRightTermination.terminationReason: Lorem Ipsum' - ), + lambda original_passivation: update_inner_dictionary_key( + original_passivation, + 'studyRightPassivations.additionalInfo', + 'studyRightPassivations.additionalInfo: Lorem Ipsum' + ) ], studyRightGraduation=None, acceptedSelectionPath=None, diff --git a/funidata_utils/request_utils/async_httpx_requests.py b/funidata_utils/request_utils/async_httpx_requests.py index aad81ba..efae1e2 100644 --- a/funidata_utils/request_utils/async_httpx_requests.py +++ b/funidata_utils/request_utils/async_httpx_requests.py @@ -9,6 +9,7 @@ from typing import Tuple, Any, Callable, Literal import httpx +from httpx import AsyncClient, AsyncHTTPTransport from ..utils import flatten, group_by, batch @@ -20,12 +21,15 @@ def _collect_suitable_batches_grouped_by_key( items_by_key: dict[Any, list[dict]], sorting_function: Callable = None, - batch_size_trigger: int = 500, + batch_size_trigger: int | None = 500, ) -> list[list[list[dict]]]: batches: list[list[list[dict]]] = [] current_batch = [] current_batch_size = 0 + if not batch_size_trigger: + batch_size_trigger = 500 + sendable_lists_of_items = list(items_by_key.values()) for index, values in enumerate(sendable_lists_of_items): @@ -54,14 +58,15 @@ async def send_get_httpx( "https://": httpx.AsyncHTTPTransport(proxy=proxies.get('https')), } if proxies else None client = httpx.AsyncClient(mounts=proxy_mounts, auth=auth) - response = await client.get( - path, - auth=auth, - params=params, - timeout=600, - follow_redirects=allow_redirects, - ) - return response + async with client: + response = await client.get( + path, + auth=auth, + params=params, + timeout=600, + follow_redirects=allow_redirects, + ) + return response async def _binary_search_enabled_post_httpx( @@ -232,6 +237,9 @@ async def send_post_with_binary_err_search_httpx( binary_err_search_sublists: bool = True, method: Literal['POST', 'PATCH'] = 'POST', max_parallel_requests: int = 1, + _state: dict[ + Literal['max_seen_depth', 'sent_requests'], int + ] | None = None, ) -> list[httpx.Response]: if len(payload) <= 0: raise Exception(f"Payload missing when attempting to POST to : {path}") @@ -243,35 +251,70 @@ async def send_post_with_binary_err_search_httpx( "https://": httpx.AsyncHTTPTransport(proxy=proxies.get('https')), } - client = httpx.AsyncClient(mounts=proxy_mounts, auth=auth) + client = _get_async_httpx_client(auth, proxy_mounts) semaphore = asyncio.Semaphore(max_parallel_requests) async def _with_sem(coro: typing.Coroutine): async with semaphore: return await coro - if group_by_key: - items_by_key = group_by(payload, lambda x: x[group_by_key]) - batches = _collect_suitable_batches_grouped_by_key( - items_by_key=items_by_key, - sorting_function=None, - batch_size_trigger=batch_size, - ) - """ - Creates a structure that contains the grouped data as lists of the original groups - that then reside in lists approximately of the size batch_size - Could be useful for example for grouping attainments of persons, so that the original context - of which attainments belong to which person can be separately sent in one batch. - [ - [ [1], [2,3] ], - [ [4,5,6] ], - [ [7,8], [10,11,12,13,14] ], - ] - """ + async with client: + if group_by_key: + items_by_key = group_by(payload, lambda x: x[group_by_key]) + batches = _collect_suitable_batches_grouped_by_key( + items_by_key=items_by_key, + sorting_function=None, + batch_size_trigger=batch_size, + ) + """ + Creates a structure that contains the grouped data as lists of the original groups + that then reside in lists approximately of the size batch_size + Could be useful for example for grouping attainments of persons, so that the original context + of which attainments belong to which person can be separately sent in one batch. + [ + [ [1], [2,3] ], + [ [4,5,6] ], + [ [7,8], [10,11,12,13,14] ], + ] + """ + _tasks = [ + _with_sem(_binary_search_enabled_post_httpx( + path=path, + payload=_batch, + params=params, + auth=auth, + client=client, + binary_search_depth=0, + binary_search_max_depth=binary_search_max_depth, + binary_err_search_sublists=binary_err_search_sublists, + method=method, + _state=_state, + )) + for _batch in batches + ] + results = await asyncio.gather(*_tasks) + return flatten(results) + + # Is not group_by'ed -> If batch size is not configured, try sending everything + if not batch_size: + return await _binary_search_enabled_post_httpx( + path=path, + payload=payload, + params=params, + auth=auth, + client=client, + binary_search_depth=0, + binary_search_max_depth=binary_search_max_depth, + binary_err_search_sublists=binary_err_search_sublists, + method=method, + _state=_state, + ) + + # When batch size is configured, batch the payloads _tasks = [ _with_sem(_binary_search_enabled_post_httpx( path=path, - payload=_batch, + payload=batched_payload, params=params, auth=auth, client=client, @@ -279,40 +322,14 @@ async def _with_sem(coro: typing.Coroutine): binary_search_max_depth=binary_search_max_depth, binary_err_search_sublists=binary_err_search_sublists, method=method, + _state=_state, )) - for _batch in batches + for batched_payload in batch(payload, batch_size) ] results = await asyncio.gather(*_tasks) return flatten(results) - # Is not group_by'ed -> If batch size is not configured, try sending everything - if not batch_size: - return await _binary_search_enabled_post_httpx( - path=path, - payload=payload, - params=params, - auth=auth, - client=client, - binary_search_depth=0, - binary_search_max_depth=binary_search_max_depth, - binary_err_search_sublists=binary_err_search_sublists, - method=method, - ) - # When batch size is configured, batch the payloads - _tasks = [ - _with_sem(_binary_search_enabled_post_httpx( - path=path, - payload=batched_payload, - params=params, - auth=auth, - client=client, - binary_search_depth=0, - binary_search_max_depth=binary_search_max_depth, - binary_err_search_sublists=binary_err_search_sublists, - method=method, - )) - for batched_payload in batch(payload, batch_size) - ] - results = await asyncio.gather(*_tasks) - return flatten(results) +def _get_async_httpx_client(auth: tuple[str, str] | None, proxy_mounts: dict[str, AsyncHTTPTransport] | None) -> AsyncClient: + client = httpx.AsyncClient(mounts=proxy_mounts, auth=auth) + return client diff --git a/funidata_utils/request_utils/httpx_requests.py b/funidata_utils/request_utils/httpx_requests.py index 337afcd..06ecd37 100644 --- a/funidata_utils/request_utils/httpx_requests.py +++ b/funidata_utils/request_utils/httpx_requests.py @@ -1,13 +1,13 @@ # Copyright (c) 2025 Funidata Oy. # All rights reserved. # ------------------------------------------------------------------------------ - +import asyncio from typing import Tuple, Any, Callable, Literal import httpx from httpx import Client -from ..utils import flatten, group_by +from . import async_httpx_requests ACCEPTED_RESPONSE_CODES = {200, 201, 202, 204} @@ -51,13 +51,14 @@ def send_get_httpx( params: dict | None = None, ) -> httpx.Response: client = _get_httpx_client(auth, proxies) - response = client.get( - path, - auth=auth, - params=params, - timeout=600, - ) - return response + with client: + response = client.get( + path, + auth=auth, + params=params, + timeout=600, + ) + return response def _get_httpx_client(auth: tuple[str, str] | None, proxies: dict[Any, Any] | None) -> Client: @@ -69,95 +70,6 @@ def _get_httpx_client(auth: tuple[str, str] | None, proxies: dict[Any, Any] | No return client -def _binary_search_enabled_post_httpx( - path: str, - payload: list[dict] | list[list[dict]], - client: httpx.Client, - auth: Tuple[str, str] | None = None, - params: dict | None = None, - binary_search_depth: int = 0, - binary_search_max_depth: int | None = None, - binary_err_search_sublists: bool = True, - method: Literal['POST', 'PATCH'] = 'POST', -) -> list[httpx.Response]: - is_complex_list_of_batches = False - if isinstance(payload, list) and all(isinstance(x, list) for x in payload[::3]): - is_complex_list_of_batches = True - - match method: - case 'POST': - response = client.post( - path, - auth=auth, - json=flatten(payload) if is_complex_list_of_batches else payload, - params=params, - timeout=60, - ) - - case 'PATCH': - response = client.patch( - path, - auth=auth, - json=flatten(payload) if is_complex_list_of_batches else payload, - params=params, - timeout=60, - ) - - case _: - raise Exception(f'Unsupported method: {method}') - - if ( - binary_search_max_depth == 0 or - (binary_search_max_depth and binary_search_depth >= binary_search_max_depth) - or response.status_code in ACCEPTED_RESPONSE_CODES - ): - return [response] - - if not is_complex_list_of_batches: - if len(payload) <= 1: - return [response] - else: - if len(payload) <= 1 and not binary_err_search_sublists: - return [response] - - if len(payload) == 1 and binary_err_search_sublists: - return _binary_search_enabled_post_httpx( - path=path, - payload=flatten(payload), - auth=auth, - params=params, - client=client, - binary_search_depth=binary_search_depth + 1, - binary_search_max_depth=binary_search_max_depth, - binary_err_search_sublists=False, - method=method, - ) - - first_half_responses = _binary_search_enabled_post_httpx( - path=path, - payload=payload[::2], - auth=auth, - params=params, - client=client, - binary_search_depth=binary_search_depth + 1, - binary_search_max_depth=binary_search_max_depth, - binary_err_search_sublists=binary_err_search_sublists, - method=method, - ) - second_half_responses = _binary_search_enabled_post_httpx( - path=path, - payload=payload[1::2], - auth=auth, - params=params, - client=client, - binary_search_depth=binary_search_depth + 1, - binary_search_max_depth=binary_search_max_depth, - binary_err_search_sublists=binary_err_search_sublists, - method=method, - ) - return first_half_responses + second_half_responses - - def send_post_with_binary_err_search_httpx( path: str, payload: list[dict], @@ -169,82 +81,21 @@ def send_post_with_binary_err_search_httpx( binary_search_max_depth: int | None = None, binary_err_search_sublists: bool = True, method: Literal['POST', 'PATCH'] = 'POST', -) -> list[httpx.Response]: - if len(payload) <= 0: - raise Exception(f"Payload missing when attempting to POST to : {path}") - - proxy_mounts = None - if proxies: - proxy_mounts = { - "http://": httpx.HTTPTransport(proxy=proxies.get('http')), - "https://": httpx.HTTPTransport(proxy=proxies.get('https')), - } - - client = httpx.Client(mounts=proxy_mounts, auth=auth) - - if group_by_key: - items_by_key = group_by(payload, lambda x: x[group_by_key]) - batches = _collect_suitable_batches_grouped_by_key( - items_by_key=items_by_key, - sorting_function=None, - batch_size_trigger=batch_size, - ) - """ - Creates a structure that contains the grouped data as lists of the original groups - that then reside in lists approximately of the size batch_size - Could be useful for example for grouping attainments of persons, so that the original context - of which attainments belong to which person can be separately sent in one batch. - [ - [ [1], [2,3] ], - [ [4,5,6] ], - [ [7,8], [10,11,12,13,14] ], - ] - """ - responses = [] - - for _batch in batches: - batch_responses = _binary_search_enabled_post_httpx( - path=path, - payload=_batch, - params=params, - auth=auth, - client=client, - binary_search_depth=0, - binary_search_max_depth=binary_search_max_depth, - binary_err_search_sublists=binary_err_search_sublists, - method=method, - ) - responses += batch_responses - return responses - - # Is not group_by'ed -> If batch size is not configured, try sending everything - if not batch_size: - return _binary_search_enabled_post_httpx( + _state: dict[ + Literal['max_seen_depth', 'sent_requests'], int + ] | None = None, ) -> list[httpx.Response]: + return asyncio.run( + async_httpx_requests.send_post_with_binary_err_search_httpx( path=path, payload=payload, - params=params, + group_by_key=group_by_key, auth=auth, - client=client, - binary_search_depth=0, - binary_search_max_depth=binary_search_max_depth, - binary_err_search_sublists=binary_err_search_sublists, - method=method, - ) - - # When batch size is configured, batch the payload - responses = [] - for batched_payload in batch(payload, batch_size): - batch_responses = _binary_search_enabled_post_httpx( - path=path, - payload=batched_payload, + proxies=proxies, params=params, - auth=auth, - client=client, - binary_search_depth=0, + batch_size=batch_size, binary_search_max_depth=binary_search_max_depth, binary_err_search_sublists=binary_err_search_sublists, method=method, + _state=_state, ) - responses += batch_responses - - return responses + ) diff --git a/funidata_utils/schemas/sisu/public_person.py b/funidata_utils/schemas/sisu/public_person.py index f7ead5c..d26999f 100644 --- a/funidata_utils/schemas/sisu/public_person.py +++ b/funidata_utils/schemas/sisu/public_person.py @@ -9,7 +9,7 @@ class PublicPerson(SisBase): id: OTM_ID_REGEX_VALIDATED_STR universityOrgIds: conlist(OTM_ID_REGEX_VALIDATED_STR, max_length=SIS_MAX_SMALL_SET_SIZE) - titles: LocalizedString | None = None + titles: list[LocalizedString] | None = None firstName: str | None = None lastName: str | None = None emailAddress: str | None = None diff --git a/funidata_utils/schemas/sisu/study_right.py b/funidata_utils/schemas/sisu/study_right.py index e2036d4..a3b175c 100644 --- a/funidata_utils/schemas/sisu/study_right.py +++ b/funidata_utils/schemas/sisu/study_right.py @@ -14,6 +14,7 @@ PastDate, ) +from .study_right_cooperation_network_status import CooperationNetworkStatus from .common import ( SIS_MAX_LONG_STRING_LENGTH, OTM_ID_REGEX_PATTERN, @@ -95,6 +96,7 @@ class StudyRightCancellation(BaseModel): cancellationType: Literal[ 'RESCINDED', 'CANCELLED_BY_ADMINISTRATION', + 'TERMINATED', ] @field_serializer('cancellationDate') @@ -193,8 +195,8 @@ class StudyRight(BaseModel): codeUrns: list[CodeUrnsStr] additionalInformation: Optional[dict] = None # basedOnEnrolmentRights: bool < Apparently removed from sisu model at some point in time - cooperationNetworkRights: Optional[str] = None - cooperationNetworkStatus: Optional[dict] = None + cooperationNetworkRights: Optional[list[dict]] = None + cooperationNetworkStatus: Optional[CooperationNetworkStatus] = None schoolEducationLanguageUrn: Annotated[STRIPPED_STR | None, Field(pattern=sis_code_urn_pattern('school-education-language'))] = None @field_serializer('snapshotDateTime') diff --git a/funidata_utils/sis_integration/exports.py b/funidata_utils/sis_integration/exports.py index 33d9d82..c3754c0 100644 --- a/funidata_utils/sis_integration/exports.py +++ b/funidata_utils/sis_integration/exports.py @@ -95,6 +95,14 @@ def export_from_endpoint_generator( params = {} scrambling_warning_triggered_keys = set() + _scrambling_classes = None + if scrambling_classes: + _scrambling_classes = [ + *scrambling_classes, + SisMetadataScrambler, + UnprocessedKeysDropperScrambler(scrambling_warning_triggered_keys) + ] + while True: sis_response = send_get_httpx( path=f"{sis_settings.host}{endpoint}", @@ -105,13 +113,10 @@ def export_from_endpoint_generator( if sis_response.status_code == 200: response_json = sis_response.json() entities: list[dict] = response_json.get("entities", []) - if scrambling_classes: - scrambling_classes.append(SisMetadataScrambler) - scrambling_classes.append(UnprocessedKeysDropperScrambler(scrambling_warning_triggered_keys)) - + if _scrambling_classes: for entity in entities: processed_keys = set() - for scrambling_class in scrambling_classes: + for scrambling_class in _scrambling_classes: entity = scrambling_class.scramble(entity, processed_keys) yield entities diff --git a/funidata_utils/utils.py b/funidata_utils/utils.py index 58dee1c..814518d 100644 --- a/funidata_utils/utils.py +++ b/funidata_utils/utils.py @@ -87,7 +87,7 @@ def recursive_dict_fetch( if keys[0] not in entity: match missing_key_handler: case 'skip': - return entity + return None case 'exception' | _: raise KeyError(f"Could not find {keys[0]} in entity") @@ -142,7 +142,7 @@ def _dict_update_by_key_split( case 'exception': raise ValueError(f"Expected nested dictionaries, {_current_entity_ref} is not a dict") case 'skip': - return entity + return None case _: raise Exception("Unhandled case for missing inner dict key") diff --git a/tests/scramblers/__init__.py b/tests/scramblers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scramblers/ilmo/__init__.py b/tests/scramblers/ilmo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scramblers/ilmo/test_enrolment_scrambler.py b/tests/scramblers/ilmo/test_enrolment_scrambler.py new file mode 100644 index 0000000..f17aaf5 --- /dev/null +++ b/tests/scramblers/ilmo/test_enrolment_scrambler.py @@ -0,0 +1,162 @@ +import copy + +import pytest + +from funidata_utils.data_scramblers.ilmo_enrolment import IlmoEnrolmentScrambler + + +@pytest.mark.unit +def test_enrolment_scrambler_scrambles_keys(): + data = { + "documentState": "DRAFT", + "id": "otm-123456", + "personId": "otm-123456", + "courseUnitRealisationId": "otm-123456", + "courseUnitId": "otm-123456", + "assessmentItemId": "otm-123456", + "studyRightId": "otm-123456", + "openUniversityCartId": "otm-123456", + "openUniversityCartItemId": "otm-123456", + "state": "NOT_ENROLLED", + "processingState": "NOT_PROCESSED", + "studySubGroups": [ + { + "studySubGroupId": "otm-123456", + "enrolmentStudySubGroupPriority": "PRIMARY", + "isInCalendar": True + } + ], + "studyGroupSets": [ + { + "studyGroupSetId": "otm-123456", + "targetStudySubGroupAmount": 0 + } + ], + "confirmedStudySubGroupIds": [ + "otm-123456" + ], + "tentativeStudySubGroupIds": [ + "otm-123456" + ], + "enrolmentDateTime": "2026-09-21T07:45:37.841Z", + "isInCalendar": True, + "colorIndex": 0, + "quotaIds": [ + "otm-123456" + ], + "activeQuotaId": "otm-123456", + "allocatedQuotaId": "otm-123456", + "maximumQuotaIds": [ + "otm-123456" + ], + "enrolmentRightId": "otm-123456", + "replacedByEnrolmentId": "otm-123456", + "cooperationNetworkStatus": { + "direction": "INBOUND", + "organisationTkCode": "string", + "outboundStatus": "NOT_VALID", + "rejectionReason": { + "fi": "Finnish version", + "sv": "Swedish version", + "en": "English version" + }, + "outboundStatusMessage": "string", + "cooperationNetworkId": "otm-123456", + "universityOrgId": "otm-123456", + "homeStudyRightId": "string", + "selectionItemStatusInfo": "string" + }, + "studentConsentForOutboundDataTransfer": { + "consentApprovalType": "APPROVED", + "consentTargetType": "CROSS_STUDY_DATA_TRANSFER_ON_OUTBOUND_ENROLMENT", + "clause": "string" + } + } + + scrambled_data = IlmoEnrolmentScrambler.scramble(copy.deepcopy(data), set()) + + assert len(IlmoEnrolmentScrambler.dict_items_with_scrambling()) >= 2 + for key, _ in IlmoEnrolmentScrambler.dict_items_with_scrambling(): + if isinstance(scrambled_data.get(key), bool): + # Cannot reliably test for bool 50/50 changes + continue + + assert scrambled_data.get(key, -42) != data.get(key, -42) + + +@pytest.mark.unit +def test_enrolment_scrambler_handles_lists_correctly(): + data = { + "documentState": "DRAFT", + "id": "otm-123456", + "personId": "otm-123456", + "courseUnitRealisationId": "otm-123456", + "courseUnitId": "otm-123456", + "assessmentItemId": "otm-123456", + "studyRightId": "otm-123456", + "openUniversityCartId": "otm-123456", + "openUniversityCartItemId": "otm-123456", + "state": "NOT_ENROLLED", + "processingState": "NOT_PROCESSED", + "studySubGroups": [ + { + "studySubGroupId": "otm-123456", + "enrolmentStudySubGroupPriority": "PRIMARY", + "isInCalendar": True + } + ], + "studyGroupSets": [ + { + "studyGroupSetId": "otm-123456", + "targetStudySubGroupAmount": 0 + } + ], + "confirmedStudySubGroupIds": [ + "otm-123456" + ], + "tentativeStudySubGroupIds": [ + "otm-123456" + ], + "enrolmentDateTime": "2026-09-21T07:45:37.841Z", + "isInCalendar": True, + "colorIndex": 0, + "quotaIds": [ + "otm-123456" + ], + "activeQuotaId": "otm-123456", + "allocatedQuotaId": "otm-123456", + "maximumQuotaIds": [ + "otm-123456" + ], + "enrolmentRightId": "otm-123456", + "replacedByEnrolmentId": "otm-123456", + "cooperationNetworkStatus": { + "direction": "INBOUND", + "organisationTkCode": "string", + "outboundStatus": "NOT_VALID", + "rejectionReason": { + "fi": "Finnish version", + "sv": "Swedish version", + "en": "English version" + }, + "outboundStatusMessage": "string", + "cooperationNetworkId": "otm-123456", + "universityOrgId": "otm-123456", + "homeStudyRightId": "string", + "selectionItemStatusInfo": "string" + }, + "studentConsentForOutboundDataTransfer": { + "consentApprovalType": "APPROVED", + "consentTargetType": "CROSS_STUDY_DATA_TRANSFER_ON_OUTBOUND_ENROLMENT", + "clause": "string" + } + } + + scrambled_data = IlmoEnrolmentScrambler.scramble(copy.deepcopy(data), set()) + list_keys = { + 'maximumQuotaIds', 'quotaIds', + 'tentativeStudySubGroupIds', 'confirmedStudySubGroupIds', + 'studyGroupSets', 'studySubGroups', + } + for key in list_keys: + assert isinstance(scrambled_data[key], list), scrambled_data[key] diff --git a/tests/scramblers/kori/__init__.py b/tests/scramblers/kori/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scramblers/kori/test_kori_person_scrambler.py b/tests/scramblers/kori/test_kori_person_scrambler.py new file mode 100644 index 0000000..aa92535 --- /dev/null +++ b/tests/scramblers/kori/test_kori_person_scrambler.py @@ -0,0 +1,68 @@ +import copy + +import pytest + +from funidata_utils.data_scramblers.kori_person import KoriPersonScrambler +from funidata_utils.schemas.sisu import PublicPerson + + +@pytest.mark.unit +def test_public_person_scrambler_scrambles_keys(): + data = { + "documentState": "DRAFT", + "id": "otm-123456", + "universityOrgIds": [ + "otm-123456" + ], + "titles": [ + { + "fi": "Finnish version", + "sv": "Swedish version", + "en": "English version" + } + ], + "firstName": "string", + "lastName": "string", + "emailAddress": "string" + } + + scrambled_data = KoriPersonScrambler.scramble(copy.deepcopy(data), set()) + + assert scrambled_data['emailAddress'] != 'string' + assert len(KoriPersonScrambler.dict_items_with_scrambling()) >= 4 + for key, _ in KoriPersonScrambler.dict_items_with_scrambling(): + if isinstance(scrambled_data.get(key), bool): + # Cannot reliably test for bool 50/50 changes + continue + + assert scrambled_data.get(key, -42) != data.get(key, -42) + + PublicPerson.model_validate(scrambled_data) + + +@pytest.mark.unit +def test_public_person_scrambler_handles_lists_correctly(): + data = { + "documentState": "DRAFT", + "id": "otm-123456", + "universityOrgIds": [ + "otm-123456" + ], + "titles": [ + { + "fi": "Finnish version", + "sv": "Swedish version", + "en": "English version" + } + ], + "firstName": "string", + "lastName": "string", + "emailAddress": "string" + } + + scrambled_data = KoriPersonScrambler.scramble(copy.deepcopy(data), set()) + list_keys = {'titles', 'universityOrgIds'} + for key in list_keys: + assert isinstance(scrambled_data[key], list), scrambled_data[key] + + PublicPerson.model_validate(scrambled_data) diff --git a/tests/scramblers/ori/__init__.py b/tests/scramblers/ori/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scramblers/ori/test_attained_qualification_scrambler.py b/tests/scramblers/ori/test_attained_qualification_scrambler.py new file mode 100644 index 0000000..1b1936b --- /dev/null +++ b/tests/scramblers/ori/test_attained_qualification_scrambler.py @@ -0,0 +1,93 @@ +import copy + +import pytest + +from funidata_utils.data_scramblers.attained_qualification import AttainedQualificationScrambler +from funidata_utils.schemas.sisu import AttainedQualification + + +@pytest.mark.unit +def test_attained_qualification_scrambler_scrambles_keys(): + data = { + "documentState": "DRAFT", + "id": "otm-123456", + "qualificationId": "otm-123456", + "personId": "otm-123456", + "studyRightId": "otm-123456", + "moduleGroupId": "otm-123456", + "additionalInformation": "string", + "associatedStudies": [ + { + "description": "string", + "credits": 0, + "attainmentLocation": "urn:code:educational-institution:*", + "associatedDegree": "string", + "attainmentDate": "2026-09-18" + } + ], + "attainmentDate": "2026-09-18", + "studyFieldUrn": "urn:code:study-field:*", + "attainmentMethod": "AUTOMATIC", + "credits": 0, + "attainmentIds": [ + "otm-123456" + ], + "childAttainedQualificationIds": [ + "otm-123456" + ], + "registrationDate": "2026-09-18", + "verifierPersonId": "otm-123456" + } + + scrambled_data = AttainedQualificationScrambler.scramble(copy.deepcopy(data), set()) + + assert len(AttainedQualificationScrambler.dict_items_with_scrambling()) >= 2 + for key, _ in AttainedQualificationScrambler.dict_items_with_scrambling(): + if isinstance(scrambled_data.get(key), bool): + # Cannot reliably test for bool 50/50 changes + continue + + assert scrambled_data.get(key, -42) != data.get(key, -42) + + AttainedQualification.model_validate(scrambled_data) + + +@pytest.mark.unit +def test_attained_qualification_scrambler_handles_lists_correctly(): + data = { + "documentState": "DRAFT", + "id": "otm-123456", + "qualificationId": "otm-123456", + "personId": "otm-123456", + "studyRightId": "otm-123456", + "moduleGroupId": "otm-123456", + "additionalInformation": "string", + "associatedStudies": [ + { + "description": "string", + "credits": 0, + "attainmentLocation": "urn:code:educational-institution:*", + "associatedDegree": "string", + "attainmentDate": "2026-09-18" + } + ], + "attainmentDate": "2026-09-18", + "studyFieldUrn": "urn:code:study-field:*", + "attainmentMethod": "AUTOMATIC", + "credits": 0, + "attainmentIds": [ + "otm-123456" + ], + "childAttainedQualificationIds": [ + "otm-123456" + ], + "registrationDate": "2026-09-18", + "verifierPersonId": "otm-123456" + } + + scrambled_data = AttainedQualificationScrambler.scramble(copy.deepcopy(data), set()) + list_keys = {'associatedStudies', 'attainmentIds', 'childAttainedQualificationIds'} + for key in list_keys: + assert isinstance(scrambled_data[key], list), scrambled_data[key] + + AttainedQualification.model_validate(scrambled_data) diff --git a/tests/scramblers/test_attainment_scrambler.py b/tests/scramblers/ori/test_attainment_scrambler.py similarity index 100% rename from tests/scramblers/test_attainment_scrambler.py rename to tests/scramblers/ori/test_attainment_scrambler.py diff --git a/tests/scramblers/ori/test_mobility_period_scrambler.py b/tests/scramblers/ori/test_mobility_period_scrambler.py new file mode 100644 index 0000000..240c5e2 --- /dev/null +++ b/tests/scramblers/ori/test_mobility_period_scrambler.py @@ -0,0 +1,42 @@ +import copy + +import pytest + +from funidata_utils.data_scramblers.mobility_period import MobilityPeriodScrambler +from funidata_utils.schemas.sisu import MobilityPeriod + + +@pytest.mark.unit +def test_mobility_period_scrambler_scrambles_keys(): + data = { + "documentState": "DRAFT", + "id": "otm-123456", + "studyRightId": "otm-123456", + "personId": "otm-123456", + "mobilityDirection": "INBOUND", + "activityPeriod": { + "startDate": "2026-09-18", + "endDate": "2026-09-18" + }, + "phase": "PHASE1", + "mobilityProgramUrn": "urn:code:mobility-program:123", + "mobilityProgramDescription": "string", + "mobilityTypeUrn": "urn:code:mobility-type:123", + "countryUrn": "urn:code:country:123", + "internationalInstitutionUrn": "urn:code:international-institution:123", + "organisation": "string", + "mobilityStudyRightTypeUrn": "urn:code:mobility-study-right-type:123", + "virtualMobilityType": "None", + "isCancelled": True + } + scrambled_data = MobilityPeriodScrambler.scramble(copy.deepcopy(data), set()) + + assert len(MobilityPeriodScrambler.dict_items_with_scrambling()) >= 2 + for key, _ in MobilityPeriodScrambler.dict_items_with_scrambling(): + if isinstance(scrambled_data.get(key), bool): + # Cannot reliably test for bool 50/50 changes + continue + + assert scrambled_data.get(key, -42) != data.get(key, -42) + + MobilityPeriod.model_validate(scrambled_data) diff --git a/tests/scramblers/test_ori_worfklow_scramblers.py b/tests/scramblers/ori/test_ori_worfklow_scramblers.py similarity index 100% rename from tests/scramblers/test_ori_worfklow_scramblers.py rename to tests/scramblers/ori/test_ori_worfklow_scramblers.py diff --git a/tests/scramblers/test_ori_person_scrambler.py b/tests/scramblers/ori/test_private_person_scrambler.py similarity index 100% rename from tests/scramblers/test_ori_person_scrambler.py rename to tests/scramblers/ori/test_private_person_scrambler.py diff --git a/tests/scramblers/ori/test_study_right_scrambler.py b/tests/scramblers/ori/test_study_right_scrambler.py new file mode 100644 index 0000000..405296d --- /dev/null +++ b/tests/scramblers/ori/test_study_right_scrambler.py @@ -0,0 +1,546 @@ +import copy + +import pytest + +from funidata_utils.data_scramblers.study_right import StudyRightScrambler +from funidata_utils.schemas.sisu import StudyRight + + +@pytest.mark.unit +def test_study_right_scrambler_scrambles_keys(): + data = { + "documentState": "DRAFT", + "id": "otm-123456", + "snapshotDateTime": "2026-09-21T07:57:56.057Z", + "studentId": "otm-123456", + "educationId": "otm-123456", + "organisationId": "otm-123456", + "learningOpportunityId": "otm-123456", + "admissionTargetId": "otm-123456", + "admissionIdentifier": "string", + "decreeOnUniversityDegreesUrn": "urn:code:decree-on-university-degrees:123", + "studyRightExpirationRulesUrn": "urn:code:study-right-expiration-rules:123", + "degreeRegulations": "string", + "valid": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + }, + "grantDate": "2026-09-21", + "studyStartDate": "2026-09-21", + "alternativeStudyStartDate": "2026-09-21", + "transferOutDate": "2026-09-21", + "transferOutUniversityUrn": "urn:code:educational-institution:123", + "homeOrganisationUrn": "urn:code:educational-institution:123", + "termRegistrations": [ + { + "localId": "otm-123456", + "studyTerm": { + "studyYearStartYear": 0, + "termIndex": 0 + }, + "registrationDate": "2026-09-21", + "termRegistrationType": "ATTENDING", + "previousRegistrationType": "ATTENDING", + "previousRegistrationDate": "2026-09-21", + "statutoryAbsence": True, + "statutoryAbsenceDate": "2026-09-21", + "statutoryAbsenceChangedBy": "otm-123456", + "tuitionFeePaymentState": "PAID" + } + ], + "studyRightExtensions": [ + { + "localId": "otm-123456", + "state": "ACTIVE", + "extensionCount": 1, + "extensionStartDate": "2026-09-21", + "grantDate": "2026-09-21", + "workflowId": "otm-123456", + "grantReason": "string", + "grantedBy": "otm-123456", + "deleteDate": "2026-09-21", + "deleteReason": "string", + "deletedBy": "otm-123456" + } + ], + "studyRightCancellation": { + "cancellationDate": "2020-09-21", + "cancellationReason": "string", + "cancellationType": "RESCINDED" + }, + "studyRightPassivations": [ + { + "localId": "otm-123456", + "passivationType": "DEGREE_REGULATIONS", + "valid": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + }, + "additionalInfo": "string" + } + ], + "studyRightGraduation": { + "phase1GraduationDate": "2026-09-21", + "phase2GraduationDate": "2026-09-21" + }, + "acceptedSelectionPath": { + "educationPhase1GroupId": "otm-123456", + "educationPhase1ChildGroupId": "otm-123456", + "educationPhase2GroupId": "otm-123456", + "educationPhase2ChildGroupId": "otm-123456" + }, + "requestedSelectionPath": { + "educationPhase1GroupId": "otm-123456", + "educationPhase1ChildGroupId": "otm-123456", + "educationPhase2GroupId": "otm-123456", + "educationPhase2ChildGroupId": "otm-123456" + }, + "studyRightTransfer": { + "originalStartDate": "2026-09-21", + "originalUniversityUrn": "urn:code:educational-institution:123", + "usedTerms": 0, + "usedAbsenceTerms": 0, + "usedStatutoryAbsenceTerms": 0, + "transferComments": "string" + }, + "phase1MinorSelections": [ + { + "localId": "otm-123456", + "validityPeriod": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + }, + "acceptorPersonId": "otm-123456", + "acceptanceDate": "2026-09-21", + "moduleGroupId": "otm-123456", + "selectionState": "REQUESTED", + "selectionType": "urn:code:study-right-selection-type:minor-study-right" + } + ], + "phase2MinorSelections": [ + { + "localId": "otm-123456", + "validityPeriod": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + }, + "acceptorPersonId": "otm-123456", + "acceptanceDate": "2026-09-21", + "moduleGroupId": "otm-123456", + "selectionState": "REQUESTED", + "selectionType": "urn:code:study-right-selection-type:minor-study-right" + } + ], + "state": "NOT_STARTED", + "statePeriods": [ + { + "state": "NOT_STARTED", + "startDate": "2026-09-21", + "endDate": "2026-09-21" + } + ], + "personalizedSelectionPath": { + "phase1": { + "rationale": "string", + "createdBy": "otm-123456", + "createdOn": "2026-09-21T07:57:56.058Z", + "moduleGroupId": "otm-123456", + "childModuleGroupId": "otm-123456", + "childNamingUrn": "urn:code:education-option-naming-type:123", + "degreeTitleUrn": "urn:code:degree-title:123", + "educationClassificationUrn": "urn:code:education-classification:123" + }, + "phase2": { + "rationale": "string", + "createdBy": "otm-123456", + "createdOn": "2026-09-21T07:57:56.058Z", + "moduleGroupId": "otm-123456", + "childModuleGroupId": "otm-123456", + "childNamingUrn": "urn:code:education-option-naming-type:123", + "degreeTitleUrn": "urn:code:degree-title:123", + "educationClassificationUrn": "urn:code:education-classification:123" + } + }, + "courseUnitSelections": [ + { + "localId": "otm-123456", + "validityPeriod": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + }, + "acceptorPersonId": "otm-123456", + "acceptanceDate": "2026-09-21", + "courseUnitGroupId": "otm-123456" + } + ], + "moduleSelections": [ + { + "localId": "otm-123456", + "validityPeriod": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + }, + "acceptorPersonId": "otm-123456", + "acceptanceDate": "2026-09-21", + "moduleGroupId": "otm-123456" + } + ], + "studyFieldUrn": "urn:code:study-field:123", + "phase1EducationClassificationUrn": "urn:code:education-classification:123", + "phase2EducationClassificationUrn": "urn:code:education-classification:123", + "phase1EducationClassificationLocked": True, + "phase2EducationClassificationLocked": True, + "fundingSourceUrn": "urn:code:funding-source:123", + "phase1QualificationUrns": [ + "string" + ], + "phase2QualificationUrns": [ + "string" + ], + "phase1EducationLocationUrn": "urn:code:municipality:123", + "phase2EducationLocationUrn": "urn:code:municipality:123", + "phase1InternationalContractualDegree": { + "localId": "otm-123456", + "internationalContractualDegreeAgreementId": "otm-123456", + "attainableDegrees": [ + { + "degreeName": "string", + "internationalInstitutionUrn": "urn:code:international-institution:123" + } + ] + }, + "phase2InternationalContractualDegree": { + "localId": "otm-123456", + "internationalContractualDegreeAgreementId": "otm-123456", + "attainableDegrees": [ + { + "degreeName": "string", + "internationalInstitutionUrn": "urn:code:international-institution:123" + } + ] + }, + "admissionTypeUrn": "urn:code:admission-type:123", + "codeUrns": [ + "urn:code:code" + ], + "additionalInformation": { + "fi": "Finnish version", + "sv": "Swedish version", + "en": "English version" + }, + "cooperationNetworkRights": [ + { + "localId": "otm-123456", + "cooperationNetworkId": "otm-123456", + "validityPeriod": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + }, + "updateType": "MANUAL", + "targetGroupUpdateDate": "2026-09-21" + } + ], + "cooperationNetworkStatus": { + "direction": "INBOUND", + "organisationTkCode": "string", + "outboundStatus": "NOT_VALID", + "rejectionReason": { + "fi": "Finnish version", + "sv": "Swedish version", + "en": "English version" + }, + "outboundStatusMessage": "string", + "cooperationNetworkId": "otm-123456", + "universityOrgId": "otm-123456", + "homeStudyRightId": "string", + "inboundStatus": "ACCEPTED", + "homeStudyRightValidity": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + } + }, + "schoolEducationLanguageUrn": "urn:code:school-education-language:123" + } + scrambled_data = StudyRightScrambler.scramble(copy.deepcopy(data), set()) + + assert len(StudyRightScrambler.dict_items_with_scrambling()) >= 2 + for key, _ in StudyRightScrambler.dict_items_with_scrambling(): + if isinstance(scrambled_data.get(key), bool): + # Cannot reliably test for bool 50/50 changes + continue + + assert scrambled_data.get(key, -42) != data.get(key, -42) + + StudyRight.model_validate(scrambled_data) + + +@pytest.mark.unit +def test_study_right_scrambler_handles_lists_correctly(): + data = { + "documentState": "DRAFT", + "id": "otm-123456", + "snapshotDateTime": "2026-09-21T07:57:56.057Z", + "studentId": "otm-123456", + "educationId": "otm-123456", + "organisationId": "otm-123456", + "learningOpportunityId": "otm-123456", + "admissionTargetId": "otm-123456", + "admissionIdentifier": "string", + "decreeOnUniversityDegreesUrn": "urn:code:decree-on-university-degrees:123", + "studyRightExpirationRulesUrn": "urn:code:study-right-expiration-rules:123", + "degreeRegulations": "string", + "valid": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + }, + "grantDate": "2026-09-21", + "studyStartDate": "2026-09-21", + "alternativeStudyStartDate": "2026-09-21", + "transferOutDate": "2026-09-21", + "transferOutUniversityUrn": "urn:code:educational-institution:123", + "homeOrganisationUrn": "urn:code:educational-institution:123", + "termRegistrations": [ + { + "localId": "otm-123456", + "studyTerm": { + "studyYearStartYear": 0, + "termIndex": 0 + }, + "registrationDate": "2026-09-21", + "termRegistrationType": "ATTENDING", + "previousRegistrationType": "ATTENDING", + "previousRegistrationDate": "2026-09-21", + "statutoryAbsence": True, + "statutoryAbsenceDate": "2026-09-21", + "statutoryAbsenceChangedBy": "otm-123456", + "tuitionFeePaymentState": "PAID" + } + ], + "studyRightExtensions": [ + { + "localId": "otm-123456", + "state": "ACTIVE", + "extensionCount": 1, + "extensionStartDate": "2026-09-21", + "grantDate": "2026-09-21", + "workflowId": "otm-123456", + "grantReason": "string", + "grantedBy": "otm-123456", + "deleteDate": "2026-09-21", + "deleteReason": "string", + "deletedBy": "otm-123456" + } + ], + "studyRightCancellation": { + "cancellationDate": "2020-09-21", + "cancellationReason": "string", + "cancellationType": "RESCINDED" + }, + "studyRightPassivations": [ + { + "localId": "otm-123456", + "passivationType": "DEGREE_REGULATIONS", + "valid": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + }, + "additionalInfo": "string" + } + ], + "studyRightGraduation": { + "phase1GraduationDate": "2026-09-21", + "phase2GraduationDate": "2026-09-21" + }, + "acceptedSelectionPath": { + "educationPhase1GroupId": "otm-123456", + "educationPhase1ChildGroupId": "otm-123456", + "educationPhase2GroupId": "otm-123456", + "educationPhase2ChildGroupId": "otm-123456" + }, + "requestedSelectionPath": { + "educationPhase1GroupId": "otm-123456", + "educationPhase1ChildGroupId": "otm-123456", + "educationPhase2GroupId": "otm-123456", + "educationPhase2ChildGroupId": "otm-123456" + }, + "studyRightTransfer": { + "originalStartDate": "2026-09-21", + "originalUniversityUrn": "urn:code:educational-institution:123", + "usedTerms": 0, + "usedAbsenceTerms": 0, + "usedStatutoryAbsenceTerms": 0, + "transferComments": "string" + }, + "phase1MinorSelections": [ + { + "localId": "otm-123456", + "validityPeriod": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + }, + "acceptorPersonId": "otm-123456", + "acceptanceDate": "2026-09-21", + "moduleGroupId": "otm-123456", + "selectionState": "REQUESTED", + "selectionType": "urn:code:study-right-selection-type:minor-study-right" + } + ], + "phase2MinorSelections": [ + { + "localId": "otm-123456", + "validityPeriod": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + }, + "acceptorPersonId": "otm-123456", + "acceptanceDate": "2026-09-21", + "moduleGroupId": "otm-123456", + "selectionState": "REQUESTED", + "selectionType": "urn:code:study-right-selection-type:minor-study-right" + } + ], + "state": "NOT_STARTED", + "statePeriods": [ + { + "state": "NOT_STARTED", + "startDate": "2026-09-21", + "endDate": "2026-09-21" + } + ], + "personalizedSelectionPath": { + "phase1": { + "rationale": "string", + "createdBy": "otm-123456", + "createdOn": "2026-09-21T07:57:56.058Z", + "moduleGroupId": "otm-123456", + "childModuleGroupId": "otm-123456", + "childNamingUrn": "urn:code:education-option-naming-type:123", + "degreeTitleUrn": "urn:code:degree-title:123", + "educationClassificationUrn": "urn:code:education-classification:123" + }, + "phase2": { + "rationale": "string", + "createdBy": "otm-123456", + "createdOn": "2026-09-21T07:57:56.058Z", + "moduleGroupId": "otm-123456", + "childModuleGroupId": "otm-123456", + "childNamingUrn": "urn:code:education-option-naming-type:123", + "degreeTitleUrn": "urn:code:degree-title:123", + "educationClassificationUrn": "urn:code:education-classification:123" + } + }, + "courseUnitSelections": [ + { + "localId": "otm-123456", + "validityPeriod": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + }, + "acceptorPersonId": "otm-123456", + "acceptanceDate": "2026-09-21", + "courseUnitGroupId": "otm-123456" + } + ], + "moduleSelections": [ + { + "localId": "otm-123456", + "validityPeriod": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + }, + "acceptorPersonId": "otm-123456", + "acceptanceDate": "2026-09-21", + "moduleGroupId": "otm-123456" + } + ], + "studyFieldUrn": "urn:code:study-field:123", + "phase1EducationClassificationUrn": "urn:code:education-classification:123", + "phase2EducationClassificationUrn": "urn:code:education-classification:123", + "phase1EducationClassificationLocked": True, + "phase2EducationClassificationLocked": True, + "fundingSourceUrn": "urn:code:funding-source:123", + "phase1QualificationUrns": [ + "string" + ], + "phase2QualificationUrns": [ + "string" + ], + "phase1EducationLocationUrn": "urn:code:municipality:123", + "phase2EducationLocationUrn": "urn:code:municipality:123", + "phase1InternationalContractualDegree": { + "localId": "otm-123456", + "internationalContractualDegreeAgreementId": "otm-123456", + "attainableDegrees": [ + { + "degreeName": "string", + "internationalInstitutionUrn": "urn:code:international-institution:123" + } + ] + }, + "phase2InternationalContractualDegree": { + "localId": "otm-123456", + "internationalContractualDegreeAgreementId": "otm-123456", + "attainableDegrees": [ + { + "degreeName": "string", + "internationalInstitutionUrn": "urn:code:international-institution:123" + } + ] + }, + "admissionTypeUrn": "urn:code:admission-type:123", + "codeUrns": [ + "urn:code:code" + ], + "additionalInformation": { + "fi": "Finnish version", + "sv": "Swedish version", + "en": "English version" + }, + "cooperationNetworkRights": [ + { + "localId": "otm-123456", + "cooperationNetworkId": "otm-123456", + "validityPeriod": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + }, + "updateType": "MANUAL", + "targetGroupUpdateDate": "2026-09-21" + } + ], + "cooperationNetworkStatus": { + "direction": "INBOUND", + "organisationTkCode": "string", + "outboundStatus": "NOT_VALID", + "rejectionReason": { + "fi": "Finnish version", + "sv": "Swedish version", + "en": "English version" + }, + "outboundStatusMessage": "string", + "cooperationNetworkId": "otm-123456", + "universityOrgId": "otm-123456", + "homeStudyRightId": "string", + "inboundStatus": "ACCEPTED", + "homeStudyRightValidity": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + } + }, + "schoolEducationLanguageUrn": "urn:code:school-education-language:123" + } + scrambled_data = StudyRightScrambler.scramble(copy.deepcopy(data), set()) + list_keys = { + 'termRegistrations', 'studyRightExtensions', 'studyRightPassivations', + 'phase1MinorSelections', 'phase2MinorSelections', 'statePeriods', + 'courseUnitSelections', 'moduleSelections', + 'phase1QualificationUrns', 'phase2QualificationUrns', + 'codeUrns', 'cooperationNetworkRights' + } + for key in list_keys: + if key in {'termRegistrations'}: + # skip termregs since it's ... not a real part of studyrights. + continue + assert isinstance(scrambled_data[key], list), (key, scrambled_data[key], data[key]) + + StudyRight.model_validate(scrambled_data) diff --git a/tests/scramblers/ori/test_thesis_scrambler.py b/tests/scramblers/ori/test_thesis_scrambler.py new file mode 100644 index 0000000..9090f4c --- /dev/null +++ b/tests/scramblers/ori/test_thesis_scrambler.py @@ -0,0 +1,69 @@ +import copy + +import pytest + +from funidata_utils.data_scramblers.thesis import ThesisScrambler +from funidata_utils.schemas.sisu import Thesis + + +@pytest.mark.unit +def test_thesis_scrambler_scrambles(): + data = { + "documentState": "ACTIVE", + "id": "otm-123456", + "personId": "otm-123456", + "attainmentId": "otm-123456", + "title": { + "fi": "Finnish version", + "sv": "Swedish version", + "en": "English version" + }, + "subject": { + "fi": "Finnish version", + "sv": "Swedish version", + "en": "English version" + }, + "thesisTypeUrn": "urn:code:thesis-type:amk-bachelors-thesis", + "responsibilityInfos": [ + { + "text": { + "fi": "Finnish version", + "sv": "Swedish version", + "en": "English version" + }, + "personId": "otm-123456", + "roleUrn": "urn:code:attainment-acceptor-type:approved-by", + "title": { + "fi": "Finnish version", + "sv": "Swedish version", + "en": "English version" + } + } + ], + "organisations": [ + { + "organisationId": "otm-123456", + "educationalInstitutionUrn": "urn:code:educational-institution:01", + "roleUrn": "urn:code:organisation-role:responsible-organisation", + "share": 1 + } + ], + "courseUnitId": "otm-123456", + "courseUnitGroupId": "otm-123456", + "state": "ATTAINED", + "publicInspectionDate": "2026-09-21", + "commissionType": "NONE" + } + + scrambled_data = ThesisScrambler.scramble(copy.deepcopy(data), set()) + + assert len(ThesisScrambler.dict_items_with_scrambling()) >= 3 + for key, _ in ThesisScrambler.dict_items_with_scrambling(): + if isinstance(scrambled_data.get(key), bool): + # Cannot reliably test for bool 50/50 changes + continue + + assert scrambled_data.get(key, -42) != data.get(key, -42) + + # TODO: Update when real responsibility info scrambling is implemented + # Thesis.model_validate(scrambled_data) \ No newline at end of file diff --git a/tests/scramblers/ori/test_tuition_fee_obligation_period_scrambler.py b/tests/scramblers/ori/test_tuition_fee_obligation_period_scrambler.py new file mode 100644 index 0000000..b638b31 --- /dev/null +++ b/tests/scramblers/ori/test_tuition_fee_obligation_period_scrambler.py @@ -0,0 +1,35 @@ +import copy + +import pytest + +from funidata_utils.data_scramblers.tuition_fee_obligation_period import TuitionFeeObligationPeriodScrambler + + +@pytest.mark.unit +def test_tuition_fee_scrambler_scrambles(): + data = { + "documentState": "ACTIVE", + "id": "otm-123456", + "studyRightId": "otm-123456", + "valid": { + "startDate": "2026-09-21", + "endDate": "2026-09-21" + }, + "tuitionFee": 0, + "exempt": True, + "additionalInfo": { + "fi": "Finnish version", + "sv": "Swedish version", + "en": "English version" + } + } + + scrambled_data = TuitionFeeObligationPeriodScrambler.scramble(copy.deepcopy(data), set()) + + assert len(TuitionFeeObligationPeriodScrambler.dict_items_with_scrambling()) >= 1 + for key, _ in TuitionFeeObligationPeriodScrambler.dict_items_with_scrambling(): + if isinstance(scrambled_data.get(key), bool): + # Cannot reliably test for bool 50/50 changes + continue + + assert scrambled_data.get(key, -42) != data.get(key, -42) \ No newline at end of file diff --git a/tests/scramblers/osuva/__init__.py b/tests/scramblers/osuva/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scramblers/test_osuva_plan_scrambler.py b/tests/scramblers/osuva/test_osuva_plan_scrambler.py similarity index 100% rename from tests/scramblers/test_osuva_plan_scrambler.py rename to tests/scramblers/osuva/test_osuva_plan_scrambler.py diff --git a/tests/test_sync_import_compatibility.py b/tests/test_sync_import_compatibility.py new file mode 100644 index 0000000..93db880 --- /dev/null +++ b/tests/test_sync_import_compatibility.py @@ -0,0 +1,156 @@ +from typing import Literal + +import pytest + +from funidata_utils.request_utils import async_httpx_requests +from funidata_utils.request_utils.httpx_requests import send_post_with_binary_err_search_httpx +from tests.helpers import mock_client, get_entity_counts_by_status_code, _get_mock_async_httpx_client + + +def test_recursive_import_batching_with_sublists_off_no_fails(mock_client, monkeypatch): + monkeypatch.setattr(async_httpx_requests, "_get_async_httpx_client", _get_mock_async_httpx_client) + + test_data = [ + { + "id": 2, + "person": 1, + }, + { + "id": 3, + "person": 1 + }, + { + "id": 3, + "person": 2 + }, + { + "id": 4, + "person": 2 + }, + { + "id": 4, + "person": 3 + }, + { + "id": 5, + "person": 3 + } + ] + + _state: dict[ + Literal['max_seen_depth', 'sent_requests'], int + ] = {'max_seen_depth': 0, 'sent_requests': 0} + results = send_post_with_binary_err_search_httpx( + path="http://localhost", + payload=test_data, + binary_err_search_sublists=False, + binary_search_max_depth=None, + batch_size=10, + _state=_state, + group_by_key='person', + ) + assert _state['max_seen_depth'] == 0 + assert _state['sent_requests'] == 1 + assert get_entity_counts_by_status_code(results)[200] == 6 + assert get_entity_counts_by_status_code(results).get(422) is None + + +def test_recursive_import_batching_with_sublists_off_one_fail(mock_client, monkeypatch): + monkeypatch.setattr(async_httpx_requests, "_get_async_httpx_client", _get_mock_async_httpx_client) + + test_data = [ + { + "id": 2, + "person": 1, + "invalid": True + }, + { + "id": 3, + "person": 1 + }, + { + "id": 3, + "person": 2 + }, + { + "id": 4, + "person": 2 + }, + { + "id": 4, + "person": 3 + }, + { + "id": 5, + "person": 3 + } + ] + + _state: dict[ + Literal['max_seen_depth', 'sent_requests'], int + ] = {'max_seen_depth': 0, 'sent_requests': 0} + results = send_post_with_binary_err_search_httpx( + path="http://localhost", + payload=test_data, + binary_err_search_sublists=False, + binary_search_max_depth=None, + batch_size=10, + group_by_key='person', + _state=_state, + ) + assert _state['max_seen_depth'] == 1 + assert _state['sent_requests'] == 3 + assert get_entity_counts_by_status_code(results)[200] == 4 + assert get_entity_counts_by_status_code(results)[422] == 2 + + +def test_recursive_import_batching_with_sublists_off_multiple_fails(mock_client, monkeypatch): + monkeypatch.setattr(async_httpx_requests, "_get_async_httpx_client", _get_mock_async_httpx_client) + + test_data = [ + { + "id": 2, + "person": 1, + "invalid": True + }, + { + "id": 3, + "person": 1, + "invalid": True, + }, + { + "id": 3, + "person": 2 + }, + { + "id": 4, + "person": 2 + }, + { + "id": 4, + "person": 3 + }, + { + "id": 5, + "person": 3, + "invalid": True, + } + ] + + _state: dict[ + Literal['max_seen_depth', 'sent_requests'], int + ] = {'max_seen_depth': 0, 'sent_requests': 0} + results = send_post_with_binary_err_search_httpx( + path="http://localhost", + payload=test_data, + binary_err_search_sublists=False, + binary_search_max_depth=None, + batch_size=10, + group_by_key='person', + _state=_state, + ) + assert _state['max_seen_depth'] == 2 + assert _state['sent_requests'] == 5 + + assert get_entity_counts_by_status_code(results)[200] == 2 + assert get_entity_counts_by_status_code(results)[422] == 4