From bc3c8d0c63799c69f67d5ee8fd95f2e05a8c09aa Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 10 Sep 2026 11:02:40 -0600 Subject: [PATCH 1/4] add mopra coverage --- imap_processing/ialirt/generate_coverage.py | 12 +++++++++++ .../ialirt/unit/test_generate_coverage.py | 20 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/imap_processing/ialirt/generate_coverage.py b/imap_processing/ialirt/generate_coverage.py index 99bbc699d9..4a957488ee 100644 --- a/imap_processing/ialirt/generate_coverage.py +++ b/imap_processing/ialirt/generate_coverage.py @@ -1,6 +1,7 @@ """Coverage time for each station.""" import logging +from datetime import time as dt_time import numpy as np @@ -159,6 +160,17 @@ def generate_coverage( # noqa: PLR0912 time_range[outage_mask], format_str="ISOC" ) + # Mopra: fixed daily allocation (20:45-00:30 UTC), no geometry needed. + # Split into two same-day windows to avoid the midnight rollover. + mopra_evening = StationProperties(0, 0, 0, 0, dt_time(20, 45), dt_time(23, 59, 59)) + mopra_morning = StationProperties(0, 0, 0, 0, dt_time(0, 0), dt_time(0, 30)) + mopra_mask = create_schedule_mask(mopra_evening, time_range) | create_schedule_mask( + mopra_morning, time_range + ) + total_visible_mask |= mopra_mask + coverage_dict["Mopra"] = et_to_utc(time_range[mopra_mask], format_str="ISOC") + outage_dict["Mopra"] = np.array([], dtype=" Date: Mon, 28 Sep 2026 10:21:29 -0600 Subject: [PATCH 2/4] do not allow dsn overlap --- imap_processing/ialirt/generate_coverage.py | 14 +- .../ialirt/unit/test_generate_coverage.py | 28 +++- .../ialirt/unit/test_verify_iois_packets.py | 152 ++++++++++++++++++ 3 files changed, 189 insertions(+), 5 deletions(-) create mode 100644 imap_processing/tests/ialirt/unit/test_verify_iois_packets.py diff --git a/imap_processing/ialirt/generate_coverage.py b/imap_processing/ialirt/generate_coverage.py index 4a957488ee..7012251520 100644 --- a/imap_processing/ialirt/generate_coverage.py +++ b/imap_processing/ialirt/generate_coverage.py @@ -14,6 +14,7 @@ ALL_STATIONS = [ "Kiel", + "Mopra", "DSS-24", "DSS-25", "DSS-26", @@ -167,9 +168,20 @@ def generate_coverage( # noqa: PLR0912 mopra_mask = create_schedule_mask(mopra_evening, time_range) | create_schedule_mask( mopra_morning, time_range ) + + mopra_outage_mask: np.ndarray = np.zeros(time_range.shape, dtype=bool) + if outages and "Mopra" in outages: + for start, end in outages["Mopra"]: + start_et = str_to_et(start) + end_et = str_to_et(end) + mopra_outage_mask |= (time_range >= start_et) & (time_range <= end_et) + + mopra_mask[mopra_outage_mask] = False + # DSN contacts block Mopra + mopra_mask[dsn_outage_mask] = False total_visible_mask |= mopra_mask coverage_dict["Mopra"] = et_to_utc(time_range[mopra_mask], format_str="ISOC") - outage_dict["Mopra"] = np.array([], dtype="= "20:45" for t in hhmm) + assert len(mopra_times) == 46 + + # DSN contacts and Mopra outages block Mopra coverage. + dsn = {"DSS-34": [("2026-09-22T20:58:00Z", "2026-09-22T21:32:00Z")]} + outages = {"Mopra": [("2026-09-21T23:58:00Z", "2026-09-22T00:12:00Z")]} + coverage_dict, outage_dict = generate_coverage( + "2026-09-22T00:00:00Z", outages=outages, dsn=dsn + ) - assert "2026-09-22T20:45:00.000" in mopra_times - assert "2026-09-22T00:30:00.000" in mopra_times - assert "2026-09-22T12:00:00.000" not in mopra_times + mopra_times = coverage_dict["Mopra"] + assert "2026-09-22T21:15:00.000" not in mopra_times + assert "2026-09-22T00:05:00.000" not in mopra_times + assert "2026-09-22T00:05:00.000" in outage_dict["Mopra"] + # 7 DSN samples (21:00-21:30) + 3 outage samples (00:00-00:10) removed. + assert len(mopra_times) == 46 - 7 - 3 diff --git a/imap_processing/tests/ialirt/unit/test_verify_iois_packets.py b/imap_processing/tests/ialirt/unit/test_verify_iois_packets.py new file mode 100644 index 0000000000..8198b75c7f --- /dev/null +++ b/imap_processing/tests/ialirt/unit/test_verify_iois_packets.py @@ -0,0 +1,152 @@ +"""Reproduce the manual verification of raw ``iois_1_packets_*`` capture files. + +These are ad hoc telemetry captures that live locally in ``~/Downloads`` and +are not part of the repo's committed/external test data set, so the +``xarray_data`` fixture skips the whole module if a file isn't present on +disk. + +Two things are checked, matching the manual verification: + +* Header consistency: every packet shares the same APID and CCSDS header + constants ("expected sync pattern"), and the same packet-length field. +* Source sequence counter: any gap in ``src_seq_ctr`` is fully explained by + elapsed seconds on the onboard clock (``sc_sclk_sec``), accounting for + 14-bit rollover -- i.e. gaps reflect real missing/uncaptured packets, not + counter corruption. +""" + +from pathlib import Path + +import pytest +import xarray as xr + +from imap_processing import imap_module_directory +from imap_processing.ialirt.utils.time import calculate_time +from imap_processing.spice.time import met_to_utc +from imap_processing.utils import packet_file_to_datasets + +IALIRT_XTCE_PATH = ( + imap_module_directory / "ialirt" / "packet_definitions" / "ialirt.xml" +) +DOWNLOADS_DIR = Path.home() / "Downloads" + +SC_APID = 478 +EXPECTED_PACKET_DATA_LENGTH = 176 # PKT_LEN field value for the SC (apid 478) packet +SEQ_CTR_MAX = 16384 # 14-bit rollover +SUB_SEC_CONVERSION = 256 # fine-time units per second for SC_SCLK_SUB_SEC + +PACKET_FILES = [ + "iois_1_packets_2026_268_21_14_27", + "iois_1_packets_2026_268_21_15_28", + "iois_1_packets_2026_268_21_16_29", + "iois_1_packets_2026_268_21_17_30", + "iois_1_packets_2026_268_21_18_31", + "iois_1_packets_2026_268_21_19_32", + "iois_1_packets_2026_268_21_20_33", + "iois_1_packets_2026_268_21_21_34", + "iois_1_packets_2026_268_21_22_35", + "iois_1_packets_2026_268_21_23_36", + "iois_1_packets_2026_268_21_24_37", + "iois_1_packets_2026_268_21_25_38", + "iois_1_packets_2026_268_21_26_39", + "iois_1_packets_2026_268_21_28_41", + "iois_1_packets_2026_268_21_29_42", + "iois_1_packets_2026_268_21_30_43", + "iois_1_packets_2026_268_21_31_44", + "iois_1_packets_2026_268_21_32_45", + "iois_1_packets_2026_268_21_33_46", + "iois_1_packets_2026_268_21_34_47", + "iois_1_packets_2026_268_21_35_48", + "iois_1_packets_2026_268_21_36_49", + "iois_1_packets_2026_268_21_37_50", + "iois_1_packets_2026_268_21_38_51", + "iois_1_packets_2026_268_21_39_52", + "iois_1_packets_2026_268_21_40_54", +] + + +def _packet_path(filename: str) -> Path: + path = DOWNLOADS_DIR / filename + if not path.exists(): + pytest.skip(f"{path} not found locally") + return path + + +@pytest.fixture +def xarray_data(): + """Create merged xarray data for all captured SC (apid 478) packets.""" + packet_paths = tuple(_packet_path(filename) for filename in PACKET_FILES) + + xarray_data = tuple( + packet_file_to_datasets(packet, IALIRT_XTCE_PATH, use_derived_value=False)[ + SC_APID + ] + for packet in packet_paths + ) + + merged_xarray_data = xr.concat(xarray_data, dim="epoch").sortby("epoch") + return merged_xarray_data + + +def test_header_fields_are_consistent(xarray_data): + """Check APID, CCSDS header constants, and packet length.""" + assert len(xarray_data["epoch"]) > 0 + + assert set(xarray_data["pkt_apid"].values.tolist()) == {SC_APID} + assert set(xarray_data["version"].values.tolist()) == {0} + assert set(xarray_data["type"].values.tolist()) == {0} + assert set(xarray_data["sec_hdr_flg"].values.tolist()) == {1} + assert set(xarray_data["pkt_len"].values.tolist()) == {EXPECTED_PACKET_DATA_LENGTH} + + +def _find_sequence_counter_gaps(seq_counters) -> list[tuple[int, int]]: + """Find every place src_seq_ctr does not increment by exactly 1. + + This is the naive check (same logic as + ``imap_processing.utils._check_source_sequence_counter``) that first + surfaced the gaps -- it does not know about elapsed clock time, so a gap + here just means "not (previous + 1) % SEQ_CTR_MAX", real dropped packet + or not. + """ + gaps = [] + for i in range(1, len(seq_counters)): + expected_next = (int(seq_counters[i - 1]) + 1) % SEQ_CTR_MAX + if seq_counters[i] != expected_next: + gaps.append((int(seq_counters[i - 1]), int(seq_counters[i]))) + return gaps + + +def test_sequence_counter_gaps_match_elapsed_clock_time(xarray_data): + """Check that src_seq_ctr jumps are explained by elapsed sc_sclk_sec.""" + seq_counters = xarray_data["src_seq_ctr"].values + sclk_seconds = xarray_data["sc_sclk_sec"].values + + # sc_sclk_sec alone is only the coarse (integer-second) spacecraft clock + # count -- valid for elapsed-time math on its own, but not a real + # calendar time. Combine it with the fine time (sc_sclk_sub_sec) and run + # it through the SCLK kernel (via met_to_utc) to get an actual UTC + # timestamp for each packet. + met = calculate_time( + xarray_data["sc_sclk_sec"], xarray_data["sc_sclk_sub_sec"], SUB_SEC_CONVERSION + ) + utc_times = met_to_utc(met.values) + + gaps = _find_sequence_counter_gaps(seq_counters) + print(f"gaps = {gaps}") + + # Every gap found above should be fully explained by elapsed seconds on + # the onboard clock (accounting for 14-bit rollover) -- i.e. it reflects + # real missing/uncaptured packets, not counter corruption. + for i in range(1, len(seq_counters)): + elapsed_seconds = int(sclk_seconds[i]) - int(sclk_seconds[i - 1]) + expected_seq_ctr = (int(seq_counters[i - 1]) + elapsed_seconds) % SEQ_CTR_MAX + assert seq_counters[i] == expected_seq_ctr, ( + f"Packet {i}: src_seq_ctr {seq_counters[i]} does not match " + f"expected {expected_seq_ctr} given {elapsed_seconds}s elapsed " + f"on sc_sclk_sec" + ) + if seq_counters[i] != (int(seq_counters[i - 1]) + 1) % SEQ_CTR_MAX: + print( + f" gap: seq {seq_counters[i - 1]} ({utc_times[i - 1]} UTC) -> " + f"{seq_counters[i]} ({utc_times[i]} UTC)" + ) From a5bde09dac1130b2bd62cf7b3fe3bd6f120be8ed Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 28 Sep 2026 11:34:32 -0600 Subject: [PATCH 3/4] update --- imap_processing/ialirt/generate_coverage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/ialirt/generate_coverage.py b/imap_processing/ialirt/generate_coverage.py index 7012251520..d7a223e983 100644 --- a/imap_processing/ialirt/generate_coverage.py +++ b/imap_processing/ialirt/generate_coverage.py @@ -87,7 +87,7 @@ def generate_coverage( # noqa: PLR0912 uksa: list | None = None, ) -> tuple[dict, dict]: """ - Build the output dictionary containing coverage and outage time for each station. + Build the output dictionary containing coverage and outage time for station. Parameters ---------- From 1e2dfdb5842ddb99dfddd14eccacaff1b210e529 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 28 Sep 2026 11:45:42 -0600 Subject: [PATCH 4/4] Remove local-only IOIS verification test Co-Authored-By: Claude Opus 5.5 --- .../ialirt/unit/test_verify_iois_packets.py | 152 ------------------ 1 file changed, 152 deletions(-) delete mode 100644 imap_processing/tests/ialirt/unit/test_verify_iois_packets.py diff --git a/imap_processing/tests/ialirt/unit/test_verify_iois_packets.py b/imap_processing/tests/ialirt/unit/test_verify_iois_packets.py deleted file mode 100644 index 8198b75c7f..0000000000 --- a/imap_processing/tests/ialirt/unit/test_verify_iois_packets.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Reproduce the manual verification of raw ``iois_1_packets_*`` capture files. - -These are ad hoc telemetry captures that live locally in ``~/Downloads`` and -are not part of the repo's committed/external test data set, so the -``xarray_data`` fixture skips the whole module if a file isn't present on -disk. - -Two things are checked, matching the manual verification: - -* Header consistency: every packet shares the same APID and CCSDS header - constants ("expected sync pattern"), and the same packet-length field. -* Source sequence counter: any gap in ``src_seq_ctr`` is fully explained by - elapsed seconds on the onboard clock (``sc_sclk_sec``), accounting for - 14-bit rollover -- i.e. gaps reflect real missing/uncaptured packets, not - counter corruption. -""" - -from pathlib import Path - -import pytest -import xarray as xr - -from imap_processing import imap_module_directory -from imap_processing.ialirt.utils.time import calculate_time -from imap_processing.spice.time import met_to_utc -from imap_processing.utils import packet_file_to_datasets - -IALIRT_XTCE_PATH = ( - imap_module_directory / "ialirt" / "packet_definitions" / "ialirt.xml" -) -DOWNLOADS_DIR = Path.home() / "Downloads" - -SC_APID = 478 -EXPECTED_PACKET_DATA_LENGTH = 176 # PKT_LEN field value for the SC (apid 478) packet -SEQ_CTR_MAX = 16384 # 14-bit rollover -SUB_SEC_CONVERSION = 256 # fine-time units per second for SC_SCLK_SUB_SEC - -PACKET_FILES = [ - "iois_1_packets_2026_268_21_14_27", - "iois_1_packets_2026_268_21_15_28", - "iois_1_packets_2026_268_21_16_29", - "iois_1_packets_2026_268_21_17_30", - "iois_1_packets_2026_268_21_18_31", - "iois_1_packets_2026_268_21_19_32", - "iois_1_packets_2026_268_21_20_33", - "iois_1_packets_2026_268_21_21_34", - "iois_1_packets_2026_268_21_22_35", - "iois_1_packets_2026_268_21_23_36", - "iois_1_packets_2026_268_21_24_37", - "iois_1_packets_2026_268_21_25_38", - "iois_1_packets_2026_268_21_26_39", - "iois_1_packets_2026_268_21_28_41", - "iois_1_packets_2026_268_21_29_42", - "iois_1_packets_2026_268_21_30_43", - "iois_1_packets_2026_268_21_31_44", - "iois_1_packets_2026_268_21_32_45", - "iois_1_packets_2026_268_21_33_46", - "iois_1_packets_2026_268_21_34_47", - "iois_1_packets_2026_268_21_35_48", - "iois_1_packets_2026_268_21_36_49", - "iois_1_packets_2026_268_21_37_50", - "iois_1_packets_2026_268_21_38_51", - "iois_1_packets_2026_268_21_39_52", - "iois_1_packets_2026_268_21_40_54", -] - - -def _packet_path(filename: str) -> Path: - path = DOWNLOADS_DIR / filename - if not path.exists(): - pytest.skip(f"{path} not found locally") - return path - - -@pytest.fixture -def xarray_data(): - """Create merged xarray data for all captured SC (apid 478) packets.""" - packet_paths = tuple(_packet_path(filename) for filename in PACKET_FILES) - - xarray_data = tuple( - packet_file_to_datasets(packet, IALIRT_XTCE_PATH, use_derived_value=False)[ - SC_APID - ] - for packet in packet_paths - ) - - merged_xarray_data = xr.concat(xarray_data, dim="epoch").sortby("epoch") - return merged_xarray_data - - -def test_header_fields_are_consistent(xarray_data): - """Check APID, CCSDS header constants, and packet length.""" - assert len(xarray_data["epoch"]) > 0 - - assert set(xarray_data["pkt_apid"].values.tolist()) == {SC_APID} - assert set(xarray_data["version"].values.tolist()) == {0} - assert set(xarray_data["type"].values.tolist()) == {0} - assert set(xarray_data["sec_hdr_flg"].values.tolist()) == {1} - assert set(xarray_data["pkt_len"].values.tolist()) == {EXPECTED_PACKET_DATA_LENGTH} - - -def _find_sequence_counter_gaps(seq_counters) -> list[tuple[int, int]]: - """Find every place src_seq_ctr does not increment by exactly 1. - - This is the naive check (same logic as - ``imap_processing.utils._check_source_sequence_counter``) that first - surfaced the gaps -- it does not know about elapsed clock time, so a gap - here just means "not (previous + 1) % SEQ_CTR_MAX", real dropped packet - or not. - """ - gaps = [] - for i in range(1, len(seq_counters)): - expected_next = (int(seq_counters[i - 1]) + 1) % SEQ_CTR_MAX - if seq_counters[i] != expected_next: - gaps.append((int(seq_counters[i - 1]), int(seq_counters[i]))) - return gaps - - -def test_sequence_counter_gaps_match_elapsed_clock_time(xarray_data): - """Check that src_seq_ctr jumps are explained by elapsed sc_sclk_sec.""" - seq_counters = xarray_data["src_seq_ctr"].values - sclk_seconds = xarray_data["sc_sclk_sec"].values - - # sc_sclk_sec alone is only the coarse (integer-second) spacecraft clock - # count -- valid for elapsed-time math on its own, but not a real - # calendar time. Combine it with the fine time (sc_sclk_sub_sec) and run - # it through the SCLK kernel (via met_to_utc) to get an actual UTC - # timestamp for each packet. - met = calculate_time( - xarray_data["sc_sclk_sec"], xarray_data["sc_sclk_sub_sec"], SUB_SEC_CONVERSION - ) - utc_times = met_to_utc(met.values) - - gaps = _find_sequence_counter_gaps(seq_counters) - print(f"gaps = {gaps}") - - # Every gap found above should be fully explained by elapsed seconds on - # the onboard clock (accounting for 14-bit rollover) -- i.e. it reflects - # real missing/uncaptured packets, not counter corruption. - for i in range(1, len(seq_counters)): - elapsed_seconds = int(sclk_seconds[i]) - int(sclk_seconds[i - 1]) - expected_seq_ctr = (int(seq_counters[i - 1]) + elapsed_seconds) % SEQ_CTR_MAX - assert seq_counters[i] == expected_seq_ctr, ( - f"Packet {i}: src_seq_ctr {seq_counters[i]} does not match " - f"expected {expected_seq_ctr} given {elapsed_seconds}s elapsed " - f"on sc_sclk_sec" - ) - if seq_counters[i] != (int(seq_counters[i - 1]) + 1) % SEQ_CTR_MAX: - print( - f" gap: seq {seq_counters[i - 1]} ({utc_times[i - 1]} UTC) -> " - f"{seq_counters[i]} ({utc_times[i]} UTC)" - )