From c4c1d23a5389529aa09efef410154aff7188072a Mon Sep 17 00:00:00 2001 From: EllieKallmier <61219730+EllieKallmier@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:28:59 +1100 Subject: [PATCH 1/5] add non-rez geographies N0 and V0 explicitly to renewable_energy_zones iasr table --- src/ispypsa/templater/create_template.py | 7 +- src/ispypsa/templater/geography.py | 74 +++++++++++++++++++ ...create_ispypsa_inputs_new_table_formats.py | 48 ++++++------ .../test_create_ispypsa_inputs_template.py | 34 ++++++--- tests/test_templater/test_geography.py | 58 +++++++++++++++ 5 files changed, 183 insertions(+), 38 deletions(-) diff --git a/src/ispypsa/templater/create_template.py b/src/ispypsa/templater/create_template.py index cdc45900..40147e3d 100644 --- a/src/ispypsa/templater/create_template.py +++ b/src/ispypsa/templater/create_template.py @@ -26,7 +26,10 @@ _template_sub_regional_flow_path_costs, _template_sub_regional_flow_paths, ) -from ispypsa.templater.geography import _template_network_geography +from ispypsa.templater.geography import ( + _add_non_rez_geographies_to_rez_table, + _template_network_geography, +) from ispypsa.templater.helpers import _build_geo_region_lookup from ispypsa.templater.network_expansion import ( _extract_flow_path_costs_from_iasr, @@ -184,6 +187,8 @@ def create_ispypsa_inputs_template( # templater path) and inline this branch. if FEATURE_FLAGS["use_new_table_format"]: template = {} + # Explicitly add N0/V0 rows to the 'renewable_energy_zones' IASR table + iasr_tables = _add_non_rez_geographies_to_rez_table(iasr_tables) sub_regional_geography = _template_network_geography( iasr_tables["sub_regional_reference_nodes"], iasr_tables["renewable_energy_zones"], diff --git a/src/ispypsa/templater/geography.py b/src/ispypsa/templater/geography.py index 676ac4f2..ed9e8dca 100644 --- a/src/ispypsa/templater/geography.py +++ b/src/ispypsa/templater/geography.py @@ -2,6 +2,27 @@ from ispypsa.templater.mappings import _NEM_REGION_IDS, _SINGLE_REGION_ID +# Though not present in the renewable_energy_zones IASR table (v7.#), two additional +# 'non-REZ' areas are defined in the ISP to host lower resource quality new entrant VRE. +# See IASR 2026, p.164 for ISP reasoning. These get added to the 'renewable_energy_zones' +# table by `_add_non_rez_geographies_to_rez_table` before templating, to ensure +# the corresponding geo_ids exist and are mapped to the correct sub/NEM regions. +# See Open-ISP/ISPyPSA#133. +_NON_REZ_GEOGRAPHIES = [ + { + "ID": "N0", + "Name": "New South Wales Non-REZ", + "NEM region": "NSW", + "ISP sub-region": "CNSW", + }, + { + "ID": "V0", + "Name": "Victoria Non-REZ", + "NEM region": "VIC", + "ISP sub-region": "SEV", + }, +] + def _template_network_geography( sub_regional_reference_nodes: pd.DataFrame, @@ -274,3 +295,56 @@ def _rez_rows_for_single_region( def _extract_subregion_id(series: pd.Series) -> pd.Series: """Extract subregion ID from 'Name (ID)' format, e.g. 'Northern Queensland (NQ)' -> 'NQ'.""" return series.str.extract(r"\(([A-Z]+)\)", expand=False) + + +# --- non-REZ geographies --- + + +def _add_non_rez_geographies_to_rez_table( + iasr_tables: dict[str, pd.DataFrame], +) -> dict[str, pd.DataFrame]: + """Returns a shallow copy of iasr_tables with non-REZ areas N0 and V0 appended + to renewable_energy_zones. + + Raises: + ValueError: if any non-REZ ID is already present in renewable_energy_zones. + + I/O Example: + iasr_tables["renewable_energy_zones"]: + ID, Name, NEM region, ISP sub-region + Q1, Far North, QLD, NQ + N3, Central-West, NSW, CNSW + + returns: + iasr_tables["renewable_energy_zones"]: + ID, Name, NEM region, ISP sub-region + Q1, Far North, QLD, NQ + N3, Central-West, NSW, CNSW + N0, New South Wales Non-REZ, NSW, CNSW # added + V0, Victoria Non-REZ, VIC, SEV # added + + All other iasr_tables are returned unchanged. + """ + _assert_non_rezs_dont_exist(iasr_tables["renewable_energy_zones"]) + additional_geographies = pd.DataFrame(_NON_REZ_GEOGRAPHIES) + table = pd.concat( + [iasr_tables["renewable_energy_zones"], additional_geographies], + axis=0, + ignore_index=True, + ) + return {**iasr_tables, "renewable_energy_zones": table} + + +def _assert_non_rezs_dont_exist(table: pd.DataFrame) -> None: + """Raises if any non-REZ ID is already in the table, which would duplicate it.""" + additional_non_rez_ids = {row["ID"] for row in _NON_REZ_GEOGRAPHIES} + existing_ids = set(table["ID"].values) + + already_present = additional_non_rez_ids & existing_ids + if already_present: + msg = ( + f"Non-REZ ID(s) {sorted(already_present)} are already defined in " + "'renewable_energy_zones' IASR table. Check '_NON_REZ_GEOGRAPHIES' " + "is up-to-date." + ) + raise ValueError(msg) diff --git a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py index 815d705c..e68d0009 100644 --- a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py +++ b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py @@ -233,12 +233,14 @@ def test_create_ispypsa_inputs_task_new_format( _NUM_SUBREGIONS_75 = 15 _NUM_REGIONS_75 = 5 _NUM_REZS_75 = 47 +_NUM_NON_REZS_75 = 2 _NUM_FLOW_PATHS_75 = 18 # REZs whose row in `initial_transmission_limits` has all-NaN limit columns. # Each collapses to a single placeholder row in network_transmission_path_limits # instead of the usual `NUM_TIMESLICES × NUM_DIRECTIONS` rows. -_NUM_REZS_WITHOUT_LIMITS_75 = 15 +# All non-REZ geographies (N0/V0) have no limits so counts are added here. +_NUM_REZS_WITHOUT_LIMITS_75 = 15 + _NUM_NON_REZS_75 # Flow paths in the 7.5 input whose augmentation key has no exact match in # `flow_path_transfer_capability` and that therefore get a synthetic @@ -276,10 +278,13 @@ def test_create_ispypsa_inputs_task_new_format( _PATHS_PER_GRANULARITY_75 = { "sub_regions": ( - _NUM_FLOW_PATHS_75 + _NUM_REZS_75 + _NUM_PARALLEL_PATHS_INJECTED_75_SUB_REGIONS + _NUM_FLOW_PATHS_75 + + _NUM_REZS_75 + + _NUM_NON_REZS_75 + + _NUM_PARALLEL_PATHS_INJECTED_75_SUB_REGIONS ), - "nem_regions": _NUM_INTER_REGIONAL_FLOW_PATHS_75 + _NUM_REZS_75, - "single_region": _NUM_REZS_75, + "nem_regions": _NUM_INTER_REGIONAL_FLOW_PATHS_75 + _NUM_REZS_75 + _NUM_NON_REZS_75, + "single_region": _NUM_REZS_75 + _NUM_NON_REZS_75, } # Drift-detection only — per-option year coverage varies. @@ -307,10 +312,6 @@ def test_create_ispypsa_inputs_task_new_format( "single_region": 135, } -# "Non-REZ" placeholder REZ IDs used by generation technologies with no specific -# REZ location (state-wide VRE/DER). See Open-ISP/ISPyPSA#133 -_NON_REZ_PLACEHOLDER_GEO_IDS = {"N0", "V0"} - # Drift-detection only — no collapse step for existing/planned generators (unlike # new entrant), so this is the same at every granularity. _EXPECTED_GENERATORS_EXISTING_PLANNED_ROWS_75 = 531 @@ -321,8 +322,7 @@ def test_create_ispypsa_inputs_task_new_format( # REZ sub-zone ids used by some existing/planned generators that don't appear in # renewable_energy_zones (only their parent REZ does, e.g. "Q8" not "Q8a"/"Q8b"/ -# "Q8c"). Same gap class as _NON_REZ_PLACEHOLDER_GEO_IDS (#133), different cause. -# NOTE: placeholder fix for now - will be properly addressed in future PR (next up) +# "Q8c"). NOTE: placeholder fix for now - will be properly addressed in future PR (next up) _MISSING_REZ_SUBZONE_GEO_IDS = {"Q8a", "Q8b", "Q8c"} # Per-reference-year window patterns decoded from the shipped RefYear5000 @@ -382,7 +382,10 @@ def test_create_ispypsa_inputs_new_format( storage_existing_planned = pd.read_csv(output_dir / "storage_existing_planned.csv") # network_geography — one row per (sub-)region or REZ; geo_ids are unique. - assert len(geo) == _GEOS_PER_GRANULARITY_75[granularity] + _NUM_REZS_75 + assert ( + len(geo) + == _GEOS_PER_GRANULARITY_75[granularity] + _NUM_REZS_75 + _NUM_NON_REZS_75 + ) assert geo["geo_id"].is_unique # network_transmission_paths — derived from named structural quantities: @@ -423,14 +426,12 @@ def test_create_ispypsa_inputs_new_format( # generators_new_entrant / storage_new_entrant — row counts drop at coarser # granularities as sub-region-located technology options merge into one row # per region (REZ-located rows are unaffected). geo_ids are real network_geography - # entries, except the pre-existing "Non-REZ" placeholder gap (ISPyPSA#133) + # entries. assert ( len(gens_new_entrant) == _EXPECTED_GENERATORS_NEW_ENTRANT_ROWS_75[granularity] ) assert gens_new_entrant["name"].is_unique - assert set(gens_new_entrant["geo_id"]) <= ( - set(geo["geo_id"]) | _NON_REZ_PLACEHOLDER_GEO_IDS - ) + assert set(gens_new_entrant["geo_id"]) <= (set(geo["geo_id"])) assert ( len(storage_new_entrant) == _EXPECTED_STORAGE_NEW_ENTRANT_ROWS_75[granularity] @@ -440,30 +441,27 @@ def test_create_ispypsa_inputs_new_format( # generators_existing_planned — one row per existing/planned generating unit, # no collapse step (unlike new entrant) so the row count doesn't vary by - # granularity. geo_ids are real network_geography entries, except the two - # pre-existing gaps noted above ("Non-REZ" placeholders and REZ sub-zones). + # granularity. geo_ids are real network_geography entries, except the + # pre-existing gap noted above (REZ sub-zones). assert len(gens_existing_planned) == _EXPECTED_GENERATORS_EXISTING_PLANNED_ROWS_75 assert gens_existing_planned["name"].is_unique assert set(gens_existing_planned["geo_id"]) <= ( - set(geo["geo_id"]) | _NON_REZ_PLACEHOLDER_GEO_IDS | _MISSING_REZ_SUBZONE_GEO_IDS + set(geo["geo_id"]) | _MISSING_REZ_SUBZONE_GEO_IDS ) # as above - storage_existing_planned — one row per existing/planned storage unit, # no collapse step so the row count doesn't vary. geo_ids are real network_geography - # entries, except the two pre-existing gaps noted above ("Non-REZ" placeholders - # and REZ sub-zones). + # entries, except the pre-existing gap noted above (REZ sub-zones). assert len(storage_existing_planned) == _EXPECTED_STORAGE_EXISTING_PLANNED_ROWS_75 assert storage_existing_planned["name"].is_unique assert set(storage_existing_planned["geo_id"]) <= ( - set(geo["geo_id"]) | _NON_REZ_PLACEHOLDER_GEO_IDS | _MISSING_REZ_SUBZONE_GEO_IDS + set(geo["geo_id"]) | _MISSING_REZ_SUBZONE_GEO_IDS ) # costs_connection - every new entrant technology + geo_id + year has a row - # geo_id's are real network_geography entries + _NON_REZ_PLACEHOLDER_GEO_IDS + # geo_id's are real network_geography entries assert len(costs_connection) == _EXPECTED_COSTS_CONNECTION_ROWS_75[granularity] - assert set(costs_connection["geo_id"]) <= ( - set(geo["geo_id"]) | _NON_REZ_PLACEHOLDER_GEO_IDS - ) + assert set(costs_connection["geo_id"]) <= set(geo["geo_id"]) # timeslices — per-reference-year patterns decoded from the shipped # RefYear5000 calendar, identical at every granularity. diff --git a/tests/test_templater/test_create_ispypsa_inputs_template.py b/tests/test_templater/test_create_ispypsa_inputs_template.py index 62729baf..0517f638 100644 --- a/tests/test_templater/test_create_ispypsa_inputs_template.py +++ b/tests/test_templater/test_create_ispypsa_inputs_template.py @@ -318,12 +318,14 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): # (CNSW-SNW corridor) has valid endpoints in the geography. CQ is included so # the CQ-NQ flow path's region prefix resolves (every flow-path endpoint must # appear in the geography). + # SEV is added to ensure the non-REZ geo V0 has its endpoint present too. sub_regional_reference_nodes = csv_str_to_df(""" NEM region, ISP sub-region, Sub-regional reference node Queensland, Northern Queensland (NQ), Ross 275 kV Queensland, Central Queensland (CQ), Stanwell 275 kV New South Wales, Central New South Wales (CNSW), Wellington 330 kV New South Wales, Southern New South Wales (SNW), Lower Tumut 330 kV + Victoria, South East Victoria (SEV), Hazelwood 500kV """) renewable_energy_zones = csv_str_to_df(""" ID, Name, NEM region, ISP sub-region @@ -452,14 +454,15 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): geography = result["network_geography"] assert set(geography.columns) == {"geo_id", "geo_type", "region_id", "subregion_id"} - # 4 subregions (NQ + CQ + CNSW + SNW) + 2 REZs. - assert len(geography) == 6 + # 5 subregions (NQ + CQ + CNSW + SNW + SEV) + 2 REZs + 2 non-REZs (N0, V0) + assert len(geography) == 9 paths = result["network_transmission_paths"] assert set(paths.columns) == {"path_id", "geo_from", "geo_to", "carrier"} # CQ-NQ + 2 REZ connections (Q1-NQ, N3-CNSW) + CNSW-SNW_NTH + CNSW-SNW_STH # + CNSW-SNW (injected by _new_parallel_path_rows). - assert len(paths) == 6 + # + N0-CNSW and V0-SEV (non-REZ geos) + assert len(paths) == 8 # Specifically pin the new parallel-path row — a regression in the # _append_new_parallel_paths wiring would fail this assertion. assert "CNSW-SNW" in set(paths["path_id"]) @@ -469,7 +472,8 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): # 3 flow paths × 6 (CQ-NQ + 2 CNSW-SNW siblings) + Q1-NQ × 6 (REZ mirrored) # + N3-CNSW × 1 (collapsed, absent from initial_transmission_limits) # + CNSW-SNW × 6 zero-capacity rows from _new_parallel_path_rows. - assert len(limits) == 31 + # + N0-CNSW × 1 + V0-SEV × 1 + assert len(limits) == 33 expansion_options = result["network_expansion_options"] assert set(expansion_options.columns) == { @@ -625,6 +629,7 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): New South Wales, Central New South Wales (CNSW), Wellington 330 kV New South Wales, Northern NSW (NNSW), Armidale 330 kV New South Wales, Southern NSW (SNW), Lower Tumut 330 kV + Victoria, South East Victoria (SEV), Hazelwood 500kV """) renewable_energy_zones = csv_str_to_df(""" ID, Name, NEM region, ISP sub-region @@ -748,18 +753,21 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): geography = result["network_geography"] assert set(geography.columns) == {"geo_id", "geo_type", "region_id"} - assert len(geography) == 4 # 2 unique NEM regions (QLD, NSW) + 2 REZs + # 3 unique NEM regions (QLD, NSW, VIC) + 2 REZs + 2 non-REZs + assert len(geography) == 7 paths = result["network_transmission_paths"] assert set(paths.columns) == {"path_id", "geo_from", "geo_to", "carrier"} - # CQ-NQ (intra-QLD) dropped; NNSW-SQ (cross-region) kept; 2 REZ paths kept. - assert len(paths) == 3 + # CQ-NQ (intra-QLD) dropped; NNSW-SQ (cross-region) kept; + # 2 REZ and 2 non-REZ paths kept. + assert len(paths) == 5 limits = result["network_transmission_path_limits"] assert set(limits.columns) == {"path_id", "direction", "timeslice", "capacity"} # NNSW-SQ -> NSW-QLD: 6 rows. Q1-QLD: 6 rows. N3-NSW: 1 collapsed row. - assert len(limits) == 13 + # N0-NSW: 1 collapsed row. V0-VIC: 1 collapsed row. + assert len(limits) == 15 expansion_options = result["network_expansion_options"] # CQ-NQ augmentation dropped (intra-QLD); NNSW-SQ re-keyed to NSW-QLD; @@ -827,6 +835,7 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): Queensland, Central Queensland (CQ), Stanwell 275 kV New South Wales, Central New South Wales (CNSW), Wellington 330 kV New South Wales, Southern NSW (SNW), Lower Tumut 330 kV + Victoria, South East Victoria (SEV), Hazelwood 500kV """) renewable_energy_zones = csv_str_to_df(""" ID, Name, NEM region, ISP sub-region @@ -939,17 +948,18 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): geography = result["network_geography"] assert set(geography.columns) == {"geo_id", "geo_type", "region_id"} - assert len(geography) == 3 # 1 NEM row + 2 REZs + assert len(geography) == 5 # 1 NEM row + 2 REZs + 2 non-REZs paths = result["network_transmission_paths"] assert set(paths.columns) == {"path_id", "geo_from", "geo_to", "carrier"} - # CQ-NQ flow path dropped; only the two REZ-to-NEM paths remain. - assert len(paths) == 2 + # CQ-NQ flow path dropped; only two REZ-to-NEM and two non-REZ-to-NEM paths remain. + assert len(paths) == 4 limits = result["network_transmission_path_limits"] assert set(limits.columns) == {"path_id", "direction", "timeslice", "capacity"} # Q1-NEM: 6 rows. N3-NEM: 1 collapsed row. - assert len(limits) == 7 + # N0-NEM: 1 collapsed row. V0-NEM: 1 collapsed row. + assert len(limits) == 9 expansion_options = result["network_expansion_options"] # CQ-NQ augmentation dropped entirely (no flow paths exist at single_region). diff --git a/tests/test_templater/test_geography.py b/tests/test_templater/test_geography.py index 04d46506..33939f04 100644 --- a/tests/test_templater/test_geography.py +++ b/tests/test_templater/test_geography.py @@ -1,6 +1,10 @@ +import re + import pandas as pd +import pytest from ispypsa.templater.geography import ( + _add_non_rez_geographies_to_rez_table, _extract_subregion_id, _template_network_geography, ) @@ -102,3 +106,57 @@ def test_template_network_geography_single_region(csv_str_to_df): result.reset_index(drop=True), expected.reset_index(drop=True), ) + + +# --- _add_non_rez_geographies_to_rez_table --- + + +def test_add_non_rez_geographies_to_rez_table(csv_str_to_df): + renewable_energy_zones = csv_str_to_df(""" + ID, Name, NEM region, ISP sub-region + Q1, Far North QLD, QLD, NQ + N3, Central-West Orana, NSW, CNSW + """) + + iasr_tables = { + "renewable_energy_zones": renewable_energy_zones, + "another_table": csv_str_to_df(""" + First Column, Second Column + number_one, number_two + """), + } + + updated_tables = _add_non_rez_geographies_to_rez_table(iasr_tables) + + # check that returned renewable_energy_zones table has extra rows + expected_rez = csv_str_to_df(""" + ID, Name, NEM region, ISP sub-region + Q1, Far North QLD, QLD, NQ + N3, Central-West Orana, NSW, CNSW + N0, New South Wales Non-REZ, NSW, CNSW + V0, Victoria Non-REZ, VIC, SEV + """) + pd.testing.assert_frame_equal( + updated_tables["renewable_energy_zones"], expected_rez + ) + + # and that the other table passed through untouched + assert updated_tables["another_table"] is iasr_tables["another_table"] + assert iasr_tables["renewable_energy_zones"] is renewable_energy_zones + + +def test_add_non_rez_geographies_to_rez_table_guard_raises(csv_str_to_df): + # check that the guard raises correctly based on ID presence, even if other + # column values for that non-REZ row are different. + renewable_energy_zones = csv_str_to_df(""" + ID, Name, NEM region, ISP sub-region + N3, Central-West Orana, NSW, CNSW + N0, New Non-REZ Name, NSW, SNSW + """) + + iasr_tables = { + "renewable_energy_zones": renewable_energy_zones, + } + expected_msg = re.escape("Non-REZ ID(s) ['N0'] are already defined") + with pytest.raises(ValueError, match=expected_msg): + _add_non_rez_geographies_to_rez_table(iasr_tables) From 15afff90e7d45530bbbb17301ad16581e93b4d59 Mon Sep 17 00:00:00 2001 From: EllieKallmier <61219730+EllieKallmier@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:30:27 +1100 Subject: [PATCH 2/5] exclude N0/V0 non-rez geos from missing initial transmission limits log (expected) --- src/ispypsa/templater/transmission.py | 13 +++++++++++-- tests/test_templater/test_transmission.py | 17 +++++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/ispypsa/templater/transmission.py b/src/ispypsa/templater/transmission.py index e7ac6eac..ade842da 100644 --- a/src/ispypsa/templater/transmission.py +++ b/src/ispypsa/templater/transmission.py @@ -2,6 +2,7 @@ import pandas as pd +from ispypsa.templater.geography import _NON_REZ_GEOGRAPHIES from ispypsa.templater.helpers import _build_geo_region_lookup from ispypsa.templater.mappings import _CANONICAL_TIMESLICES, _SINGLE_REGION_ID @@ -605,10 +606,18 @@ def _log_rez_paths_absent_from_limits( """Logs REZ paths whose REZ ID has no row in initial_transmission_limits. These paths get NaN capacity rows from the left-merge, are collapsed by - ``_collapse_paths_with_no_limits``, and receive a default downstream. + ``_collapse_paths_with_no_limits``, and receive a default downstream. Non-REZ + geo_ids that have been added to 'renewable_energy_zones' (N0 and V0, see + ``geography._add_non_rez_geographies_to_rez_table``) are expected to have no + initial limit, so are excluded from this log. """ rez_with_limits = set(initial_transmission_limits["REZ ID"]) - missing = sorted(set(rez_connection_rows["geo_from"]) - rez_with_limits) + known_non_rezs_with_no_limits = {row["ID"] for row in _NON_REZ_GEOGRAPHIES} + missing = sorted( + set(rez_connection_rows["geo_from"]) + - rez_with_limits + - known_non_rezs_with_no_limits + ) if missing: logging.warning( f"REZs absent from initial_transmission_limits " diff --git a/tests/test_templater/test_transmission.py b/tests/test_templater/test_transmission.py index c6a1173a..201fa0bd 100644 --- a/tests/test_templater/test_transmission.py +++ b/tests/test_templater/test_transmission.py @@ -404,17 +404,30 @@ def test_logs_rez_paths_absent_from_initial_limits(csv_str_to_df, caplog): ID, Name, NEM region, ISP sub-region Q1, Far North QLD, QLD, NQ N3, Central-West, NSW, CNSW + N0, NSW Non-REZ, NSW, CNSW + V0, VIC Non-REZ, VIC, SEV """) + # add non-REZs to sub_regional_geography stub to avoid 'Path geos missing' error + sub_regional_geography = _sub_regional_geography() + non_rezs = csv_str_to_df(""" + geo_id, geo_type, region_id, subregion_id + N0, rez, NSW, CNSW + V0, rez, VIC, SEV + """) + sub_regional_geo_with_non_rezs = pd.concat( + [sub_regional_geography, non_rezs], axis=0, ignore_index=True + ) + with caplog.at_level("WARNING"): _template_network_transmission( flow_paths, initial_limits, renewable_energy_zones, - _sub_regional_geography(), + sub_regional_geo_with_non_rezs, "sub_regions", ) - + # known exceptions 'N0' and 'V0' should not be logged here - only 'N3' assert ( "REZs absent from initial_transmission_limits " "(default will be applied downstream): ['N3']" From 8bca1ce7f473c544ef75a13d5cd9b62ce6f49684 Mon Sep 17 00:00:00 2001 From: EllieKallmier <61219730+EllieKallmier@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:40:33 +1100 Subject: [PATCH 3/5] add iasr table replacement: Q8a/b/c -> Q8 in ECAA summary table 'REZ ID' col --- src/ispypsa/templater/existing_planned.py | 7 +++++++ .../test_create_ispypsa_inputs_new_table_formats.py | 13 ++----------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/ispypsa/templater/existing_planned.py b/src/ispypsa/templater/existing_planned.py index 3dbf2cfa..19ba8f8d 100644 --- a/src/ispypsa/templater/existing_planned.py +++ b/src/ispypsa/templater/existing_planned.py @@ -144,6 +144,13 @@ column="IASR ID", replacements={"KiataWF1": "KIATAWF1"}, ), + # Sub-REZ IDs Q8(a/b/c) in ECAA summary table 'REZ ID' column are not real REZs, + # but IDs used for resource-quality based build limits inside the Q8 REZ. + dict( + table_name="existing_committed_anticipated_additional_generator_summary", + column="REZ ID", + replacements={"Q8a": "Q8", "Q8b": "Q8", "Q8c": "Q8"}, + ), ] # --- public orchestrators --- diff --git a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py index e68d0009..1800070c 100644 --- a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py +++ b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py @@ -320,11 +320,6 @@ def test_create_ispypsa_inputs_task_new_format( # remains the same at every granularity. _EXPECTED_STORAGE_EXISTING_PLANNED_ROWS_75 = 111 -# REZ sub-zone ids used by some existing/planned generators that don't appear in -# renewable_energy_zones (only their parent REZ does, e.g. "Q8" not "Q8a"/"Q8b"/ -# "Q8c"). NOTE: placeholder fix for now - will be properly addressed in future PR (next up) -_MISSING_REZ_SUBZONE_GEO_IDS = {"Q8a", "Q8b", "Q8c"} - # Per-reference-year window patterns decoded from the shipped RefYear5000 # calendar and reference_year_sequence (drift-detection; # granularity-invariant). 14 distinct ids, not 15: TAS Hot Day never @@ -445,18 +440,14 @@ def test_create_ispypsa_inputs_new_format( # pre-existing gap noted above (REZ sub-zones). assert len(gens_existing_planned) == _EXPECTED_GENERATORS_EXISTING_PLANNED_ROWS_75 assert gens_existing_planned["name"].is_unique - assert set(gens_existing_planned["geo_id"]) <= ( - set(geo["geo_id"]) | _MISSING_REZ_SUBZONE_GEO_IDS - ) + assert set(gens_existing_planned["geo_id"]) <= set(geo["geo_id"]) # as above - storage_existing_planned — one row per existing/planned storage unit, # no collapse step so the row count doesn't vary. geo_ids are real network_geography # entries, except the pre-existing gap noted above (REZ sub-zones). assert len(storage_existing_planned) == _EXPECTED_STORAGE_EXISTING_PLANNED_ROWS_75 assert storage_existing_planned["name"].is_unique - assert set(storage_existing_planned["geo_id"]) <= ( - set(geo["geo_id"]) | _MISSING_REZ_SUBZONE_GEO_IDS - ) + assert set(storage_existing_planned["geo_id"]) <= set(geo["geo_id"]) # costs_connection - every new entrant technology + geo_id + year has a row # geo_id's are real network_geography entries From 429295cd17c9c83a863e246b3d6f3a09edefd157 Mon Sep 17 00:00:00 2001 From: EllieKallmier <61219730+EllieKallmier@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:01:38 +1100 Subject: [PATCH 4/5] move _NON_REZ_GEOGRAPHIES to mappings, add logging --- src/ispypsa/templater/geography.py | 52 +++++++++++++------------- src/ispypsa/templater/mappings.py | 23 ++++++++++++ src/ispypsa/templater/transmission.py | 9 +++-- tests/test_templater/test_geography.py | 12 +++++- 4 files changed, 64 insertions(+), 32 deletions(-) diff --git a/src/ispypsa/templater/geography.py b/src/ispypsa/templater/geography.py index ed9e8dca..b3602226 100644 --- a/src/ispypsa/templater/geography.py +++ b/src/ispypsa/templater/geography.py @@ -1,27 +1,12 @@ +import logging + import pandas as pd -from ispypsa.templater.mappings import _NEM_REGION_IDS, _SINGLE_REGION_ID - -# Though not present in the renewable_energy_zones IASR table (v7.#), two additional -# 'non-REZ' areas are defined in the ISP to host lower resource quality new entrant VRE. -# See IASR 2026, p.164 for ISP reasoning. These get added to the 'renewable_energy_zones' -# table by `_add_non_rez_geographies_to_rez_table` before templating, to ensure -# the corresponding geo_ids exist and are mapped to the correct sub/NEM regions. -# See Open-ISP/ISPyPSA#133. -_NON_REZ_GEOGRAPHIES = [ - { - "ID": "N0", - "Name": "New South Wales Non-REZ", - "NEM region": "NSW", - "ISP sub-region": "CNSW", - }, - { - "ID": "V0", - "Name": "Victoria Non-REZ", - "NEM region": "VIC", - "ISP sub-region": "SEV", - }, -] +from ispypsa.templater.mappings import ( + _NEM_REGION_IDS, + _NON_REZ_GEOGRAPHIES, + _SINGLE_REGION_ID, +) def _template_network_geography( @@ -310,6 +295,11 @@ def _add_non_rez_geographies_to_rez_table( ValueError: if any non-REZ ID is already present in renewable_energy_zones. I/O Example: + _NON_REZ_GEOGRAPHIES (constant, mappings.py): + ID, Name, NEM region, ISP sub-region + N0, New South Wales Non-REZ, NSW, CNSW + V0, Victoria Non-REZ, VIC, SEV + iasr_tables["renewable_energy_zones"]: ID, Name, NEM region, ISP sub-region Q1, Far North, QLD, NQ @@ -326,20 +316,19 @@ def _add_non_rez_geographies_to_rez_table( All other iasr_tables are returned unchanged. """ _assert_non_rezs_dont_exist(iasr_tables["renewable_energy_zones"]) - additional_geographies = pd.DataFrame(_NON_REZ_GEOGRAPHIES) table = pd.concat( - [iasr_tables["renewable_energy_zones"], additional_geographies], + [iasr_tables["renewable_energy_zones"], _NON_REZ_GEOGRAPHIES], axis=0, ignore_index=True, ) + _log_non_rezs_added_to_rez_table() return {**iasr_tables, "renewable_energy_zones": table} def _assert_non_rezs_dont_exist(table: pd.DataFrame) -> None: """Raises if any non-REZ ID is already in the table, which would duplicate it.""" - additional_non_rez_ids = {row["ID"] for row in _NON_REZ_GEOGRAPHIES} - existing_ids = set(table["ID"].values) - + additional_non_rez_ids = set(_NON_REZ_GEOGRAPHIES["ID"]) + existing_ids = set(table["ID"]) already_present = additional_non_rez_ids & existing_ids if already_present: msg = ( @@ -348,3 +337,12 @@ def _assert_non_rezs_dont_exist(table: pd.DataFrame) -> None: "is up-to-date." ) raise ValueError(msg) + + +def _log_non_rezs_added_to_rez_table() -> None: + """Logs non-REZs added to the renewable_energy_zones table.""" + msg = ( + "IASR TABLE EDIT: Added non-REZ geographies to 'renewable_energy_zones' " + f"IASR table: {sorted(_NON_REZ_GEOGRAPHIES['ID'])}" + ) + logging.info(msg) diff --git a/src/ispypsa/templater/mappings.py b/src/ispypsa/templater/mappings.py index ae6222ec..ff2b8818 100644 --- a/src/ispypsa/templater/mappings.py +++ b/src/ispypsa/templater/mappings.py @@ -663,6 +663,29 @@ # lowercased NEM region id. _CANONICAL_TIMESLICES = ("peak_demand", "summer_typical", "winter_reference") +# Two non-REZ areas are defined in the ISP to host lower resource quality new +# entrant VRE (see IASR 2026, p.164 for ISP reasoning). They are not present in the +# 'renewable_energy_zones' table in the IASR workbook, so they get manually added +# by `_add_non_rez_geographies_to_rez_table` before templating, to ensure +# the corresponding geo_ids exist and are mapped to the correct sub/NEM regions. +# See Open-ISP/ISPyPSA#133. +_NON_REZ_GEOGRAPHIES = pd.DataFrame( + [ + { + "ID": "N0", + "Name": "New South Wales Non-REZ", + "NEM region": "NSW", + "ISP sub-region": "CNSW", + }, + { + "ID": "V0", + "Name": "Victoria Non-REZ", + "NEM region": "VIC", + "ISP sub-region": "SEV", + }, + ] +) + """ New entrant property columns (keys) mapped to the IASR table and columns that contain property values and the technology for which the values apply. Consumed by diff --git a/src/ispypsa/templater/transmission.py b/src/ispypsa/templater/transmission.py index ade842da..d5755724 100644 --- a/src/ispypsa/templater/transmission.py +++ b/src/ispypsa/templater/transmission.py @@ -2,9 +2,12 @@ import pandas as pd -from ispypsa.templater.geography import _NON_REZ_GEOGRAPHIES from ispypsa.templater.helpers import _build_geo_region_lookup -from ispypsa.templater.mappings import _CANONICAL_TIMESLICES, _SINGLE_REGION_ID +from ispypsa.templater.mappings import ( + _CANONICAL_TIMESLICES, + _NON_REZ_GEOGRAPHIES, + _SINGLE_REGION_ID, +) _HVDC_PATH_IDS = {"NNSW-SQ_Terranora", "WNV-CSA_Murraylink", "TAS-SEV"} @@ -612,7 +615,7 @@ def _log_rez_paths_absent_from_limits( initial limit, so are excluded from this log. """ rez_with_limits = set(initial_transmission_limits["REZ ID"]) - known_non_rezs_with_no_limits = {row["ID"] for row in _NON_REZ_GEOGRAPHIES} + known_non_rezs_with_no_limits = set(_NON_REZ_GEOGRAPHIES["ID"]) missing = sorted( set(rez_connection_rows["geo_from"]) - rez_with_limits diff --git a/tests/test_templater/test_geography.py b/tests/test_templater/test_geography.py index 33939f04..41d19f80 100644 --- a/tests/test_templater/test_geography.py +++ b/tests/test_templater/test_geography.py @@ -111,7 +111,7 @@ def test_template_network_geography_single_region(csv_str_to_df): # --- _add_non_rez_geographies_to_rez_table --- -def test_add_non_rez_geographies_to_rez_table(csv_str_to_df): +def test_add_non_rez_geographies_to_rez_table(csv_str_to_df, caplog): renewable_energy_zones = csv_str_to_df(""" ID, Name, NEM region, ISP sub-region Q1, Far North QLD, QLD, NQ @@ -126,7 +126,8 @@ def test_add_non_rez_geographies_to_rez_table(csv_str_to_df): """), } - updated_tables = _add_non_rez_geographies_to_rez_table(iasr_tables) + with caplog.at_level("INFO"): + updated_tables = _add_non_rez_geographies_to_rez_table(iasr_tables) # check that returned renewable_energy_zones table has extra rows expected_rez = csv_str_to_df(""" @@ -140,6 +141,13 @@ def test_add_non_rez_geographies_to_rez_table(csv_str_to_df): updated_tables["renewable_energy_zones"], expected_rez ) + # and that the additions were logged: + msg = ( + "IASR TABLE EDIT: Added non-REZ geographies to 'renewable_energy_zones' " + "IASR table: ['N0', 'V0']" + ) + assert msg in caplog.text + # and that the other table passed through untouched assert updated_tables["another_table"] is iasr_tables["another_table"] assert iasr_tables["renewable_energy_zones"] is renewable_energy_zones From 2cfdab7d47324cb632b583eee30949168761761d Mon Sep 17 00:00:00 2001 From: EllieKallmier <61219730+EllieKallmier@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:04:28 +1100 Subject: [PATCH 5/5] remove stale comments --- .../test_create_ispypsa_inputs_new_table_formats.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py index 1800070c..249bdadc 100644 --- a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py +++ b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py @@ -436,21 +436,20 @@ def test_create_ispypsa_inputs_new_format( # generators_existing_planned — one row per existing/planned generating unit, # no collapse step (unlike new entrant) so the row count doesn't vary by - # granularity. geo_ids are real network_geography entries, except the - # pre-existing gap noted above (REZ sub-zones). + # granularity. geo_ids are real network_geography entries. assert len(gens_existing_planned) == _EXPECTED_GENERATORS_EXISTING_PLANNED_ROWS_75 assert gens_existing_planned["name"].is_unique assert set(gens_existing_planned["geo_id"]) <= set(geo["geo_id"]) # as above - storage_existing_planned — one row per existing/planned storage unit, # no collapse step so the row count doesn't vary. geo_ids are real network_geography - # entries, except the pre-existing gap noted above (REZ sub-zones). + # entries. assert len(storage_existing_planned) == _EXPECTED_STORAGE_EXISTING_PLANNED_ROWS_75 assert storage_existing_planned["name"].is_unique assert set(storage_existing_planned["geo_id"]) <= set(geo["geo_id"]) # costs_connection - every new entrant technology + geo_id + year has a row - # geo_id's are real network_geography entries + # geo_id's are real network_geography entries. assert len(costs_connection) == _EXPECTED_COSTS_CONNECTION_ROWS_75[granularity] assert set(costs_connection["geo_id"]) <= set(geo["geo_id"])