From 33ecde93c394f603f6fc4a56177979e9ba7e85fa Mon Sep 17 00:00:00 2001 From: Joshua Hampton Date: Wed, 26 Aug 2026 11:14:50 +0100 Subject: [PATCH 1/6] Create abstract base class and refactor readers --- checksit/readers/badc_csv.py | 30 +++++++++++-- checksit/readers/base.py | 28 ++++++++++++ checksit/readers/cdl.py | 86 +++++++++++++++++++----------------- checksit/readers/image.py | 68 ++++++++++++++-------------- checksit/readers/pp.py | 41 ++++++++++++++++- checksit/readers/yml.py | 34 +++++++++++++- 6 files changed, 207 insertions(+), 80 deletions(-) create mode 100644 checksit/readers/base.py diff --git a/checksit/readers/badc_csv.py b/checksit/readers/badc_csv.py index 332e73ef..d08fcce5 100644 --- a/checksit/readers/badc_csv.py +++ b/checksit/readers/badc_csv.py @@ -1,6 +1,8 @@ #https://github.com/cedadev/badc-csv/blob/main/badctextfile.py from .badctextfile import BADCTextFile - +from .base import BaseReader +from typing import List, Dict +""" req_dicts = "dimensions", "variables", "global_attributes" @@ -21,5 +23,27 @@ def read(fpath: str, verbose: bool = False) -> BADCCSVHeader: d = {"global_attributes": dict(bm.globalRecords)} # "variables": bm.varRecords} return BADCCSVHeader(fpath, d) - - +""" +class BADCCSVHeader(BaseReader): + def __init__( + self, + inpt: str, + verbose: bool = False, + ) -> None: + """Initialise the BADCCSVHeader. + + Args: + inpt: The input file path. + verbose: Print verbose output during parsing + """ + self.inpt = inpt + self.verbose = verbose + self.fmt_errors: List[str] = [] + self.global_attrs: Dict[str, str] = {} + self.dimensions: Dict[str, str] = {} + self.variables: Dict[str, Dict[str, str]] = {} + + def read(self) -> None: + """Read BADC CSV file""" + content = BADCTextFile(open(self.inpt))._metadata + self.global_attrs = dict(content.globalRecords) diff --git a/checksit/readers/base.py b/checksit/readers/base.py new file mode 100644 index 00000000..99dacc0b --- /dev/null +++ b/checksit/readers/base.py @@ -0,0 +1,28 @@ +from abc import ABC, abstractmethod +from typing import List, Dict, Union + +class BaseReader(ABC): + inpt: str + verbose: bool + fmt_errors: List[str] + global_attrs: Dict[str, str] + dimensions: Dict[str, str] + variables: Dict[str, Dict[str, str]] + + @abstractmethod + def read(self) -> None: + """Read file""" + pass + + def to_dict(self) -> Dict[str, Union[Dict[str, str], Dict[str, Dict[str, str]], str]]: + """Convert parsed data into dict + + Returns: + dictionary mess + """ + return { + "dimensions": self.dimensions, + "variables": self.variables, + "global_attributes": self.global_attrs, + "inpt": self.inpt, + } diff --git a/checksit/readers/cdl.py b/checksit/readers/cdl.py index f3c75b4e..97674ef2 100644 --- a/checksit/readers/cdl.py +++ b/checksit/readers/cdl.py @@ -5,9 +5,10 @@ import yaml import subprocess as sp import sys -from typing import Tuple, List, Dict, Union +from typing import Tuple, List, Dict from ..cvs import vocabs, vocabs_prefix +from .base import BaseReader def get_output(cmd: str) -> str: @@ -23,7 +24,7 @@ def get_output(cmd: str) -> str: return subp.stdout.read().decode("utf-8") -class CDLParser: +class CDLParser(BaseReader): """Parse a CDL file or netCDF file into dictionaries. Extract information from netCDF files or CDL files into a dictionaries for @@ -47,7 +48,7 @@ def __init__( inpt: str, verbose: bool = False, ) -> None: - """Initialise the CDLParser and parse the input file. + """Initialise the CDLParser. Args: inpt: The input file path or CDL content. @@ -55,24 +56,27 @@ def __init__( """ self.inpt = inpt self.verbose = verbose - self.fmt_errors = [] - self._parse(inpt) - self._check_format() - - def _parse(self, inpt: str) -> None: + self.fmt_errors: List[str] = [] + self.global_attrs: Dict[str, str] = {} + self.dimensions: Dict[str, str] = {} + self.variables: Dict[str, Dict[str, str]] = {} + #self.read() + #self._check_format() + + def read(self) -> None: """Parse the input file or CDL content into dictionaries. Args: inpt: The input file path or CDL content. """ if self.verbose: - print(f"[INFO] Parsing input: {inpt[:100]}...") - if inpt.endswith(".nc"): - self.cdl = get_output(f"ncdump -h {inpt}") - elif inpt.endswith(".cdl"): - self.cdl = open(inpt).read() + print(f"[INFO] Parsing input: {self.inpt[:100]}...") + if self.inpt.endswith(".nc"): + self.cdl = get_output(f"ncdump -h {self.inpt}") + elif self.inpt.endswith(".cdl"): + self.cdl = open(self.inpt).read() else: - self.cdl = inpt + self.cdl = self.inpt cdl_lines: List[str] = self.cdl.strip().split("\n") @@ -343,30 +347,30 @@ def to_yaml(self) -> str: sort_keys=False, ) - def to_dict(self) -> Dict[str, Union[Dict[str, str], Dict[str, Dict[str, str]], str, List[str]]]: - """Return the parsed CDL content as a dictionary. - - Returns: - A dictionary of the parsed CDL content, with keys "dimensions", - "variables", "global_attributes" and "inpt", where "inpt" is the input - file path or CDL content. - """ - return { - "dimensions": self.dimensions, - "variables": self.variables, - "global_attributes": self.global_attrs, - "inpt": self.inpt, - } - - -def read(fpath: str, verbose: bool = False) -> CDLParser: - """Read a CDL file or netCDF file and parse it into a CDLParser object. - - Args: - fpath: The file path to read. - verbose: Print verbose output during parsing. - - Returns: - A CDLParser object containing the parsed CDL content. - """ - return CDLParser(fpath, verbose=verbose) +# def to_dict(self) -> Dict[str, Union[Dict[str, str], Dict[str, Dict[str, str]], str, List[str]]]: +# """Return the parsed CDL content as a dictionary. +# +# Returns: +# A dictionary of the parsed CDL content, with keys "dimensions", +# "variables", "global_attributes" and "inpt", where "inpt" is the input +# file path or CDL content. +# """ +# return { +# "dimensions": self.dimensions, +# "variables": self.variables, +# "global_attributes": self.global_attrs, +# "inpt": self.inpt, +# } + + +#def read(fpath: str, verbose: bool = False) -> CDLParser: +# """Read a CDL file or netCDF file and parse it into a CDLParser object. +# +# Args: +# fpath: The file path to read. +# verbose: Print verbose output during parsing. +# +# Returns: +# A CDLParser object containing the parsed CDL content. +# """ +# return CDLParser(fpath, verbose=verbose) diff --git a/checksit/readers/image.py b/checksit/readers/image.py index 2d7fd733..c74a09bb 100644 --- a/checksit/readers/image.py +++ b/checksit/readers/image.py @@ -2,7 +2,8 @@ """ import subprocess as sp import yaml -from typing import Tuple, Dict, Union +from typing import Tuple, Dict +from .base import BaseReader def get_output(cmd: str) -> Tuple[str, str]: """Get the output of a shell command. @@ -17,7 +18,7 @@ def get_output(cmd: str) -> Tuple[str, str]: return subp.stdout.read().decode("charmap"), subp.stderr.read().decode("charmap") -class ImageParser: +class ImageParser(BaseReader): """Parse an image file into dictionaries. Extract information from an image file into a dictionary for tags, labelled as @@ -38,7 +39,7 @@ def __init__( inpt: str, verbose: bool = False ) -> None: - """Initialise the ImageParser and parse the input file. + """Initialise the ImageParser. Args: inpt: The input file path. @@ -46,20 +47,19 @@ def __init__( """ self.inpt = inpt self.verbose = verbose - self.base_exiftool_arguments = ["exiftool", "-G1", "-j", "-c", "%+.6f"] + self._base_exiftool_arguments = ["exiftool", "-G1", "-j", "-c", "%+.6f"] + self.global_attrs: Dict[str, str] = {} + self.dimensions: Dict[str, str] = {} + self.variables: Dict[str, Dict[str, str]] = {} self._find_exiftool() - self._parse(inpt) + #self.read() - def _parse(self, inpt: str) -> None: - """Parse the input file using exiftool. - - Args: - inpt: The input file path. - """ + def read(self) -> None: + """Parse the input file using exiftool.""" if self.verbose: - print(f"[INFO] Parsing input: {inpt[:100]}...") + print(f"[INFO] Parsing input: {self.inpt[:100]}...") self.global_attrs = {} - exiftool_arguments = self.base_exiftool_arguments + [inpt] + exiftool_arguments = self._base_exiftool_arguments + [self.inpt] exiftool_return_string = sp.check_output(exiftool_arguments) raw_global_attrs = yaml.load(exiftool_return_string, Loader=yaml.SafeLoader)[0] for tag_name in raw_global_attrs.keys(): @@ -100,24 +100,24 @@ def _attrs_dict(self, content_lines): attr_dict[key] = value return attr_dict - def to_dict(self) -> Dict[str, Union[str, Dict[str, str]]]: - """Convert the ImageParser object data to a dictionary. - - Returns: - Dictionary containing metadata tags and values as "global_attributes", and - the input file path as "inpt". - """ - return {"global_attributes": self.global_attrs, "inpt": self.inpt} - - -def read(fpath: str, verbose: bool = False) -> ImageParser: - """Read an image file and return an ImageParser object. - - Args: - fpath: The path to the image file. - verbose: Print verbose output during parsing. - - Returns: - An ImageParser object containing the metadata tags and values. - """ - return ImageParser(fpath, verbose=verbose) +# def to_dict(self) -> Dict[str, Union[str, Dict[str, str]]]: +# """Convert the ImageParser object data to a dictionary. +# +# Returns: +# Dictionary containing metadata tags and values as "global_attributes", and +# the input file path as "inpt". +# """ +# return {"global_attributes": self.global_attrs, "inpt": self.inpt} + + +#def read(fpath: str, verbose: bool = False) -> ImageParser: +# """Read an image file and return an ImageParser object. +# +# Args: +# fpath: The path to the image file. +# verbose: Print verbose output during parsing. +# +# Returns: +# An ImageParser object containing the metadata tags and values. +# """ +# return ImageParser(fpath, verbose=verbose) diff --git a/checksit/readers/pp.py b/checksit/readers/pp.py index 7c684a0e..eabbc148 100644 --- a/checksit/readers/pp.py +++ b/checksit/readers/pp.py @@ -1,6 +1,8 @@ -import sys import cf +from .base import BaseReader +from typing import List, Dict +""" req_dicts = "dimensions", "variables", "global_attributes" class PPHeader: @@ -26,5 +28,42 @@ def read(fpath: str, verbose: bool = False) -> PPHeader: d["variables"][sn] = {"shape": sh} return PPHeader(fpath, d) +""" +class PPHeader(BaseReader): + def __init__( + self, + inpt: str, + verbose: bool = False, + ) -> None: + """Initialise the PPHeader. + + Args: + inpt: The input file path. + verbose: Print verbose output during parsing + """ + self.inpt = inpt + self.verbose = verbose + self.fmt_errors: List[str] = [] + self.global_attrs: Dict[str, str] = {} + self.dimensions: Dict[str, str] = {} + self.variables: Dict[str, Dict[str, str]] = {} + + def read(self) -> None: + """Read YAML file""" + fieldlist = cf.read(self.inpt) + content = {"variables": {}} + + for field in fieldlist: + sn = field.standard_name + sh = list(field.shape) + + content["variables"][sn] = {"shape": sh} + + if "global_attributes" in content: + self.global_attrs = content["global_attributes"] + if "dimensions" in content: + self.dimensions = content["dimensions"] + if "variables" in content: + self.variables = content["variables"] diff --git a/checksit/readers/yml.py b/checksit/readers/yml.py index fe4b60f5..2eab70d0 100644 --- a/checksit/readers/yml.py +++ b/checksit/readers/yml.py @@ -1,5 +1,7 @@ import yaml - +from typing import List, Dict +from .base import BaseReader +""" req_dicts = "dimensions", "variables", "global_attributes" class YAMLFile: @@ -17,5 +19,35 @@ def to_dict(self): def read(fpath: str, verbose: bool = False) -> YAMLFile: d = yaml.load(open(fpath), Loader=yaml.SafeLoader) return YAMLFile(fpath, d) +""" + +class YAMLFile(BaseReader): + def __init__( + self, + inpt: str, + verbose: bool = False, + ) -> None: + """Initialise the YAMLParser. + + Args: + inpt: The input file path. + verbose: Print verbose output during parsing + """ + self.inpt = inpt + self.verbose = verbose + self.fmt_errors: List[str] = [] + self.global_attrs: Dict[str, str] = {} + self.dimensions: Dict[str, str] = {} + self.variables: Dict[str, Dict[str, str]] = {} + + def read(self) -> None: + """Read YAML file""" + content = yaml.load(open(self.inpt), Loader=yaml.SafeLoader) + if "global_attributes" in content: + self.global_attrs = content["global_attributes"] + if "dimensions" in content: + self.dimensions = content["dimensions"] + if "variables" in content: + self.variables = content["variables"] From b4727f0a55da742b5206ee1a91df2ae0b1dccb7f Mon Sep 17 00:00:00 2001 From: Joshua Hampton Date: Wed, 26 Aug 2026 11:19:32 +0100 Subject: [PATCH 2/6] Use new reader classes --- checksit/check.py | 23 ++++++++++++----------- tests/test_readers.py | 12 ++++++++---- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/checksit/check.py b/checksit/check.py index c8ef3ca7..dae5df64 100644 --- a/checksit/check.py +++ b/checksit/check.py @@ -800,19 +800,20 @@ def parse_file_header( ext = extension(file_path) if ext in ("nc", "cdl"): - reader = cdl + reader = cdl.CDLParser(file_path, verbose) elif ext in ("pp"): - reader = pp - elif ext in ("txt"): - reader = badc_csv + reader = pp.PPHeader(file_path, verbose) elif ext in ("yml"): - reader = yml + reader = yml.YAMLFile(file_path, verbose) + elif ext in ("txt"): + reader = badc_csv.BADCCSVHeader(file_path, verbose) elif ext.lower() in IMAGE_EXTENSIONS: - reader = image + reader = image.ImageParser(file_path, verbose) else: raise Exception(f"No known reader for file with extension: {ext}") - content = reader.read(file_path, verbose=verbose) + reader.read() + if auto_cache: base = os.path.splitext(os.path.basename(file_path))[0] @@ -820,22 +821,22 @@ def parse_file_header( conf["settings"]["default_template_cache_dir"], base ) - if reader == cdl: + if isinstance(reader, cdl.CDLParser): # Special case for NetCDF files using CDL with open(f"{output_path}.cdl", "w") as writer: - writer.write(content.cdl) + writer.write(reader.cdl) else: # All others use YAML with open(f"{output_path}.yml", "w") as writer: yaml.dump( - content.to_dict(), + reader.to_dict(), writer, Dumper=yaml.SafeDumper, default_flow_style=False, sort_keys=False, ) - return content + return reader def check_file(file_path: str, **kwargs) -> None: diff --git a/tests/test_readers.py b/tests/test_readers.py index a8975c8e..2846be72 100644 --- a/tests/test_readers.py +++ b/tests/test_readers.py @@ -1,7 +1,8 @@ import os import pytest -from checksit.readers.cdl import read as read_cdl +#from checksit.readers.cdl import read as read_cdl +from checksit.readers.cdl import CDLParser from .common import TESTDATA_DIR @@ -15,7 +16,8 @@ def check(file_path, mappings=None, rules=None, ignore_attrs=None, ignore_all_gl def test_cdl_reader_multiline_parser_1(): cci_file = os.path.join(TESTDATA_DIR, "esacci/ESACCI-GHG-L2-CH4-CO-TROPOMI-WFMD-20171110-fv2.cdl") - resp = read_cdl(cci_file) + resp = CDLParser(cci_file) + resp.read() d = resp.to_dict() assert d["variables"]["pressure_levels"]["comment"] == \ @@ -31,14 +33,16 @@ def test_cdl_reader_multiline_parser_1(): @pytest.mark.xfail(reason="File contains badly defined number attributes in strings - so let it fail for now.") def test_cdl_reader_multiline_parser_2(): cci_file = os.path.join(TESTDATA_DIR, "esacci/ESACCI-GHG-L2-CO2-GOSAT2-SRFP-20191231-fv2.cdl") - resp = read_cdl(cci_file) + resp = CDLParser(cci_file) + resp.read() d = resp.to_dict() def test_cdl_reader_netcdf(): ncfile = os.path.join(TESTDATA_DIR, "netcdf/test_netcdf.nc") - resp = read_cdl(ncfile) + resp = CDLParser(ncfile) + resp.read() d = resp.to_dict() assert sorted(d.keys()) == sorted(["global_attributes", "dimensions", "variables", "inpt"]) From 65485b04dbe58e2acafe96a93d2c5b8482eede20 Mon Sep 17 00:00:00 2001 From: Joshua Hampton Date: Wed, 26 Aug 2026 11:32:44 +0100 Subject: [PATCH 3/6] Add missing self --- checksit/readers/cdl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checksit/readers/cdl.py b/checksit/readers/cdl.py index 97674ef2..f59b253f 100644 --- a/checksit/readers/cdl.py +++ b/checksit/readers/cdl.py @@ -90,7 +90,7 @@ def read(self) -> None: for s in self.CDL_SPLITTERS: if s not in cdl_lines: print( - f"Please check your command - invalid file or CDL contents provided: '{inpt[:100]}...'" + f"Please check your command - invalid file or CDL contents provided: '{self.inpt[:100]}...'" ) sys.exit(1) From ffbf2e4f62b7fcf3e35460eea8b56426101aa7d2 Mon Sep 17 00:00:00 2001 From: Joshua Hampton Date: Wed, 26 Aug 2026 13:37:49 +0100 Subject: [PATCH 4/6] Tidy class --- checksit/check.py | 236 ++++++++++++++-------------------------------- 1 file changed, 69 insertions(+), 167 deletions(-) diff --git a/checksit/check.py b/checksit/check.py index dae5df64..25b4de22 100644 --- a/checksit/check.py +++ b/checksit/check.py @@ -37,6 +37,7 @@ class Checker: results of the checks. Attributes: + file_path: File to check. template: Template to use for checking. Options are "auto" (default), "off", or `