Test the L0 → L0.5 step

.github/workflows/ci.yml

@@ -50,10 +50,25 @@ jobs:
           python: '3.14'
           nox_session: tests-3.14
 
+        - name: Tests, Python 3.13
+          os: windows-latest
+          python: '3.13'
+          nox_session: tests-3.13
+
+        - name: Tests, Python 3.12
+          os: ubuntu-latest
+          python: '3.12'
+          nox_session: tests-3.12
+
+        - name: Tests, Python 3.11
+          os: macos-latest
+          python: '3.11'
+          nox_session: tests-3.11
+
         - name: Tests, Python 3.10
           os: windows-latest
           python: '3.10'
-          nox_session: tests-3.13
+          nox_session: tests-3.10
 
         - name: Tests, Python 3.9
           os: ubuntu-latest

.gitignore

@@ -216,3 +216,6 @@ __marimo__/
 
 # Streamlit
 .streamlit/secrets.toml
+
+# Miscellaneous
+Untitled*.ipynb

noxfile.py

@@ -24,9 +24,9 @@
 import os
 import pathlib
 
-import nox  # ty:ignore[unresolved-import]
-import nox.command  # ty:ignore[unresolved-import]
-import nox_uv  # ty:ignore[unresolved-import]
+import nox
+import nox.command
+import nox_uv
 
 nox.options.default_venv_backend = "uv"
 
@@ -40,14 +40,18 @@
 )
 SUPPORTED_OPERATING_SYSTEMS: tuple[str, ...] = ("linux", "macos", "windows")
 
-MAXPYTHON = max(SUPPORTED_PYTHON_VERSIONS)
-MINPYTHON = min(SUPPORTED_PYTHON_VERSIONS)
+MAXPYTHON = SUPPORTED_PYTHON_VERSIONS[-1]
+MINPYTHON = SUPPORTED_PYTHON_VERSIONS[0]
 
 RUNNING_ON_CI: bool = os.getenv("CI") is not None
 RUNNING_ON_RTD: bool = os.getenv("READTHEDOCS") is not None
 
 DOCPYTHON = "3.14"
 
+REPO_ROOT = pathlib.Path(__file__).parent
+DATA_DIR = REPO_ROOT / "tests" / "data"
+SSR_DIR = DATA_DIR / "sci" / "sweap" / "raw" / "ssr"
+
 
 @nox_uv.session(uv_groups=["test"], python=SUPPORTED_PYTHON_VERSIONS)
 def tests(session: nox.Session) -> None:

pyproject.toml

@@ -1,3 +1,3 @@
 [build-system]
 build-backend = "uv_build"
 requires = [ "uv-build>=0.12.5,<0.13" ]
@@ -25,39 +25,16 @@
   "Programming Language :: Python :: 3.15",
 ]
 dependencies = [
-  "argcomplete>=3.6.3",
-  "bokeh>=2.3.3",
-  "cached-property>=1.5.2",
-  "certifi>=2020.4.5.1",
-  "charset-normalizer>=3.4.7",
-  "cycler>=0.11",
-  "et-xmlfile>=2",
-  "h5py>=3.1",
-  "idna>=3.16",
-  "imap-tools>=1.13",
-  "jinja2>=3.0.3",
-  "kiwisolver>=1.3.1",
-  "markupsafe>=2.0.1",
-  "matplotlib>=3.3.4",
-  "numpy>=1.19.5",
-  "openpyxl>=3.1.5",
-  "packaging>=21.3",
-  "pillow>=8.4",
-  "pip>=26.0.1",
-  "pyparsing>=3.0.7",
-  "python-dateutil>=2.8.2",
-  "pyyaml>=6",
-  "requests>=2.32.5",
-  "scipy>=1.5.4",
-  "setuptools>=82.0.1",
-  "six>=1.16",
-  "spacepy>=0.4.1",
-  "spiceypy>=4.0.1",
-  "tornado>=6.1",
-  "typing-extensions>=4.1.1",
-  "urllib3>=2.6.3",
-  "xlrd>=1.2",
+  "numpy>=1.18.5,<1.22; python_version<'3.11'",
+  "numpy>=1.23.2; python_version>='3.11'",
+  # matplotlib < 3.9 (used with numpy < 1.22 on Python < 3.11) calls
+  # pyparsing's parseString, which pyparsing 3.3 deprecates.
+  "pyparsing<3.3; python_version<'3.11'",
+  "python-dateutil>=2.9",
+  "spacepy>=0.7",
+  "spiceypy>=6",
 ]
+scripts.levelup = "pyfaradaycup.pipeline.swp_spc_l02l1:main"
 
 [dependency-groups]
 dev = [
@@ -68,9 +45,13 @@
   { include-group = "zizmor" },
 ]
 test = [
+  "cdflib",
   "pytest",
+  "pytest-check>=2.9.1",
   "pytest-filter-subpackage",
   "pytest-xdist",
+  "xarray>=2022.6,<2023.10; python_version<'3.11'",
+  "xarray>=2024.6; python_version>='3.11'",
 ]
 docs = [
   "sphinx",

src/pyfaradaycup/data/README.md

@@ -0,0 +1,18 @@
+The data files in this directory were transferred from the server used to process the data pipeline.
+
+The following files and directories came from `/psp/code/`.
+
+- `cdf_skeletons/`
+- `SC_HK.blk`
+- `SC_HK.qlf`
+- `sc_hk_def/`
+- `sweap_tlm.blk`
+
+The following files and directories came from `/psp/data/`.
+
+- `moc_data_products/operations_sclk_kernel/spp_sclk_*.tsc`
+- `moc_data_products/leap_second_kernel/naif0012.tls`
+- `sc_hsk/`
+
+> [!NOTE]
+> Whitespace and permissions changes were made to several of the ASCII files.

src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py

@@ -22,6 +22,7 @@
 
 import datetime
 import os
+import pathlib
 import re
 import struct
 import sys
@@ -30,20 +31,17 @@
 import dateutil.parser
 import numpy as np
 
-# import Tkinter
-# import tkFileDialog
+DATA_DIR = pathlib.Path(__file__).parent.parent / "data"
 
 
-#########################################
+# import Tkinter
+# import tkFileDialog
 
 
 def read_stdin(ptp=False, verbose=False):  # ruff:ignore[ANN001, ANN201, FBT002]
     """Parse binary stream on stdin"""  # ruff:ignore[D400]
 
 
-#########################################
-
-
 def file2bytestr(path="", verbose=False, gzip=False):  # ruff:ignore[ANN001, ANN201, ARG001, FBT002]
     """
     Read the entire contents of a file into a bytes object.
@@ -84,15 +82,9 @@ def file2bytestr(path="", verbose=False, gzip=False):  # ruff:ignore[ANN001, ANN
     except:  # ruff:ignore[E722]
         print("***ERROR*** [ccsds_reader_pipeline] Could not read in file...exiting")  # ruff:ignore[T201]
         print(sys.exc_info())  # ruff:ignore[T201]
-        import pdb  # ruff:ignore[PLC0415, T100]
-
-        pdb.set_trace()  # ruff:ignore[T100]
         sys.exit()
 
 
-#########################################
-
-
 def choose_file(path="", ptp=False, verbose=False):  # ruff:ignore[ANN001, ANN201, ARG001, FBT002]
     # make sure file exists
     """
@@ -138,9 +130,6 @@ def choose_file(path="", ptp=False, verbose=False):  # ruff:ignore[ANN001, ANN20
     return path
 
 
-#########################################
-
-
 def wrapper_status(path="", verbose=False, gzip=False, spconly=False):  # ruff:ignore[ANN001, ANN201, ARG001, FBT002]
     """
     Read CCSDS headers from SWEM wrapper packets and the packets inside them.
@@ -237,9 +226,6 @@ def wrapper_status(path="", verbose=False, gzip=False, spconly=False):  # ruff:i
     return data
 
 
-#########################################
-
-
 def read_file(path="", verbose=False, gzip=False):  # ruff:ignore[ANN001, ANN201, C901, FBT002]
     """Read a CCSDS File and return data structure"""  # ruff:ignore[D400]
     # get a filename if not specified
@@ -331,9 +317,6 @@ def read_file(path="", verbose=False, gzip=False):  # ruff:ignore[ANN001, ANN201
     return data
 
 
-#########################################
-
-
 def read_file_sc(path="", verbose=False, ptp=False, gzip=False):  # ruff:ignore[ANN001, ANN201, C901, FBT002, PLR0912, PLR0915]
     """Read a CCSDS File and return data structure"""  # ruff:ignore[D400]
     # get a filename if not specified
@@ -483,7 +466,6 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False):  # ruff:ignore[
         offset_bytes = (
             0  # we searched for beginning of CCSDS packets, so no offset necessary
         )
-    # import pdb; pdb.set_trace()
 
     # Find all occurrences of the beginning of a packet
     pkt_inds = np.array(
@@ -529,10 +511,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False):  # ruff:ignore[
     return data
 
 
-#########################################
-
-
-def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False):  # ruff:ignore[ANN001, ANN201, C901, FBT002, PLR0912, PLR0913, RET503]
+def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False):  # ruff:ignore[ANN001, ANN201, C901, FBT002, PLR0912, PLR0913]
     """Take a hex string and find packets"""  # ruff:ignore[D400]
     # Parse the CCSDS header
     try:
@@ -586,14 +565,6 @@ def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False):  #
 
     return ()
 
-    # we shouldn't make it here
-    import pdb  # ruff:ignore[PLC0415, T100]
-
-    pdb.set_trace()  # ruff:ignore[T100]
-
-
-#########################################
-
 
 def parse_ccsds_head(bytestr, verbose=False):  # ruff:ignore[ANN001, ANN201, ARG001, FBT002]
     """
@@ -648,9 +619,6 @@ def parse_ccsds_head(bytestr, verbose=False):  # ruff:ignore[ANN001, ANN201, ARG
     return head
 
 
-#########################################
-
-
 def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False):  # ruff:ignore[ANN001, ANN201, ARG001, C901, FBT002, PLR0912, PLR0913]
     """Parse one CCSDS packet"""  # ruff:ignore[D400]
     # The format for this APIDs packet list
@@ -713,9 +681,7 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False):  # ru
         try:
             thisval = int(thisbin, 2)
         except:  # ruff:ignore[E722]
-            import pdb  # ruff:ignore[PLC0415, T100]
-
-            pdb.set_trace()  # ruff:ignore[T100]
+            raise RuntimeError  # ruff:ignore[B904]
             thisval = -999
         thisname = form.names[i_bit]
 
@@ -742,9 +708,7 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False):  # ru
                 try:
                     thisval = int(thisbin, 2)
                 except ValueError:
-                    import pdb  # ruff:ignore[PLC0415, T100]
-
-                    pdb.set_trace()  # ruff:ignore[T100]
+                    raise ValueError  # ruff:ignore[B904]
                     thisval = -999
 
                 thisname = form.sw_data_vars[i]
@@ -755,7 +719,7 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False):  # ru
             thisdat[key].append(newdat[key])
 
 
-class apid_obj:  # ruff:ignore[ N801]
+class apid_obj:  # ruff:ignore[N801]
     """
     Store the bit layout of one packet type (APID).
 
@@ -793,8 +757,6 @@ class apid_obj:  # ruff:ignore[ N801]
     packets that have one.
     """
 
-    #########################################
-
     def __init__(self):  # ruff:ignore[ANN204]
         self.names = []
         self.bits = []
@@ -807,9 +769,6 @@ def __init__(self):  # ruff:ignore[ANN204]
         self.startbit = []
 
 
-#########################################
-
-
 def get_layout(apid, verbose=False):  # ruff:ignore[ANN001, ANN201, C901, FBT002]
     """
     Read the bit layout for one SWEAP APID from ``sweap_tlm.blk``.
@@ -836,21 +795,23 @@ def get_layout(apid, verbose=False):  # ruff:ignore[ANN001, ANN201, C901, FBT002
     so it only works on Windows.
     """
     try:
-        file = open("sweap_tlm.blk")  # ruff:ignore[PTH123, SIM115]
+        # It appears that there is a unique sweap_tlm.blk
+        file = open(DATA_DIR / "sweap_tlm.blk")  # ruff:ignore[PTH123, SIM115]
     except:  # ruff:ignore[E722]
         if verbose:
             print(  # ruff:ignore[T201]
-                "***INFO*** No local 'sweap_tlm.blk' found...using the one near ccsds_reader_pipeline.py"
+                "***INFO*** 'sweap_tlm.blk' not found...using the one in src/pyfaradaycup/data"
             )
         try:
-            thisdir = os.path.realpath(__file__)
-            thisdir = "\\".join(thisdir.split("\\")[0:-1])
-            file = open(thisdir + "\\sweap_tlm.blk")  # ruff:ignore[PTH123, SIM115]
+            # here = os.path.dirname(__file__)
+            # thisdir = os.path.realpath(__file__)
+            # thisdir = "\\".join(thisdir.split("\\")[0:-1])
+            # print(f"{thisdir = }")
+            file = open(DATA_DIR / "sweap_tlm.blk")  # ruff:ignore[PTH123, SIM115]
         except:  # ruff:ignore[E722]
+            # print(here)
             print(sys.exc_info())  # ruff:ignore[T201]
-            import pdb  # ruff:ignore[PLC0415, T100]
-
-            pdb.set_trace()  # ruff:ignore[T100]
+            raise RuntimeError(f"Unable to open {DATA_DIR}/sweap_tlm.blk")  # ruff:ignore[B904, EM102, TRY003]
     lines = file.readlines()
     for i, line in enumerate(lines):
         if line[0:8] == f"APID_{hex(apid)[2:].zfill(3)}".upper():  # ruff:ignore[FURB116]
@@ -880,9 +841,7 @@ def get_layout(apid, verbose=False):  # ruff:ignore[ANN001, ANN201, C901, FBT002
                     break
                 except:  # ruff:ignore[E722]
                     print(sys.exc_info())  # ruff:ignore[T201]
-                    import pdb  # ruff:ignore[PLC0415, T100]
-
-                    pdb.set_trace()  # ruff:ignore[T100]
+                    raise RuntimeError  # ruff:ignore[B904]
 
             start = np.array(
                 [0] + [sum(thisapid.bits[0:i]) for i in range(1, len(thisapid.bits))]
@@ -903,7 +862,6 @@ def get_layout(apid, verbose=False):  # ruff:ignore[ANN001, ANN201, C901, FBT002
     return None
 
 
-#########################################
 def get_layout_sc(apid, verbose=False, filename=""):  # ruff:ignore[ANN001, ANN201, C901, FBT002]
     """
     Read the bit layout for one spacecraft housekeeping APID.
@@ -940,9 +898,7 @@ def get_layout_sc(apid, verbose=False, filename=""):  # ruff:ignore[ANN001, ANN2
     except:  # ruff:ignore[E722]
         print("could not open SC HK BLK file")  # ruff:ignore[T201]
         print(sys.exc_info())  # ruff:ignore[T201]
-        import pdb  # ruff:ignore[PLC0415, T100]
-
-        pdb.set_trace()  # ruff:ignore[T100]
+        raise RuntimeError  # ruff:ignore[B904]
 
     lines = file.readlines()
     for i, line in enumerate(lines):
@@ -974,9 +930,7 @@ def get_layout_sc(apid, verbose=False, filename=""):  # ruff:ignore[ANN001, ANN2
                     break
                 except:  # ruff:ignore[E722]
                     print(sys.exc_info())  # ruff:ignore[T201]
-                    import pdb  # ruff:ignore[PLC0415, T100]
-
-                    pdb.set_trace()  # ruff:ignore[T100]
+                    raise RuntimeError  # ruff:ignore[B904]
             return (thisapid, length)
     # if we didn't find that APID
     print(  # ruff:ignore[T201]