Update some Python deps

Cleaner Snakemake in-line function testing
Remove unneeded spike-in-control check

Snakefile.main

@@ -380,11 +380,6 @@ def one_cell_inputs(wc):
     base = cellname_to_base(cell)
     barcodes = SC[cell].keys()
 
-    # Load the final summary to see if this is RNA or DNA. If the summary
-    # YAML is already in the output this bypasses needing EXPDIR.
-    fs = load_final_summary(f"{EXPDIR}/{cell}/",
-                            yamlfile = f"{cell}/cell_final_summary.yaml")
-
     xxpand = partial(expand, cell = cell,
                              base = base,
                              bc = barcodes,

activate_venv

@@ -64,7 +64,7 @@ else
         # https://github.com/EdinburghGenomics/blobtools/tree/tims_patches
         pip_install docopt==0.6.2
         pip_install matplotlib==3.3.3
-        pip_install pysam==0.19.0
+        pip_install pysam==0.22.1
         pip_install tqdm==4.64.0
         pip_install ujson==5.2.0
 
@@ -83,14 +83,14 @@ else
 
         # For access the Clarity
         pip_install pyclarity_lims==0.4.8
-        pip_install psycopg2-binary==2.9.3
+        pip_install psycopg2-binary==2.9.9
 
         # For get_fast5_metadata.py
-        pip_install h5py==3.8.0
+        pip_install h5py==3.11.0
 
         # For get_pod5_metadata.py
-        pip_install pyarrow==14.0.2
-        pip_install pod5==0.3.10
+        pip_install pyarrow==16.1.0
+        pip_install pod5==0.3.11
 
         # Here’s the big one
         pip_install numpy==1.23.2     # upgrade with NanoPlot

get_pod5_metadata.py

@@ -92,6 +92,8 @@ def read_pod5(p5_filename):
 
         # Just as the metadata is the same for each file, it's the same for each
         # read, so just get the first one, and dict-ify it.
+        # Note - I could get at the same info in a less convenient format with:
+        #   p5_handle.run_info_table.get_batch(0)
         read0 = vars(next(p5_handle.reads()).run_info)
 
         # Run ID used to be in the filename, but to be sure get it here

test/requirements.txt

@@ -2,7 +2,7 @@ bashmocker==0.3.0
 pyyaml==6.0.1
 yamlloader==1.1.0
 rt==2.2.2
-h5py==3.8.0
-pod5==0.2.4
+h5py==3.11.0
+pod5==0.3.11
 python-dateutil==2.8.2
 snakemake==7.18.2

test/test_snakefile_checksummer_functions.py

@@ -11,6 +11,8 @@
 from tempfile import mkstemp
 from collections import OrderedDict
 
+from snakemake import Workflow
+
 SNAKEFILE = os.path.abspath(os.path.dirname(__file__) + '/../Snakefile.checksummer')
 VERBOSE = os.environ.get('VERBOSE', '0') != '0'
 
@@ -25,28 +27,23 @@ class T(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         # So the thing is I don't want to take the functions out of the Snakefile
-        # But I really want to test them. So we have this hack. Hackety hack.
-        with open(SNAKEFILE) as sfh:
-            # Chop out everything after "## End of functions ##"
-            snake_code = list(takewhile(lambda l: not l.startswith("## End of functions ##"), sfh))
-            snake_code = compile("".join(snake_code), sfh.name, 'exec')
-
-        gdict = dict( os = os,
-                      workflow = Mock(snakefile='NONE'),
-                      snakemake = Mock(),
-                      config = dict(input_dir='.') )
-        exec(snake_code, gdict)
-        #pprint(gdict)
+        # But I really want to test them. We can get Snakemake to parse the workflow
+        # and get the functions for us.
+        wf = Workflow(snakefile=SNAKEFILE)
+
+        # We can fix the logging for Snakefile code which normally logs
+        # to the Snakemake logger.
+        wf.globals['logger'] = logging.getLogger()
+        wf.globals['logger'].setLevel(logging.DEBUG if VERBOSE else logging.CRITICAL)
+
+        wf.config.update(dict(input_dir='.'))
+        wf.include(wf.main_snakefile)
 
         # Import to global namespace.
         funcs_to_import = [ k for k, v in globals().items() if v == "_importme" ]
         for func in funcs_to_import:
-            globals()[func] = gdict[func]
+            globals()[func] = wf.globals[func]
 
-        # Now we can fix the logging for Snakefile functions which normally log
-        # to the Snakemake logger.
-        gdict['logger'] = logging.getLogger()
-        gdict['logger'].setLevel(logging.DEBUG if VERBOSE else logging.CRITICAL)
 
     def setUp(self):
         # See the errors in all their glory

test/test_snakefile_main_functions.py

@@ -11,6 +11,8 @@
 from tempfile import mkstemp
 from collections import OrderedDict
 
+from snakemake import Workflow
+
 SNAKEFILE = os.path.abspath(os.path.dirname(__file__) + '/../Snakefile.main')
 DATA_DIR = os.path.abspath(os.path.dirname(__file__) + '/examples')
 VERBOSE = os.environ.get('VERBOSE', '0') != '0'
@@ -26,30 +28,24 @@ class T(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         # So the thing is I don't want to take the functions out of the Snakefile
-        # But I really want to test them. So we have this hack. Hackety hack.
-        with open(SNAKEFILE) as sfh:
-            # Chop out everything after "## End of functions ##"
-            snake_code = list(takewhile(lambda l: not l.startswith("## End of functions ##"), sfh))
-            snake_code = compile("".join(snake_code), sfh.name, 'exec')
-
-        os.environ['TOOLBOX'] = 'NONE'
-        os.environ['REFS'] = '.'
-        gdict = dict( os = os,
-                      workflow = Mock(snakefile='NONE'),
-                      snakemake = Mock(),
-                      config = dict() )
-        exec(snake_code, gdict)
-        #pprint(gdict)
-
-        # Import to global namespace.
+        # But I really want to test them. So...
+        wf = Workflow(snakefile=SNAKEFILE)
+        # Fix the logging for the Snakefile functions which normally log
+        # to the Snakemake logger.
+        wf.globals['logger'] = logging.getLogger()
+        wf.globals['logger'].setLevel(logging.DEBUG if VERBOSE else logging.CRITICAL)
+        # There's no config to set but if there was it would be set here
+        wf.config.update(dict())
+        # Avoid changing os.environ directly as it could mess with other tests.
+        with patch('os.environ', new=dict(os.environ)) as os_environ:
+            os_environ.update(dict(TOOLBOX="NONE", REFS="."))
+            wf.include(wf.main_snakefile)
+
+        # Import the funcs we are testing to global namespace.
         funcs_to_import = [ k for k, v in globals().items() if v == "_importme" ]
         for func in funcs_to_import:
-            globals()[func] = gdict[func]
+            globals()[func] = wf.globals[func]
 
-        # Now we can fix the logging for the Snakefile functions which normally log
-        # to the Snakemake logger.
-        gdict['logger'] = logging.getLogger()
-        gdict['logger'].setLevel(logging.DEBUG if VERBOSE else logging.CRITICAL)
 
     def setUp(self):
         # See the errors in all their glory