diff --git a/.github/release_checklist.md b/.github/release_checklist.md index a38d8661..8ae899dc 100644 --- a/.github/release_checklist.md +++ b/.github/release_checklist.md @@ -9,7 +9,6 @@ Release checklist from CHANGELOG.rst. - [ ] Push tag to remote. This triggers the wheel/sdist build on github CI. - [ ] merge `main` branch back into `develop`. -- [ ] Add updated version number to develop. (`setup.py` and `src/isal/__init__.py`) - [ ] Build the new tag on readthedocs. Only build the last patch version of each minor version. So `1.1.1` and `1.2.0` but not `1.1.0`, `1.1.1` and `1.2.0`. - [ ] Create a new release on github. diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c7f686ac..2ffabcb8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 1.7.1 +----------------- ++ Fix a bug where flushing files when writing in threaded mode did not work + properly. ++ Prevent threaded opening from blocking python exit when an error is thrown + in the calling thread. + version 1.7.0 ----------------- + Include a patched ISA-L version 2.31. The applied patches make compilation diff --git a/requirements-docs.txt b/requirements-docs.txt index 89dc59f1..47081d94 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,4 +1,3 @@ sphinx sphinx-rtd-theme -# See https://github.com/sphinx-doc/sphinx-argparse/issues/56 -sphinx-argparse <0.5.0 \ No newline at end of file +sphinx-argparse diff --git a/setup.py b/setup.py index 84a92cf7..7491870d 100644 --- a/setup.py +++ b/setup.py @@ -166,6 +166,7 @@ def build_isa_l(): "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: C", diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index cd8b4238..7f1c94fc 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -60,7 +60,7 @@ def open(filename, mode="rb", compresslevel=igzip._COMPRESS_LEVEL_TRADEOFF, gzip_file = io.BufferedReader( _ThreadedGzipReader(filename, block_size=block_size)) else: - gzip_file = io.BufferedWriter( + gzip_file = FlushableBufferedWriter( _ThreadedGzipWriter( filename, mode.replace("t", "b"), @@ -101,6 +101,7 @@ def __init__(self, filename, queue_size=2, block_size=1024 * 1024): self.worker = threading.Thread(target=self._decompress) self._closed = False self.running = True + self._calling_thread = threading.current_thread() self.worker.start() def _check_closed(self, msg=None): @@ -110,7 +111,7 @@ def _check_closed(self, msg=None): def _decompress(self): block_size = self.block_size block_queue = self.queue - while self.running: + while self.running and self._calling_thread.is_alive(): try: data = self.fileobj.read(block_size) except Exception as e: @@ -118,7 +119,7 @@ def _decompress(self): return if not data: return - while self.running: + while self.running and self._calling_thread.is_alive(): try: block_queue.put(data, timeout=0.05) break @@ -166,6 +167,12 @@ def closed(self) -> bool: return self._closed +class FlushableBufferedWriter(io.BufferedWriter): + def flush(self): + super().flush() + self.raw.flush() + + class _ThreadedGzipWriter(io.RawIOBase): """ Write a gzip file using multiple threads. @@ -215,6 +222,7 @@ def __init__(self, if "b" not in mode: mode += "b" self.lock = threading.Lock() + self._calling_thread = threading.current_thread() self.exception: Optional[Exception] = None self.level = level self.previous_block = b"" @@ -308,7 +316,7 @@ def write(self, b) -> int: self.input_queues[worker_index].put((data, zdict)) return len(data) - def flush(self): + def _end_gzip_stream(self): self._check_closed() # Wait for all data to be compressed for in_q in self.input_queues: @@ -316,22 +324,27 @@ def flush(self): # Wait for all data to be written for out_q in self.output_queues: out_q.join() + # Write an empty deflate block with a lost block marker. + self.raw.write(isal_zlib.compress(b"", wbits=-15)) + trailer = struct.pack(" None: if self._closed: return - self.flush() + self._end_gzip_stream() self.stop() if self.exception: self.raw.close() self._closed = True raise self.exception - # Write an empty deflate block with a lost block marker. - self.raw.write(isal_zlib.compress(b"", wbits=-15)) - trailer = struct.pack("