From 1b5d2ba1a1adaa06f0c7b5a5e50ecad462c0b136 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 3 Dec 2021 13:32:51 +0100 Subject: [PATCH 01/94] Update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8771bba5..0ed659f4 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name="pytest-workflow", - version="1.6.0", + version="1.7.0-dev", description="A pytest plugin for configuring workflow/pipeline tests " "using YAML files", author="Leiden University Medical Center", From 82bb5065a706c875d056c8266d55bf8d4a4e24df Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 1 Feb 2022 09:42:10 +0100 Subject: [PATCH 02/94] Test for appropriate file not found message when files have been removed but not in git --- tests/test_utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index aa74946e..f6b6fddf 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -117,6 +117,16 @@ def test_duplicate_git_tree(git_dir): assert (dest / "test" / "test.txt").exists() +def test_duplicate_git_tree_file_removed_error(git_dir): + assert (git_dir / ".git").exists() + os.remove(git_dir / "test" / "test.txt") + dest = Path(tempfile.mkdtemp()) / "test" + with pytest.raises(FileNotFoundError) as e: + duplicate_tree(git_dir, dest, git_aware=True) + e.match("checked") + e.match("git rm") + + def test_duplicate(git_dir): assert (git_dir / ".git").exists() dest = Path(tempfile.mkdtemp()) / "test" From 92c0d6389048dc99e7b701dc08e12476a5f022e1 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 1 Feb 2022 09:50:18 +0100 Subject: [PATCH 03/94] Add a check for git file presence --- src/pytest_workflow/util.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/pytest_workflow/util.py b/src/pytest_workflow/util.py index 947eb71a..00bb66b9 100644 --- a/src/pytest_workflow/util.py +++ b/src/pytest_workflow/util.py @@ -72,8 +72,8 @@ def git_ls_files(path: Filepath) -> List[str]: return output.strip("\n").split("\n") -def _duplicate_tree(src: Filepath, dest: Filepath - ) -> Iterator[Tuple[str, str, bool]]: +def _recurse_directory_tree(src: Filepath, dest: Filepath + ) -> Iterator[Tuple[str, str, bool]]: """Traverses src and for each file or directory yields a path to it, its destination, and whether it is a directory.""" for entry in os.scandir(src): # type: os.DirEntry @@ -81,7 +81,7 @@ def _duplicate_tree(src: Filepath, dest: Filepath dir_src = entry.path dir_dest = os.path.join(dest, entry.name) yield dir_src, dir_dest, True - yield from _duplicate_tree(dir_src, dir_dest) + yield from _recurse_directory_tree(dir_src, dir_dest) elif entry.is_file() or entry.is_symlink(): yield entry.path, os.path.join(dest, entry.name), False else: @@ -89,8 +89,8 @@ def _duplicate_tree(src: Filepath, dest: Filepath f"Skipping {entry.path}") -def _duplicate_git_tree(src: Filepath, dest: Filepath - ) -> Iterator[Tuple[str, str, bool]]: +def _recurse_git_repository_tree(src: Filepath, dest: Filepath + ) -> Iterator[Tuple[str, str, bool]]: """Traverses src, finds all files registered in git and for each file or directory yields a path to it, its destination and whether it is a directory""" @@ -121,6 +121,12 @@ def _duplicate_git_tree(src: Filepath, dest: Filepath # Yield the actual file if the directory has already been yielded. src_path = os.path.join(src, path) + if not os.path.exists(src_path): + raise FileNotFoundError( + f"{src_path} is checked in in git, but not present in the " + f"filesystem. If the file was removed, its removal can be " + f"recorded in git with 'git rm {src_path}'. Removal can be " + f"reversed with 'git checkout {src_path}.") dest_path = os.path.join(dest, path) yield src_path, dest_path, False @@ -144,9 +150,9 @@ def duplicate_tree(src: Filepath, dest: Filepath, raise NotADirectoryError(f"Not a directory: '{src}'") if git_aware: - path_iter = _duplicate_git_tree(src, dest) + path_iter = _recurse_git_repository_tree(src, dest) else: - path_iter = _duplicate_tree(src, dest) + path_iter = _recurse_directory_tree(src, dest) if symlink: copy: Callable[[Filepath, Filepath], None] = \ functools.partial(os.symlink, target_is_directory=False) From f77e97775b98c31b56f5ba14e3bd6766d45f3414 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 1 Feb 2022 10:02:36 +0100 Subject: [PATCH 04/94] Throw informative message when file is missing from git Include copy pastable error messages --- src/pytest_workflow/util.py | 10 ++++++---- tests/test_utils.py | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/pytest_workflow/util.py b/src/pytest_workflow/util.py index 00bb66b9..910b27e1 100644 --- a/src/pytest_workflow/util.py +++ b/src/pytest_workflow/util.py @@ -123,10 +123,12 @@ def _recurse_git_repository_tree(src: Filepath, dest: Filepath src_path = os.path.join(src, path) if not os.path.exists(src_path): raise FileNotFoundError( - f"{src_path} is checked in in git, but not present in the " - f"filesystem. If the file was removed, its removal can be " - f"recorded in git with 'git rm {src_path}'. Removal can be " - f"reversed with 'git checkout {src_path}.") + f"{path} from git repository {src} is checked in in git, " + f"but not present in the filesystem. If the file was removed, " + f"its removal can be recorded in git with " + f"\"git -C '{src}' rm '{path}'\". " + f"Removal can be reversed with " + f"\"git -C '{src}' checkout '{path}'\".") dest_path = os.path.join(dest, path) yield src_path, dest_path, False diff --git a/tests/test_utils.py b/tests/test_utils.py index f6b6fddf..95a02a73 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -123,8 +123,8 @@ def test_duplicate_git_tree_file_removed_error(git_dir): dest = Path(tempfile.mkdtemp()) / "test" with pytest.raises(FileNotFoundError) as e: duplicate_tree(git_dir, dest, git_aware=True) - e.match("checked") - e.match("git rm") + e.match("checked in") + e.match(f"\"git -C '{git_dir}' rm '{str(Path('test', 'test.txt'))}'\"") def test_duplicate(git_dir): From 2c539f5c5d1065aadec6810fc219a022afc779d5 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 1 Feb 2022 10:59:32 +0100 Subject: [PATCH 05/94] update changelog --- HISTORY.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index dfd74e98..31ca2a09 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,11 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 1.7.0-dev +--------------------------- ++ Throw a more descriptive error when a file copied with the --git-aware flag + is not present on the filesystem anymore. + version 1.6.0 --------------------------- + Add a ``--git-aware`` or ``--ga`` option to only copy copy files listed by From b5bf484cf1d65d582042e81f7a80431ba5cba057 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 1 Feb 2022 11:02:54 +0100 Subject: [PATCH 06/94] Ignore cryptic mypy error --- src/pytest_workflow/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pytest_workflow/util.py b/src/pytest_workflow/util.py index 910b27e1..941fde88 100644 --- a/src/pytest_workflow/util.py +++ b/src/pytest_workflow/util.py @@ -76,7 +76,7 @@ def _recurse_directory_tree(src: Filepath, dest: Filepath ) -> Iterator[Tuple[str, str, bool]]: """Traverses src and for each file or directory yields a path to it, its destination, and whether it is a directory.""" - for entry in os.scandir(src): # type: os.DirEntry + for entry in os.scandir(src): # type: os.DirEntry # type: ignore if entry.is_dir(): dir_src = entry.path dir_dest = os.path.join(dest, entry.name) From e24d9d0a093bebbbe74cbe6fa5cc0eed9cde6832 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 1 Feb 2022 11:10:19 +0100 Subject: [PATCH 07/94] Snakemake switched stderr and stdout *again* Is this a thing people are supposed to use in production? --- tests/functional/simple_snakefile_test_cases.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/simple_snakefile_test_cases.yml b/tests/functional/simple_snakefile_test_cases.yml index 5b7f969b..0ca2f666 100644 --- a/tests/functional/simple_snakefile_test_cases.yml +++ b/tests/functional/simple_snakefile_test_cases.yml @@ -3,19 +3,19 @@ - name: test-config-missing command: snakemake -n -r -p -s SimpleSnakefile exit_code: 1 - stderr: + stdout: contains: - "You must set --config N_LINES_TO_READ=." - name: test-config-wrong-type command: snakemake -n -r -p -s SimpleSnakefile --config N_LINES_TO_READ=one exit_code: 1 - stderr: + stdout: contains: - "N_LINES_TO_READ must be an integer." - name: test-config-invalid-value command: snakemake -n -r -p -s SimpleSnakefile --config N_LINES_TO_READ=-1 exit_code: 1 - stderr: + stdout: contains: - "N_LINES_TO_READ must at least be 1." - name: test-snakemake-run From 7323b5a8bba8a306f56ce2aa0049679eea18fcbe Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 18 Feb 2022 15:50:25 +0100 Subject: [PATCH 08/94] Do not import private API in tests --- tests/test_success_messages.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_success_messages.py b/tests/test_success_messages.py index 566e486b..e88d0ddc 100644 --- a/tests/test_success_messages.py +++ b/tests/test_success_messages.py @@ -20,8 +20,6 @@ import textwrap from pathlib import Path -from _pytest.tmpdir import TempdirFactory - import pytest MOO_FILE = textwrap.dedent("""\ @@ -111,7 +109,7 @@ @pytest.fixture(scope="session") -def succeeding_tests_output(tmpdir_factory: TempdirFactory): +def succeeding_tests_output(tmpdir_factory): """This fixture was written because the testdir function has a default scope of 'function'. This is very inefficient when testing multiple success messages in the output as the whole test yaml with all commands From e4f0043c7910439633c54bae998b6c0b4b3caf92 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 18 Feb 2022 16:02:47 +0100 Subject: [PATCH 09/94] Fix warnings for pytest >=7 --- src/pytest_workflow/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pytest_workflow/plugin.py b/src/pytest_workflow/plugin.py index 5e792030..ea567409 100644 --- a/src/pytest_workflow/plugin.py +++ b/src/pytest_workflow/plugin.py @@ -114,12 +114,12 @@ def addoption(self, *args, **kwargs): return parser -def pytest_collect_file(path, parent): +def pytest_collect_file(file_path, path, parent): """Collection hook This collects the yaml files that start with "test" and end with .yaml or .yml""" if path.ext in [".yml", ".yaml"] and path.basename.startswith("test"): - return YamlFile.from_parent(parent, fspath=path) + return YamlFile.from_parent(parent, path=file_path) return None From a3bc169b8131f6f9b6d10f4d27d7da2da5ac83fc Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 18 Feb 2022 16:07:28 +0100 Subject: [PATCH 10/94] Require newer version of pytest --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0ed659f4..8eaa64df 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ # Because we use the resolve(strict=False) feature from pathlib. python_requires=">=3.6", install_requires=[ - "pytest>=5.4.0", # To use from_parent node instantiation. + "pytest>=7.0.0", # To use pathlib Path's in pytest "pyyaml", "jsonschema" ], From 823d86b48757a39a8da6c612e3060bd83fdb839d Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 18 Feb 2022 16:10:43 +0100 Subject: [PATCH 11/94] Add pytest 7.0.0 updates to the changelog --- HISTORY.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 31ca2a09..2bfdfa0f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,9 @@ Changelog version 1.7.0-dev --------------------------- ++ A minimum of pytest 7.0.0 is now a requirement for pytest-workflow. + This fixes the deprecation warnings that started on the release of pytest + 7.0.0. + Throw a more descriptive error when a file copied with the --git-aware flag is not present on the filesystem anymore. From ac8b32a6fc91d9ac9a9fc0f8d9539f0655273460 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 18 Feb 2022 16:19:46 +0100 Subject: [PATCH 12/94] Use modern temppath factory --- tests/test_success_messages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_success_messages.py b/tests/test_success_messages.py index e88d0ddc..8815bc1f 100644 --- a/tests/test_success_messages.py +++ b/tests/test_success_messages.py @@ -109,13 +109,13 @@ @pytest.fixture(scope="session") -def succeeding_tests_output(tmpdir_factory): +def succeeding_tests_output(tmp_path_factory: pytest.TempPathFactory): """This fixture was written because the testdir function has a default scope of 'function'. This is very inefficient when testing multiple success messages in the output as the whole test yaml with all commands has to be run again. This fixture runs the succeeding tests once with pytest -v""" - tempdir = str(tmpdir_factory.mktemp("succeeding_tests")) + tempdir = tmp_path_factory.mktemp("succeeding_tests") test_file = Path(tempdir, "test_succeeding.yml") with test_file.open("w") as file_handler: file_handler.write(SUCCEEDING_TESTS_YAML) From 1ce640c302bd7dbac739039b16dbcbb74b8d8076 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 18 Feb 2022 16:37:21 +0100 Subject: [PATCH 13/94] Use rootpath instead of rootdir --- src/pytest_workflow/plugin.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/pytest_workflow/plugin.py b/src/pytest_workflow/plugin.py index ea567409..40da55a8 100644 --- a/src/pytest_workflow/plugin.py +++ b/src/pytest_workflow/plugin.py @@ -173,13 +173,12 @@ def pytest_configure(config: PytestConfig): Path(basetemp) if basetemp is not None else Path(tempfile.mkdtemp(prefix="pytest_workflow_"))) - rootdir = Path(str(config.rootdir)) - # Raise an error if the workflow temporary directory of the rootdir + # Raise an error if the workflow temporary directory of the rootpath # (pytest's CWD). This will lead to infinite looping and copying. - if is_in_dir(workflow_temp_dir, rootdir): + if is_in_dir(workflow_temp_dir, config.rootpath): raise ValueError(f"'{workflow_temp_dir}' is a subdirectory of " - f"'{rootdir}'. Please select a --basetemp that is " - f"not in pytest's current working directory.") + f"'{config.rootpath}'. Please select a --basetemp " + f"that is not in pytest's current working directory.") setattr(config, "workflow_temp_dir", workflow_temp_dir) From 11a8c20b4a39bbe9588ac58a1fa8fc3c2294a1e8 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 18 Feb 2022 16:54:25 +0100 Subject: [PATCH 14/94] Use pytester API instead of testdir API --- tests/functional/test_functional.py | 36 ++++----- tests/test_fail_messages.py | 6 +- tests/test_miscellaneous_crashes.py | 8 +- tests/test_multithreading.py | 6 +- tests/test_success_messages.py | 8 +- tests/test_tags.py | 24 +++--- tests/test_temp_directory.py | 83 ++++++++++---------- tests/test_threading_errors.py | 6 +- tests/test_warnings.py | 14 ++-- tests/test_workflow_dependent_tests.py | 104 ++++++++++++------------- 10 files changed, 149 insertions(+), 146 deletions(-) diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.py index bf77bfa9..58e2e4a6 100644 --- a/tests/functional/test_functional.py +++ b/tests/functional/test_functional.py @@ -27,33 +27,33 @@ @pytest.mark.functional -def test_cromwell(testdir): - testdir.makefile(ext=".json", simple=SIMPLE_WDL_JSON.read_text()) - testdir.makefile(ext=".wdl", simple=SIMPLE_WDL.read_text()) - testdir.makefile(ext=".yml", test_wdl=SIMPLE_WDL_YAML.read_text()) - testdir.makefile(ext=".options.json", - simple=SIMPLE_WDL_OPTIONS_JSON.read_text()) - result = testdir.runpytest("-v", "--tag", "cromwell", - "--keep-workflow-wd-on-fail") +def test_cromwell(pytester): + pytester.makefile(ext=".json", simple=SIMPLE_WDL_JSON.read_text()) + pytester.makefile(ext=".wdl", simple=SIMPLE_WDL.read_text()) + pytester.makefile(ext=".yml", test_wdl=SIMPLE_WDL_YAML.read_text()) + pytester.makefile(ext=".options.json", + simple=SIMPLE_WDL_OPTIONS_JSON.read_text()) + result = pytester.runpytest("-v", "--tag", "cromwell", + "--keep-workflow-wd-on-fail") exit_code = result.ret assert exit_code == 0 assert "simple wdl" in result.stdout.str() @pytest.mark.functional -def test_miniwdl(testdir): - testdir.makefile(ext=".json", simple=SIMPLE_WDL_JSON.read_text()) - testdir.makefile(ext=".wdl", simple=SIMPLE_WDL.read_text()) - testdir.makefile(ext=".yml", test_wdl=SIMPLE_WDL_YAML.read_text()) - result = testdir.runpytest("-v", "--tag", "miniwdl", - "--keep-workflow-wd-on-fail") +def test_miniwdl(pytester): + pytester.makefile(ext=".json", simple=SIMPLE_WDL_JSON.read_text()) + pytester.makefile(ext=".wdl", simple=SIMPLE_WDL.read_text()) + pytester.makefile(ext=".yml", test_wdl=SIMPLE_WDL_YAML.read_text()) + result = pytester.runpytest("-v", "--tag", "miniwdl", + "--keep-workflow-wd-on-fail") assert result.ret == 0 @pytest.mark.functional -def test_snakemake(testdir): - testdir.makefile(ext="", SimpleSnakefile=SNAKEFILE.read_text()) - testdir.makefile(ext=".yml", test_snakemake=SNAKEFILE_YAML.read_text()) - result = testdir.runpytest("-v", "--keep-workflow-wd-on-fail") +def test_snakemake(pytester): + pytester.makefile(ext="", SimpleSnakefile=SNAKEFILE.read_text()) + pytester.makefile(ext=".yml", test_snakemake=SNAKEFILE_YAML.read_text()) + result = pytester.runpytest("-v", "--keep-workflow-wd-on-fail") exit_code = result.ret assert exit_code == 0 diff --git a/tests/test_fail_messages.py b/tests/test_fail_messages.py index bd373e0e..e1bdc97f 100644 --- a/tests/test_fail_messages.py +++ b/tests/test_fail_messages.py @@ -164,9 +164,9 @@ @pytest.mark.parametrize(["test", "message"], FAILURE_MESSAGE_TESTS) -def test_messages(test: str, message: str, testdir): - testdir.makefile(".yml", textwrap.dedent(test)) +def test_messages(test: str, message: str, pytester): + pytester.makefile(".yml", textwrap.dedent(test)) # Ideally this should be run in a LC_ALL=C environment. But this is not # possible due to multiple levels of process launching. - result = testdir.runpytest("-v") + result = pytester.runpytest("-v") assert message in result.stdout.str() diff --git a/tests/test_miscellaneous_crashes.py b/tests/test_miscellaneous_crashes.py index 0c47809a..4c78f47e 100644 --- a/tests/test_miscellaneous_crashes.py +++ b/tests/test_miscellaneous_crashes.py @@ -17,10 +17,10 @@ from .test_success_messages import SIMPLE_ECHO -def test_same_name_different_files(testdir): - testdir.makefile(".yml", test_a=SIMPLE_ECHO) - testdir.makefile(".yml", test_b=SIMPLE_ECHO) - result = testdir.runpytest() +def test_same_name_different_files(pytester): + pytester.makefile(".yml", test_a=SIMPLE_ECHO) + pytester.makefile(".yml", test_b=SIMPLE_ECHO) + result = pytester.runpytest() assert result.ret != 0 assert ("Workflow name 'simple echo' used more than once" in result.stdout.str()) diff --git a/tests/test_multithreading.py b/tests/test_multithreading.py index 91fee6ed..51498800 100644 --- a/tests/test_multithreading.py +++ b/tests/test_multithreading.py @@ -36,10 +36,10 @@ @pytest.mark.parametrize(["threads"], [(1,), (2,), (4,)]) -def test_multithreaded(threads, testdir): +def test_multithreaded(threads, pytester): test = MULTHITHREADED_TEST test_number = len(test) - testdir.makefile(".yml", test=yaml.safe_dump(test)) + pytester.makefile(".yml", test=yaml.safe_dump(test)) # Calculate how many iterations are needed to process all the tests # For example: 4 tests with 2 threads. 2 can be finished simultaneously @@ -49,7 +49,7 @@ def test_multithreaded(threads, testdir): else test_number // threads + 1) start_time = time.time() - testdir.runpytest("-v", "--wt", str(threads)) + pytester.runpytest("-v", "--wt", str(threads)) end_time = time.time() completion_time = end_time - start_time # If the completion time is shorter than (iterations * SLEEP_TIME), too diff --git a/tests/test_success_messages.py b/tests/test_success_messages.py index 8815bc1f..209e5300 100644 --- a/tests/test_success_messages.py +++ b/tests/test_success_messages.py @@ -110,7 +110,7 @@ @pytest.fixture(scope="session") def succeeding_tests_output(tmp_path_factory: pytest.TempPathFactory): - """This fixture was written because the testdir function has a default + """This fixture was written because the pytester function has a default scope of 'function'. This is very inefficient when testing multiple success messages in the output as the whole test yaml with all commands has to be run again. @@ -137,8 +137,8 @@ def test_message_success_no_errors_or_fails(succeeding_tests_output): assert "FAIL" not in succeeding_tests_output -def test_message_directory_kept_no_errors_or_fails(testdir): - testdir.makefile(".yml", test=SUCCEEDING_TESTS_YAML) - result = testdir.runpytest("-v", "--keep-workflow-wd") +def test_message_directory_kept_no_errors_or_fails(pytester): + pytester.makefile(".yml", test=SUCCEEDING_TESTS_YAML) + result = pytester.runpytest("-v", "--keep-workflow-wd") assert "ERROR" not in result.stdout.str() assert "FAIL" not in result.stdout.str() diff --git a/tests/test_tags.py b/tests/test_tags.py index b37b4cd8..aa1282ce 100644 --- a/tests/test_tags.py +++ b/tests/test_tags.py @@ -39,35 +39,35 @@ """) -def test_name_tag_with_space(testdir): - testdir.makefile(".yml", test_tags=TAG_TESTS) - result = testdir.runpytest("-v", "--tag", "three again").stdout.str() +def test_name_tag_with_space(pytester): + pytester.makefile(".yml", test_tags=TAG_TESTS) + result = pytester.runpytest("-v", "--tag", "three again").stdout.str() assert "three again" in result assert "four" not in result assert "nine" not in result -def test_name_tag(testdir): - testdir.makefile(".yml", test_tags=TAG_TESTS) - result = testdir.runpytest("-v", "--tag", "three").stdout.str() +def test_name_tag(pytester): + pytester.makefile(".yml", test_tags=TAG_TESTS) + result = pytester.runpytest("-v", "--tag", "three").stdout.str() assert "three" in result assert "three again" not in result assert "four" not in result assert "nine" not in result -def test_category_tag(testdir): - testdir.makefile(".yml", test_tags=TAG_TESTS) - result = testdir.runpytest("-v", "--tag", "odd").stdout.str() +def test_category_tag(pytester): + pytester.makefile(".yml", test_tags=TAG_TESTS) + result = pytester.runpytest("-v", "--tag", "odd").stdout.str() assert "three" in result assert "three again" in result assert "nine" in result assert "four" not in result -def test_category_tag2(testdir): - testdir.makefile(".yml", test_tags=TAG_TESTS) - result = testdir.runpytest("-v", "--tag", "even").stdout.str() +def test_category_tag2(pytester): + pytester.makefile(".yml", test_tags=TAG_TESTS) + result = pytester.runpytest("-v", "--tag", "even").stdout.str() assert "three" not in result assert "three again" not in result assert "four" in result diff --git a/tests/test_temp_directory.py b/tests/test_temp_directory.py index 85319c50..3aa45e0d 100644 --- a/tests/test_temp_directory.py +++ b/tests/test_temp_directory.py @@ -21,23 +21,26 @@ import textwrap from pathlib import Path +import pytest + from .test_success_messages import SIMPLE_ECHO -def test_directory_kept(testdir): - testdir.makefile(".yml", test=SIMPLE_ECHO) - result = testdir.runpytest("-v", "--keep-workflow-wd") - working_dir = re.search(r"command: echo moo\n\tdirectory: ([\w/_-]*)", - result.stdout.str()).group(1) +def test_directory_kept(pytester: pytest.Pytester): + pytester.makefile(".yml", test=SIMPLE_ECHO) + result = pytester.runpytest("-v", "--keep-workflow-wd") + match = re.search(r"command: echo moo\n\tdirectory: ([\w/_-]*)", + result.stdout.str()) + assert match is not None + working_dir = match.group(1) assert Path(working_dir).exists() assert Path(working_dir, "log.out").exists() assert Path(working_dir, "log.err").exists() - shutil.rmtree(str(testdir)) -def test_directory_not_kept(testdir): - testdir.makefile(".yml", test=SIMPLE_ECHO) - result = testdir.runpytest("-v") +def test_directory_not_kept(pytester): + pytester.makefile(".yml", test=SIMPLE_ECHO) + result = pytester.runpytest("-v") working_dir = re.search(r"command: echo moo\n\tdirectory: ([\w/_-]*)", result.stdout.str()).group(1) assert not Path(working_dir).exists() @@ -46,10 +49,10 @@ def test_directory_not_kept(testdir): ) in result.stdout.str() -def test_basetemp_correct(testdir): - testdir.makefile(".yml", test=SIMPLE_ECHO) +def test_basetemp_correct(pytester): + pytester.makefile(".yml", test=SIMPLE_ECHO) tempdir = tempfile.mkdtemp() - result = testdir.runpytest("-v", "--basetemp", tempdir) + result = pytester.runpytest("-v", "--basetemp", tempdir) message = (f"\tcommand: echo moo\n" f"\tdirectory: {tempdir}/simple_echo\n" f"\tstdout: {tempdir}/simple_echo/log.out\n" @@ -57,16 +60,16 @@ def test_basetemp_correct(testdir): assert message in result.stdout.str() -def test_basetemp_can_be_used_twice(testdir): - testdir.makefile(".yml", test=SIMPLE_ECHO) +def test_basetemp_can_be_used_twice(pytester): + pytester.makefile(".yml", test=SIMPLE_ECHO) tempdir = tempfile.mkdtemp() # First run to fill up tempdir - testdir.runpytest("-v", "--keep-workflow-wd", "--basetemp", tempdir) + pytester.runpytest("-v", "--keep-workflow-wd", "--basetemp", tempdir) # Make sure directory is there. assert Path(tempdir, "simple_echo").exists() # Run again with same basetemp. - result = testdir.runpytest("-v", "--keep-workflow-wd", "--basetemp", - tempdir) + result = pytester.runpytest("-v", "--keep-workflow-wd", "--basetemp", + tempdir) exit_code = result.ret assert (f"'{tempdir}/simple_echo' already exists. Deleting ..." in result.stdout.str()) @@ -74,27 +77,27 @@ def test_basetemp_can_be_used_twice(testdir): shutil.rmtree(tempdir) -def test_basetemp_will_be_created(testdir): - testdir.makefile(".yml", test=SIMPLE_ECHO) +def test_basetemp_will_be_created(pytester): + pytester.makefile(".yml", test=SIMPLE_ECHO) # This creates an empty dir tempdir_base = tempfile.mkdtemp() # This path should not exist tempdir = Path(tempdir_base, "non", "existing") # If pytest-workflow does not handle non-existing nested directories well # it should crash. - result = testdir.runpytest("-v", "--keep-workflow-wd", "--basetemp", - str(tempdir)) + result = pytester.runpytest("-v", "--keep-workflow-wd", "--basetemp", + str(tempdir)) assert tempdir.exists() assert result.ret == 0 shutil.rmtree(tempdir_base) -def test_basetemp_can_not_be_in_rootdir(testdir): - testdir.makefile(".yml", test=SIMPLE_ECHO) - testdir_path = Path(str(testdir.tmpdir)) - tempdir = testdir_path / "tmp" - result = testdir.runpytest("-v", "--basetemp", str(tempdir)) - message = f"'{str(tempdir)}' is a subdirectory of '{str(testdir_path)}'" +def test_basetemp_can_not_be_in_rootdir(pytester: pytest.Pytester): + pytester.makefile(".yml", test=SIMPLE_ECHO) + pytester.makefile(".yml", test=SIMPLE_ECHO) + tempdir = pytester.path / "tmp" + result = pytester.runpytest("-v", "--basetemp", str(tempdir)) + message = f"'{str(tempdir)}' is a subdirectory of '{str(pytester.path)}'" assert message in result.stderr.str() @@ -109,9 +112,9 @@ def test_basetemp_can_not_be_in_rootdir(testdir): """ -def test_directory_kept_on_fail(testdir): - testdir.makefile(".yml", test=FAIL_TEST) - result = testdir.runpytest("-v", "--keep-workflow-wd-on-fail") +def test_directory_kept_on_fail(pytester): + pytester.makefile(".yml", test=FAIL_TEST) + result = pytester.runpytest("-v", "--keep-workflow-wd-on-fail") working_dir = re.search( r"command: bash -c 'exit 1'\n\tdirectory: ([\w/_-]*)", result.stdout.str()).group(1) @@ -123,9 +126,9 @@ def test_directory_kept_on_fail(testdir): shutil.rmtree(working_dir) -def test_directory_not_kept_on_succes(testdir): - testdir.makefile(".yml", test=SUCCESS_TEST) - result = testdir.runpytest("-v", "--kwdof") +def test_directory_not_kept_on_succes(pytester): + pytester.makefile(".yml", test=SUCCESS_TEST) + result = pytester.runpytest("-v", "--kwdof") working_dir = re.search( r"command: bash -c 'exit 0'\n\tdirectory: ([\w/_-]*)", result.stdout.str()).group(1) @@ -134,11 +137,11 @@ def test_directory_not_kept_on_succes(testdir): result.stdout.str()) -def test_directory_of_symlinks(testdir): - testdir.makefile(".yml", test=SIMPLE_ECHO) - subdir = testdir.mkdir("subdir") +def test_directory_of_symlinks(pytester): + pytester.makefile(".yml", test=SIMPLE_ECHO) + subdir = pytester.mkdir("subdir") Path(str(subdir), "subfile.txt").write_text("test") - result = testdir.runpytest("-v", "--symlink", "--kwd") + result = pytester.runpytest("-v", "--symlink", "--kwd") working_dir = re.search( r"command: echo moo\n\tdirectory: ([\w/_-]*)", result.stdout.str()).group(1) @@ -148,7 +151,7 @@ def test_directory_of_symlinks(testdir): shutil.rmtree(working_dir) -def test_directory_unremovable_message(testdir): +def test_directory_unremovable_message(pytester): # Following directory contains nested contents owned by root. test = textwrap.dedent(""" - name: Create unremovable dir @@ -156,8 +159,8 @@ def test_directory_unremovable_message(testdir): bash -c "docker run -v $(pwd):$(pwd) -w $(pwd) debian \ bash -c 'mkdir test && touch test/test'" """) - testdir.makefile(".yaml", test=test) - result = testdir.runpytest() + pytester.makefile(".yaml", test=test) + result = pytester.runpytest() assert ("Unable to remove the following directories due to permission " "errors" in result.stdout.str()) assert result.ret == 0 diff --git a/tests/test_threading_errors.py b/tests/test_threading_errors.py index 860a440d..910f96cb 100644 --- a/tests/test_threading_errors.py +++ b/tests/test_threading_errors.py @@ -19,13 +19,13 @@ import textwrap -def test_shlex_error(testdir): +def test_shlex_error(pytester): test = textwrap.dedent("""\ - name: wrong command command: a command with a dangling double-quote" """) - testdir.makefile(".yml", test=test) - result = testdir.runpytest("-v") + pytester.makefile(".yml", test=test) + result = pytester.runpytest("-v") assert "shlex" in result.stdout.str() assert "ValueError: No closing quotation" in result.stdout.str() assert "'wrong command' python error during start" in result.stdout.str() diff --git a/tests/test_warnings.py b/tests/test_warnings.py index 24589f95..3b68f36b 100644 --- a/tests/test_warnings.py +++ b/tests/test_warnings.py @@ -22,20 +22,20 @@ # Add a test to make sure no deprecation, or other warnings occur due to # changes in upstream libraries. -def test_no_warnings(testdir): +def test_no_warnings(pytester): basetemp = tempfile.mkdtemp() - testdir.makefile(".yml", test_a=MOO_FILE) - result = testdir.runpytest("--basetemp", basetemp) + pytester.makefile(".yml", test_a=MOO_FILE) + result = pytester.runpytest("--basetemp", basetemp) outcomes = result.parseoutcomes() assert outcomes.get('warnings', 0) == 0 shutil.rmtree(basetemp) -def test_git_warning(testdir): +def test_git_warning(pytester): basetemp = tempfile.mkdtemp() - testdir.mkdir(".git") - testdir.makefile(".yml", test_a=MOO_FILE) - result = testdir.runpytest("--basetemp", basetemp) + pytester.mkdir(".git") + pytester.makefile(".yml", test_a=MOO_FILE) + result = pytester.runpytest("--basetemp", basetemp) outcomes = result.parseoutcomes() assert outcomes.get('warnings') == 1 assert ".git dir detected" in result.stdout.str() diff --git a/tests/test_workflow_dependent_tests.py b/tests/test_workflow_dependent_tests.py index 48959a23..66711297 100644 --- a/tests/test_workflow_dependent_tests.py +++ b/tests/test_workflow_dependent_tests.py @@ -40,28 +40,28 @@ def test_hook_impl(): @pytest.mark.parametrize("test", [TEST_HOOK_ARGS]) -def test_not_skipped(test, testdir): - testdir.makefile(".yml", test_simple=SIMPLE_ECHO) - testdir.makefile(".py", test_hook=test) - result = testdir.runpytest() +def test_not_skipped(test, pytester): + pytester.makefile(".yml", test_simple=SIMPLE_ECHO) + pytester.makefile(".py", test_hook=test) + result = pytester.runpytest() result.assert_outcomes(passed=5) -def test_name_use_is_deprecated(testdir): - testdir.makefile(".py", test_hook=TEST_HOOK_KWARGS) - testdir.makefile(".yml", test_simple=SIMPLE_ECHO) - result = testdir.runpytest().stdout.str() +def test_name_use_is_deprecated(pytester): + pytester.makefile(".py", test_hook=TEST_HOOK_KWARGS) + pytester.makefile(".yml", test_simple=SIMPLE_ECHO) + result = pytester.runpytest().stdout.str() assert "Use pytest.mark.workflow('workflow_name') instead." in result assert "DeprecationWarning" in result @pytest.mark.parametrize("test", [TEST_HOOK_ARGS]) -def test_skipped(test, testdir): - testdir.makefile(".yml", test_simple=SIMPLE_ECHO) - testdir.makefile(".py", test_hook=test) +def test_skipped(test, pytester): + pytester.makefile(".yml", test_simple=SIMPLE_ECHO) + pytester.makefile(".py", test_hook=test) # No workflow will run because the tag does not match # ``-r s`` gives the reason why a test was skipped. - result = testdir.runpytest("-v", "-r", "s", "--tag", "flaksdlkad") + result = pytester.runpytest("-v", "-r", "s", "--tag", "flaksdlkad") result.assert_outcomes(skipped=1) assert "'simple echo' has not run." in result.stdout.str() @@ -77,30 +77,30 @@ def test_fixture_impl(workflow_dir): @pytest.mark.parametrize("test", [TEST_FIXTURE_ARGS]) -def test_workflow_dir_arg(test, testdir): +def test_workflow_dir_arg(test, pytester): # Call the test, `test_asimple` because tests are run alphabetically. # This will detect if the workflow dir has been removed. - testdir.makefile(".yml", test_asimple=SIMPLE_ECHO) - testdir.makefile(".py", test_fixture=test) - result = testdir.runpytest() + pytester.makefile(".yml", test_asimple=SIMPLE_ECHO) + pytester.makefile(".py", test_fixture=test) + result = pytester.runpytest() result.assert_outcomes(passed=5, failed=0, errors=0, skipped=0) @pytest.mark.parametrize("test", [TEST_FIXTURE_ARGS]) -def test_workflow_dir_arg_skipped(test, testdir): +def test_workflow_dir_arg_skipped(test, pytester): """Run this test to check if this does not run into fixture request errors""" - testdir.makefile(".yml", test_asimple=SIMPLE_ECHO) - testdir.makefile(".py", test_fixture=test) - result = testdir.runpytest("-v", "-r", "s", "--tag", "flaksdlkad") + pytester.makefile(".yml", test_asimple=SIMPLE_ECHO) + pytester.makefile(".py", test_fixture=test) + result = pytester.runpytest("-v", "-r", "s", "--tag", "flaksdlkad") result.assert_outcomes(skipped=1) @pytest.mark.parametrize("test", [TEST_FIXTURE_ARGS]) -def test_mark_not_unknown(test, testdir): - testdir.makefile(".yml", test_asimple=SIMPLE_ECHO) - testdir.makefile(".py", test_fixture=test) - result = testdir.runpytest("-v") +def test_mark_not_unknown(test, pytester): + pytester.makefile(".yml", test_asimple=SIMPLE_ECHO) + pytester.makefile(".py", test_fixture=test) + result = pytester.runpytest("-v") assert "PytestUnknownMarkWarning" not in result.stdout.str() @@ -113,12 +113,12 @@ def test_fixture_impl(workflow_dir): """) -def test_workflow_not_exist_dir_arg(testdir): +def test_workflow_not_exist_dir_arg(pytester): """Run this test to check if this does not run into fixture request errors""" - testdir.makefile(".yml", test_asimple=SIMPLE_ECHO) - testdir.makefile(".py", test_fixture=TEST_FIXTURE_WORKFLOW_NOT_EXIST) - result = testdir.runpytest("-v", "-r", "s") + pytester.makefile(".yml", test_asimple=SIMPLE_ECHO) + pytester.makefile(".py", test_fixture=TEST_FIXTURE_WORKFLOW_NOT_EXIST) + result = pytester.runpytest("-v", "-r", "s") result.assert_outcomes(skipped=1, passed=4) assert "'shoobiedoewap' has not run." in result.stdout.str() @@ -131,12 +131,12 @@ def test_fixture_impl(workflow_dir): """) -def test_fixture_unmarked_test(testdir): +def test_fixture_unmarked_test(pytester): """Run this test to check if this does not run into fixture request errors""" - testdir.makefile(".yml", test_asimple=SIMPLE_ECHO) - testdir.makefile(".py", test_fixture=TEST_FIXTURE_UNMARKED_TEST) - result = testdir.runpytest("-v", "-r", "s") + pytester.makefile(".yml", test_asimple=SIMPLE_ECHO) + pytester.makefile(".py", test_fixture=TEST_FIXTURE_UNMARKED_TEST) + result = pytester.runpytest("-v", "-r", "s") assert ("workflow_dir can only be requested in tests marked with " "the workflow mark.") in result.stdout.str() @@ -151,12 +151,12 @@ def test_fixture_impl(workflow_dir): """) -def test_mark_wrong_key_with_fixture(testdir): +def test_mark_wrong_key_with_fixture(pytester): """Run this test to check if this does not run into fixture request errors""" - testdir.makefile(".yml", test_asimple=SIMPLE_ECHO) - testdir.makefile(".py", test_fixture=TEST_MARK_WRONG_KEY) - result = testdir.runpytest("-v", "-r", "s") + pytester.makefile(".yml", test_asimple=SIMPLE_ECHO) + pytester.makefile(".py", test_fixture=TEST_MARK_WRONG_KEY) + result = pytester.runpytest("-v", "-r", "s") assert ("A workflow name or names should be defined in the " "workflow marker of test_fixture.py::test_fixture_impl" ) in result.stdout.str() @@ -164,7 +164,7 @@ def test_mark_wrong_key_with_fixture(testdir): result.assert_outcomes(passed=0, failed=0, skipped=0, errors=1) -def test_fixture_usable_for_file_tests(testdir): +def test_fixture_usable_for_file_tests(pytester): test_workflow = textwrap.dedent("""\ - name: number files command: >- @@ -189,13 +189,13 @@ def test_div_by_three(workflow_dir): assert int(number_file_content) % 3 == 0 """) - testdir.makefile(".yml", test_aworkflow=test_workflow) - testdir.makefile(".py", test_div=test_div_by_three) - result = testdir.runpytest("-v") + pytester.makefile(".yml", test_aworkflow=test_workflow) + pytester.makefile(".py", test_div=test_div_by_three) + result = pytester.runpytest("-v") result.assert_outcomes(passed=4, failed=0, skipped=0, errors=0) -def test_same_custom_test_multiple_times(testdir): +def test_same_custom_test_multiple_times(pytester): test_workflow = textwrap.dedent("""\ - name: one_two_three command: >- @@ -226,9 +226,9 @@ def test_div_by_three(workflow_dir): assert int(number_file.read_text()) % 3 == 0 """) - testdir.makefile(".yml", test_aworkflow=test_workflow) - testdir.makefile(".py", test_div=test_div_by_three) - result = testdir.runpytest("-v") + pytester.makefile(".yml", test_aworkflow=test_workflow) + pytester.makefile(".py", test_div=test_div_by_three) + result = pytester.runpytest("-v") result.assert_outcomes(passed=9, failed=0, skipped=0, errors=0) @@ -263,19 +263,19 @@ def test_div_by_three(workflow_dir): """) -def test_same_custom_test_multiple_times_one_error(testdir): - testdir.makefile(".yml", test_aworkflow=TEST_WORKFLOWS) - testdir.makefile(".py", test_div=TEST_DIV_BY_THREE) - result = testdir.runpytest("-v") +def test_same_custom_test_multiple_times_one_error(pytester): + pytester.makefile(".yml", test_aworkflow=TEST_WORKFLOWS) + pytester.makefile(".py", test_div=TEST_DIV_BY_THREE) + result = pytester.runpytest("-v") result.assert_outcomes(passed=8, failed=1, skipped=0, errors=0) assert ("test_div.py::test_div_by_three[two_three_five] FAILED " in result.stdout.str()) -def test_custom_tests_properly_skipped(testdir): - testdir.makefile(".yml", test_aworkflow=TEST_WORKFLOWS) - testdir.makefile(".py", test_div=TEST_DIV_BY_THREE) - result = testdir.runpytest("-v", "--tag", "one_two_three") +def test_custom_tests_properly_skipped(pytester): + pytester.makefile(".yml", test_aworkflow=TEST_WORKFLOWS) + pytester.makefile(".py", test_div=TEST_DIV_BY_THREE) + result = pytester.runpytest("-v", "--tag", "one_two_three") result.assert_outcomes(passed=3, failed=0, skipped=2, errors=0) assert ("test_div.py::test_div_by_three[two_three_five] SKIPPED " in result.stdout.str()) From 7e8c6cf0b156cd0a99fcd1d8aca73711d7d2371e Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 31 May 2022 15:01:30 +0200 Subject: [PATCH 15/94] Document the use of environment variables with pytest-workflow --- HISTORY.rst | 1 + docs/writing_tests.rst | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 2bfdfa0f..0d72de4b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ Changelog version 1.7.0-dev --------------------------- ++ Document the use of environment variables with pytest-workflow. + A minimum of pytest 7.0.0 is now a requirement for pytest-workflow. This fixes the deprecation warnings that started on the release of pytest 7.0.0. diff --git a/docs/writing_tests.rst b/docs/writing_tests.rst index f9b32b87..6adb5c6f 100644 --- a/docs/writing_tests.rst +++ b/docs/writing_tests.rst @@ -90,6 +90,30 @@ sequences. Workflow names must be unique. Pytest workflow will crash when multiple workflows have the same name, even if they are in different files. +Environment variables +---------------------- +Pytest-workflow runs tests in the same environment as in which the pytest +executable was started. However, environment variables are quoted by pytest-workflow +using `shlex.quote `_. + +.. code-block:: YAML + + - name: Try to use an environment variable + command: echo $MY_VAR + # Output will be literally "$MY_VAR" + + - name: Circumenvent shlex quoting by explicitly starting the command in a shell. + command: bash -c 'echo $MY_VAR' + # Output will be the content of $MY_VAR + + - name: Use a program that checks an environment variable + command: singularity run my_container.sif + # Correctly uses "SINGULARITY_" prefixed variables + +If you want to use shell scripting features such as environment +variables inside ``command``, you need to explicitly set the shell as shown +above. + Writing custom tests -------------------- From 904b0f74c6b4d56c2d618c0d48de80efaff400f4 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 31 May 2022 15:04:30 +0200 Subject: [PATCH 16/94] Clarify a bit better --- docs/writing_tests.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/writing_tests.rst b/docs/writing_tests.rst index 6adb5c6f..62086d68 100644 --- a/docs/writing_tests.rst +++ b/docs/writing_tests.rst @@ -93,8 +93,11 @@ sequences. Environment variables ---------------------- Pytest-workflow runs tests in the same environment as in which the pytest -executable was started. However, environment variables are quoted by pytest-workflow -using `shlex.quote `_. +executable was started. This means programs started in tests can use +environnment variables. However, environment variables inside the command +section itself are quoted by pytest-workflow using +`shlex.quote `_. +See the examples below: .. code-block:: YAML From 7543784c309c942cc6f812710f51b8668d542202 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 31 May 2022 16:00:31 +0200 Subject: [PATCH 17/94] Explicitly set documentation language to English --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 873b5186..41038374 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -64,7 +64,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. From e03acab4bf1cfe763a5401f1c671aff994b0916b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 31 May 2022 16:11:38 +0200 Subject: [PATCH 18/94] Document using pytest.ini --- HISTORY.rst | 2 ++ docs/running_pytest_workflow.rst | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 0d72de4b..32630d7f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,8 @@ Changelog version 1.7.0-dev --------------------------- ++ Document using ``pytest.ini`` as a way of setting specific per repository + settings for pytest-workflow. + Document the use of environment variables with pytest-workflow. + A minimum of pytest 7.0.0 is now a requirement for pytest-workflow. This fixes the deprecation warnings that started on the release of pytest diff --git a/docs/running_pytest_workflow.rst b/docs/running_pytest_workflow.rst index 6bee58d7..cbe704d0 100644 --- a/docs/running_pytest_workflow.rst +++ b/docs/running_pytest_workflow.rst @@ -26,6 +26,20 @@ Specific pytest options for pytest workflow :func: __pytest_workflow_cli :prog: pytest +Setting specific per-project settings using pytest.ini +----------------------------------------------------------------- +pytest can be configured `using a pytest.ini file +`_. +This mechanic can be used to set specific settings in each repository +where pytest workflow is used. + +For example a pytest.ini with the following contents:: + + [pytest] + addopts = --git-aware + +Can be used in a git repository with a workflow. + Temporary directory cleanup and creation ---------------------------------------- From 1ad509ffd1d4f6a9b38a6b82bc7f8c3c68dcf048 Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 10 Oct 2022 11:12:17 +0200 Subject: [PATCH 19/94] Nextflow test (#143) * Add files via upload * Create test * Add files via upload * Delete tests/pipelines/Nextflow directory * Rename tests/pipelines/nextflow_pipeline.nf to tests/pipelines/nextflow/nextflow_pipeline.nf * Add files via upload * Rename nextflow_pipeline.nf to nextflow_testpipeline.nf * Update test_functional.py added nextflow files and test * Update tox.ini added the testenv for nextflow * Update simple_nextflow_test_cases.yml * fixed indentation issue in test_funcional.py and added correct file types to simple_nextflow_test_cases.yml * Update HISTORY.rst * Update tox.ini --- HISTORY.rst | 1 + .../functional/simple_nextflow_test_cases.yml | 35 ++++++++++ tests/functional/test_functional.py | 11 ++++ .../nextflow/nextflow_testpipeline.nf | 64 +++++++++++++++++++ tox.ini | 7 ++ 5 files changed, 118 insertions(+) create mode 100644 tests/functional/simple_nextflow_test_cases.yml create mode 100644 tests/pipelines/nextflow/nextflow_testpipeline.nf diff --git a/HISTORY.rst b/HISTORY.rst index 0d72de4b..5d0a4804 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ Changelog version 1.7.0-dev --------------------------- ++ Add tests for nextflow. + Document the use of environment variables with pytest-workflow. + A minimum of pytest 7.0.0 is now a requirement for pytest-workflow. This fixes the deprecation warnings that started on the release of pytest diff --git a/tests/functional/simple_nextflow_test_cases.yml b/tests/functional/simple_nextflow_test_cases.yml new file mode 100644 index 00000000..17b8096d --- /dev/null +++ b/tests/functional/simple_nextflow_test_cases.yml @@ -0,0 +1,35 @@ +- name: simple nextflow + command: >- + nextflow run nextflow_testpipeline.nf --N_LINES_TO_READ=10 + files: + - path: results/rand/rand_0.txt + - path: results/rand/rand_1.txt + - path: results/rand/rand_2.txt + - path: results/rand/rand_3.txt + - path: results/rand/rand_4.txt + - path: results/rand/rand_5.txt + - path: results/rand/rand_6.txt + - path: results/rand/rand_7.txt + - path: results/rand/rand_8.txt + - path: results/rand/rand_9.txt + - path: results/b64/b64_0.txt + - path: results/b64/b64_1.txt + - path: results/b64/b64_2.txt + - path: results/b64/b64_3.txt + - path: results/b64/b64_4.txt + - path: results/b64/b64_5.txt + - path: results/b64/b64_6.txt + - path: results/b64/b64_7.txt + - path: results/b64/b64_8.txt + - path: results/b64/b64_9.txt + - path: results/randgz/randgz_0.txt.gz + - path: results/randgz/randgz_1.txt.gz + - path: results/randgz/randgz_2.txt.gz + - path: results/randgz/randgz_3.txt.gz + - path: results/randgz/randgz_4.txt.gz + - path: results/randgz/randgz_5.txt.gz + - path: results/randgz/randgz_6.txt.gz + - path: results/randgz/randgz_7.txt.gz + - path: results/randgz/randgz_8.txt.gz + - path: results/randgz/randgz_9.txt.gz + - path: results/all_data.gz diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.py index 58e2e4a6..44c0cff2 100644 --- a/tests/functional/test_functional.py +++ b/tests/functional/test_functional.py @@ -19,11 +19,13 @@ SIMPLE_WDL_YAML = Path(__file__).parent / "simple_wdl_test_cases.yml" SNAKEFILE_YAML = Path(__file__).parent / "simple_snakefile_test_cases.yml" +NEXTFLOW_YAML = Path(__file__).parent / "simple_nextflow_test_cases.yml" PIPELINE_DIR = Path(__file__).parent.parent / "pipelines" SIMPLE_WDL = Path(PIPELINE_DIR, "wdl", "simple.wdl") SIMPLE_WDL_JSON = Path(PIPELINE_DIR, "wdl", "simple.json") SIMPLE_WDL_OPTIONS_JSON = Path(PIPELINE_DIR, "wdl", "simple.options.json") SNAKEFILE = Path(PIPELINE_DIR, "snakemake", "SimpleSnakefile") +NEXTFLOWFILE = Path(PIPELINE_DIR, "nextflow", "nextflow_testpipeline.nf") @pytest.mark.functional @@ -57,3 +59,12 @@ def test_snakemake(pytester): result = pytester.runpytest("-v", "--keep-workflow-wd-on-fail") exit_code = result.ret assert exit_code == 0 + + +@pytest.mark.functional +def test_nextflow(pytester): + pytester.makefile(ext=".nf", + nextflow_testpipeline=NEXTFLOWFILE.read_text()) + pytester.makefile(ext=".yml", test_nextflow=NEXTFLOW_YAML.read_text()) + result = pytester.runpytest("-v", "--keep-workflow-wd-on-fail") + assert result.ret == 0 diff --git a/tests/pipelines/nextflow/nextflow_testpipeline.nf b/tests/pipelines/nextflow/nextflow_testpipeline.nf new file mode 100644 index 00000000..6fd32eb8 --- /dev/null +++ b/tests/pipelines/nextflow/nextflow_testpipeline.nf @@ -0,0 +1,64 @@ +#!/usr/bin/env nextflow + +params.N_LINES_TO_READ = 5 + +process read_random { + publishDir 'results/rand' + + input: + val iter + output: + val iter + path "rand_${iter}.txt" + script: + """ + head -n ${params.N_LINES_TO_READ} /dev/urandom > rand_${iter}.txt + """ +} + +process base64_random { + publishDir 'results/b64' + + input: + val iter + path "rand_${iter}.txt" + output: + val iter + path "b64_${iter}.txt" + script: + """ + cat rand_${iter}.txt | base64 > b64_${iter}.txt + """ +} + +process gzip_b64 { + publishDir 'results/randgz' + + input: + val iter + path "b64_${iter}.txt" + output: + path "randgz_${iter}.txt.gz" + script: + """ + cat b64_${iter}.txt | gzip -c > randgz_${iter}.txt.gz + """ +} + +process concat_gzip { + publishDir 'results' + + input: + path 'x' + output: + path "all_data.gz" + script: + """ + cat ${x} > all_data.gz + """ +} + +workflow { + iterations = channel.from(0..9) | read_random | base64_random | gzip_b64 + gzip_b64.out.collect() | concat_gzip +} diff --git a/tox.ini b/tox.ini index 6b75d6c2..23638e3e 100644 --- a/tox.ini +++ b/tox.ini @@ -59,3 +59,10 @@ commands = deps=miniwdl commands= python -m pytest tests/functional/test_functional.py::test_miniwdl + +[testenv:nextflow] +# Empty deps. Otherwise deps from default testenv are used. +deps= +commands = + python -m pytest tests/functional/test_functional.py::test_nextflow + From 2c3706f79b7e6ac12e7668b7b734402b02b4ac52 Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 10 Oct 2022 11:20:31 +0200 Subject: [PATCH 20/94] Added licence header --- .../nextflow/nextflow_testpipeline.nf | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/pipelines/nextflow/nextflow_testpipeline.nf b/tests/pipelines/nextflow/nextflow_testpipeline.nf index 6fd32eb8..1a1116f3 100644 --- a/tests/pipelines/nextflow/nextflow_testpipeline.nf +++ b/tests/pipelines/nextflow/nextflow_testpipeline.nf @@ -1,5 +1,25 @@ #!/usr/bin/env nextflow +# Copyright (C) 2018 Leiden University Medical Center +# This file is part of pytest-workflow +# +# pytest-workflow is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# pytest-workflow is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with pytest-workflow. If not, see Date: Mon, 10 Oct 2022 11:35:19 +0200 Subject: [PATCH 21/94] Added Nextflow example documentation --- docs/examples.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/examples.rst b/docs/examples.rst index 166e7e22..d1a2b4c5 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -109,3 +109,16 @@ The following yaml file tests a WDL pipeline run with miniwdl. Please note that the trailing slash in ``-d test-output/`` is important. It will ensure the files end up in the ``test-output`` directory. + +Nextflow example +----------------- + +An example yaml file that could be used to test a nextflow pipeline is listed +below. + +.. code-block:: yaml + + - name: My pipeline + command: nextflow run Nextflow_pipeline.nf + files: + - "my_output.txt" From 7489acb6c88b2be21e08bde7dffa93cfb394d965 Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 10 Oct 2022 13:14:54 +0200 Subject: [PATCH 22/94] expanded the nextflow example documentation --- docs/examples.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/examples.rst b/docs/examples.rst index d1a2b4c5..ed14c8bd 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -113,6 +113,30 @@ will ensure the files end up in the ``test-output`` directory. Nextflow example ----------------- +With nextflow each process is run in a unique directory where the output files will +also be stored. Nextflow can output a copy of the output files to a separate workflow-outputs directory. This can be achieved by defining a publishDir in the process. + +An example code defining a publishDir is listed below. + +.. code-block:: + + process Hello { + publishDir 'test-output' + + output: + path "HelloWorld.txt" + script: + """ + echo "Hello World!" > HelloWorld.txt + """ + } + + workflow { + Hello + } + +``publishDir`` will make it so that all the output files if the process are copied to the given directory. In this case HelloWorld.txt will be copied to the directory called test-output. + An example yaml file that could be used to test a nextflow pipeline is listed below. From 82bd9da2db4d381623564c7b183ddccb84c7fdfb Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 10 Oct 2022 13:21:38 +0200 Subject: [PATCH 23/94] fixed typos --- docs/examples.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/examples.rst b/docs/examples.rst index ed14c8bd..db762f35 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -114,9 +114,10 @@ Nextflow example ----------------- With nextflow each process is run in a unique directory where the output files will -also be stored. Nextflow can output a copy of the output files to a separate workflow-outputs directory. This can be achieved by defining a publishDir in the process. +also be stored. Nextflow can output a copy of the output files to a separate workflow-outputs +directory. This can be achieved by defining a ``publishDir`` in the process. -An example code defining a publishDir is listed below. +An example code defining a ``publishDir`` is listed below. .. code-block:: @@ -135,7 +136,7 @@ An example code defining a publishDir is listed below. Hello } -``publishDir`` will make it so that all the output files if the process are copied to the given directory. In this case HelloWorld.txt will be copied to the directory called test-output. +``publishDir`` will make it so that all the output files if the process are copied to the given directory. In this case ``HelloWorld.txt`` will be copied to the directory called ``test-output``. An example yaml file that could be used to test a nextflow pipeline is listed below. From 9bf213f277d4f0969e38dd62502aeefa653bfb68 Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 10 Oct 2022 14:38:18 +0200 Subject: [PATCH 24/94] Added outdir to nextflow_testpipeline.nf --- .../nextflow/nextflow_testpipeline.nf | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/pipelines/nextflow/nextflow_testpipeline.nf b/tests/pipelines/nextflow/nextflow_testpipeline.nf index 1a1116f3..20381250 100644 --- a/tests/pipelines/nextflow/nextflow_testpipeline.nf +++ b/tests/pipelines/nextflow/nextflow_testpipeline.nf @@ -23,8 +23,10 @@ params.N_LINES_TO_READ = 5 process read_random { - publishDir 'results/rand' - + publishDir = [ + path: { "${params.outdir}/rand'} + ] + input: val iter output: @@ -37,7 +39,9 @@ process read_random { } process base64_random { - publishDir 'results/b64' + publishDir = [ + path: { "${params.outdir}/b64'} + ] input: val iter @@ -52,7 +56,9 @@ process base64_random { } process gzip_b64 { - publishDir 'results/randgz' + publishDir = [ + path: { "${params.outdir}/randgz'} + ] input: val iter @@ -66,7 +72,9 @@ process gzip_b64 { } process concat_gzip { - publishDir 'results' + publishDir = [ + path: { "${params.outdir}'} + ] input: path 'x' From 49eb1b1be1c8788851fc1ebdea595fbf691a912c Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 10 Oct 2022 14:40:04 +0200 Subject: [PATCH 25/94] added outdir param to command --- tests/functional/simple_nextflow_test_cases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/simple_nextflow_test_cases.yml b/tests/functional/simple_nextflow_test_cases.yml index 17b8096d..c3c0f143 100644 --- a/tests/functional/simple_nextflow_test_cases.yml +++ b/tests/functional/simple_nextflow_test_cases.yml @@ -1,6 +1,6 @@ - name: simple nextflow command: >- - nextflow run nextflow_testpipeline.nf --N_LINES_TO_READ=10 + nextflow run nextflow_testpipeline.nf --N_LINES_TO_READ=10 --outdir results files: - path: results/rand/rand_0.txt - path: results/rand/rand_1.txt From ce6ea8fd32012c808ccfaa21240b5c3a5f188cbd Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 10 Oct 2022 14:59:50 +0200 Subject: [PATCH 26/94] added outdir to nextflow example documentation --- docs/examples.rst | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/examples.rst b/docs/examples.rst index db762f35..24adbab0 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -115,14 +115,17 @@ Nextflow example With nextflow each process is run in a unique directory where the output files will also be stored. Nextflow can output a copy of the output files to a separate workflow-outputs -directory. This can be achieved by defining a ``publishDir`` in the process. +directory. This can be achieved by defining a ``publishDir`` in the process. Through ``params.outdir`` +it is possible to define the output directory when running the code. An example code defining a ``publishDir`` is listed below. .. code-block:: process Hello { - publishDir 'test-output' + publishDir = [ + path: { "${params.outdir}/hello"} + ] output: path "HelloWorld.txt" @@ -136,7 +139,15 @@ An example code defining a ``publishDir`` is listed below. Hello } -``publishDir`` will make it so that all the output files if the process are copied to the given directory. In this case ``HelloWorld.txt`` will be copied to the directory called ``test-output``. +To run the code listed above the following command can be used in which ``examplecode.nf`` is the code listed above: + +.. code-block:: + + nextflow run examplecode.nf --outdir test-output + +``publishDir`` will make it so that all the output files of the process are copied to the given directory. +``--outdir`` is used to define the path the output files will go to. In this case ``HelloWorld.txt`` will +be copied to the directory called ``test-output/hello``. An example yaml file that could be used to test a nextflow pipeline is listed below. @@ -144,6 +155,6 @@ below. .. code-block:: yaml - name: My pipeline - command: nextflow run Nextflow_pipeline.nf + command: nextflow run Nextflow_pipeline.nf --outdir results files: - - "my_output.txt" + - path: "results/my_output.txt" From cd2f2b40fab979367519b4b1ce7c08bc061b2912 Mon Sep 17 00:00:00 2001 From: LucasvanBostelen <95374055+lucasvanb@users.noreply.github.com> Date: Wed, 12 Oct 2022 12:19:35 +0200 Subject: [PATCH 27/94] matched example yaml file to example code --- docs/examples.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/examples.rst b/docs/examples.rst index 24adbab0..cf918525 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -149,12 +149,12 @@ To run the code listed above the following command can be used in which ``exampl ``--outdir`` is used to define the path the output files will go to. In this case ``HelloWorld.txt`` will be copied to the directory called ``test-output/hello``. -An example yaml file that could be used to test a nextflow pipeline is listed +An example yaml file that could be used to test the nextflow pipeline from ``examplecode.nf`` is listed below. .. code-block:: yaml - name: My pipeline - command: nextflow run Nextflow_pipeline.nf --outdir results + command: nextflow run examplecode.nf --outdir test-output files: - - path: "results/my_output.txt" + - path: "test-output/hello/HelloWorld.txt" From 0827337a2e3333290c9cf4e5c0018f4ff09dba9d Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 17 Oct 2022 14:47:36 +0200 Subject: [PATCH 28/94] added standerd error and standerd out to exitcode --- src/pytest_workflow/plugin.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/pytest_workflow/plugin.py b/src/pytest_workflow/plugin.py index 40da55a8..fb7d5eae 100644 --- a/src/pytest_workflow/plugin.py +++ b/src/pytest_workflow/plugin.py @@ -19,6 +19,7 @@ import shutil import tempfile import warnings +import os from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -481,7 +482,18 @@ def runtest(self): assert self.workflow.exit_code == self.desired_exit_code def repr_failure(self, excinfo, style=None): - message = (f"'{self.workflow.name}' exited with exit code " + - f"'{self.workflow.exit_code}' instead of " - f"'{self.desired_exit_code}'.") + standerr = self.workflow.stderr_file + standout = self.workflow.stdout_file + with open(standout, "rb") as standout_file, \ + open(standerr, "rb") as standerr_file: + if os.path.getsize(standerr) >= 30: + standerr_file.seek(-30, 2) + if os.path.getsize(standout) >= 30: + standout_file.seek(-30, 2) + message = (f"'{self.workflow.name}' exited with exit code " + + f"'{self.workflow.exit_code}' instead of " + f"'{self.desired_exit_code}'.\nstderr: " + f"{standerr_file.read().strip().decode('utf-8')}" + f"\nstdout: " + f"{standout_file.read().strip().decode('utf-8')}") return message From 77774569b1532836cf91cb8884a8d9a3df8cd97d Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 28 Oct 2022 13:16:14 +0200 Subject: [PATCH 29/94] Use pytest instead of py.test --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 23638e3e..dfa109df 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ envlist=py3 deps=coverage commands = # Create HTML coverage report for humans and xml coverage report for external services. - coverage run --source=pytest_workflow -m py.test -v tests -m 'not functional' + coverage run --source=pytest_workflow -m pytest -v tests -m 'not functional' coverage html coverage xml From 701e64aa771597ca22ffa0b964e7983bc3542316 Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 28 Oct 2022 14:08:18 +0200 Subject: [PATCH 30/94] added failure message test for exitcode --- src/pytest_workflow/plugin.py | 22 +++++++++++++++------- tests/test_fail_messages.py | 5 +++++ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/pytest_workflow/plugin.py b/src/pytest_workflow/plugin.py index fb7d5eae..262acab5 100644 --- a/src/pytest_workflow/plugin.py +++ b/src/pytest_workflow/plugin.py @@ -73,7 +73,13 @@ def pytest_addoption(parser: PytestParser): "This ignores the .git directory, any untracked files and any " "files listed by .gitignore. " "Highly recommended when working in a git project.") - + parser.addoption( + "--sb", "--standerderror-bytes", + dest="standerderror_bytes", + default=1000, + type=int, + help="The number o bytes to display from the standard error and " + "standerd out on exitcode.") # Why `--tag ` and not simply use `pytest -m `? # `-m` uses a "mark expression". So you have to type a piece of python # code instead of just supplying the tags you want. This is fine for the @@ -449,7 +455,8 @@ def collect(self): tests += [ExitCodeTest.from_parent( parent=self, desired_exit_code=self.workflow_test.exit_code, - workflow=workflow)] + workflow=workflow, + standerderror_bytes=self.config.getoption("standerderror_bytes"))] tests += [ContentTestCollector.from_parent( name="stdout", parent=self, @@ -471,9 +478,10 @@ def collect(self): class ExitCodeTest(pytest.Item): def __init__(self, parent: pytest.Collector, desired_exit_code: int, - workflow: Workflow): + workflow: Workflow, standerderror_bytes: int): name = f"exit code should be {desired_exit_code}" super().__init__(name, parent=parent) + self.standerderror_bytes = standerderror_bytes self.workflow = workflow self.desired_exit_code = desired_exit_code @@ -486,10 +494,10 @@ def repr_failure(self, excinfo, style=None): standout = self.workflow.stdout_file with open(standout, "rb") as standout_file, \ open(standerr, "rb") as standerr_file: - if os.path.getsize(standerr) >= 30: - standerr_file.seek(-30, 2) - if os.path.getsize(standout) >= 30: - standout_file.seek(-30, 2) + if os.path.getsize(standerr) >= self.standerderror_bytes: + standerr_file.seek(-self.standerderror_bytes, 2) + if os.path.getsize(standout) >= self.standerderror_bytes: + standout_file.seek(-self.standerderror_bytes, 2) message = (f"'{self.workflow.name}' exited with exit code " + f"'{self.workflow.exit_code}' instead of " f"'{self.desired_exit_code}'.\nstderr: " diff --git a/tests/test_fail_messages.py b/tests/test_fail_messages.py index e1bdc97f..ae379fd5 100644 --- a/tests/test_fail_messages.py +++ b/tests/test_fail_messages.py @@ -29,6 +29,11 @@ """, "'fail_test' exited with exit code '2' instead of '0'"), ("""\ + - name: exitcode_test + command: bash -c 'printf "This code had an error" >&2 | exit 12' + """, + "stderr: This code had an error"), + ("""\ - name: create file command: echo moo files: From 051a701ab36dc64e590ee3965b900fefd951e58b Mon Sep 17 00:00:00 2001 From: LucasvanBostelen <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 28 Oct 2022 14:32:24 +0200 Subject: [PATCH 31/94] Update HISTORY.rst --- HISTORY.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 5d0a4804..0781c3fd 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,10 @@ Changelog version 1.7.0-dev --------------------------- ++ Add ``--standerderror-bytes`` or ``--sb`` option to change the maximum + number of bytes to display for the standerd error and standerd out on + command failure. ++ Add standerd error and standerd out to be displayed on command failure + Add tests for nextflow. + Document the use of environment variables with pytest-workflow. + A minimum of pytest 7.0.0 is now a requirement for pytest-workflow. From 4e923b05773b0caafd7409877588bddd38034d57 Mon Sep 17 00:00:00 2001 From: LucasvanBostelen <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 31 Oct 2022 10:16:03 +0100 Subject: [PATCH 32/94] changed standerd error/out to stderr/stdout --- HISTORY.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0781c3fd..3a90718a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,10 +9,10 @@ Changelog version 1.7.0-dev --------------------------- -+ Add ``--standerderror-bytes`` or ``--sb`` option to change the maximum - number of bytes to display for the standerd error and standerd out on ++ Add ``--stderr-bytes`` or ``--sb`` option to change the maximum + number of bytes to display for the stderr and stdout on command failure. -+ Add standerd error and standerd out to be displayed on command failure ++ Add stderr and stdout to be displayed on command failure + Add tests for nextflow. + Document the use of environment variables with pytest-workflow. + A minimum of pytest 7.0.0 is now a requirement for pytest-workflow. From 5cad7def89831d369d626787b3cff6208b217d78 Mon Sep 17 00:00:00 2001 From: LucasvanBostelen <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 31 Oct 2022 11:08:11 +0100 Subject: [PATCH 33/94] fixed typos and added SEEK_END --- src/pytest_workflow/plugin.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/pytest_workflow/plugin.py b/src/pytest_workflow/plugin.py index 262acab5..f0ceec60 100644 --- a/src/pytest_workflow/plugin.py +++ b/src/pytest_workflow/plugin.py @@ -74,12 +74,12 @@ def pytest_addoption(parser: PytestParser): "files listed by .gitignore. " "Highly recommended when working in a git project.") parser.addoption( - "--sb", "--standerderror-bytes", - dest="standerderror_bytes", + "--sb", "--stderr-bytes", + dest="stderr_bytes", default=1000, type=int, - help="The number o bytes to display from the standard error and " - "standerd out on exitcode.") + help="The number of bytes to display from the stderr and " + "stdout on exitcode.") # Why `--tag ` and not simply use `pytest -m `? # `-m` uses a "mark expression". So you have to type a piece of python # code instead of just supplying the tags you want. This is fine for the @@ -456,7 +456,7 @@ def collect(self): parent=self, desired_exit_code=self.workflow_test.exit_code, workflow=workflow, - standerderror_bytes=self.config.getoption("standerderror_bytes"))] + stderr_bytes=self.config.getoption("stderr_bytes"))] tests += [ContentTestCollector.from_parent( name="stdout", parent=self, @@ -478,10 +478,10 @@ def collect(self): class ExitCodeTest(pytest.Item): def __init__(self, parent: pytest.Collector, desired_exit_code: int, - workflow: Workflow, standerderror_bytes: int): + workflow: Workflow, stderr_bytes: int): name = f"exit code should be {desired_exit_code}" super().__init__(name, parent=parent) - self.standerderror_bytes = standerderror_bytes + self.stderr_bytes = stderr_bytes self.workflow = workflow self.desired_exit_code = desired_exit_code @@ -494,10 +494,10 @@ def repr_failure(self, excinfo, style=None): standout = self.workflow.stdout_file with open(standout, "rb") as standout_file, \ open(standerr, "rb") as standerr_file: - if os.path.getsize(standerr) >= self.standerderror_bytes: - standerr_file.seek(-self.standerderror_bytes, 2) - if os.path.getsize(standout) >= self.standerderror_bytes: - standout_file.seek(-self.standerderror_bytes, 2) + if os.path.getsize(standerr) >= self.stderr_bytes: + standerr_file.seek(-self.stderr_bytes, os.SEEK_END) + if os.path.getsize(standout) >= self.stderr_bytes: + standout_file.seek(-self.stderr_bytes, os.SEEK_END) message = (f"'{self.workflow.name}' exited with exit code " + f"'{self.workflow.exit_code}' instead of " f"'{self.desired_exit_code}'.\nstderr: " From 8979e5f16ab0fd934fd3b66853b74b2a01d1f0a6 Mon Sep 17 00:00:00 2001 From: LucasvanBostelen <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 31 Oct 2022 13:04:28 +0100 Subject: [PATCH 34/94] fixed import order --- src/pytest_workflow/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pytest_workflow/plugin.py b/src/pytest_workflow/plugin.py index f0ceec60..8ff16693 100644 --- a/src/pytest_workflow/plugin.py +++ b/src/pytest_workflow/plugin.py @@ -16,10 +16,10 @@ """core functionality of pytest-workflow plugin""" import argparse +import os import shutil import tempfile import warnings -import os from pathlib import Path from typing import Any, Dict, List, Optional, Tuple From 5ecb43991b1958428799e3b08d877125a861b629 Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 31 Oct 2022 14:19:19 +0100 Subject: [PATCH 35/94] add tests for --stderr_bytes flag --- tests/test_fail_messages.py | 2 +- tests/test_stderr_bytes.py | 45 +++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 tests/test_stderr_bytes.py diff --git a/tests/test_fail_messages.py b/tests/test_fail_messages.py index ae379fd5..42507e78 100644 --- a/tests/test_fail_messages.py +++ b/tests/test_fail_messages.py @@ -30,7 +30,7 @@ "'fail_test' exited with exit code '2' instead of '0'"), ("""\ - name: exitcode_test - command: bash -c 'printf "This code had an error" >&2 | exit 12' + command: bash -c 'printf "This code had an error" >&2 ; exit 12' """, "stderr: This code had an error"), ("""\ diff --git a/tests/test_stderr_bytes.py b/tests/test_stderr_bytes.py new file mode 100644 index 00000000..e11c5d27 --- /dev/null +++ b/tests/test_stderr_bytes.py @@ -0,0 +1,45 @@ +# Copyright (C) 2018 Leiden University Medical Center +# This file is part of pytest-workflow +# +# pytest-workflow is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# pytest-workflow is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with pytest-workflow. If not, see &2 ; exit 12' + """, + "stderr: error") +] + + +@pytest.mark.parametrize(["test", "message"], EXITCODE_MESSAGE_TESTS) +def test_messages(test: str, message: str, pytester): + pytester.makefile(".yml", textwrap.dedent(test)) + # Ideally this should be run in a LC_ALL=C environment. But this is not + # possible due to multiple levels of process launching. + result = pytester.runpytest("-v", "--sb", "5") + assert message in result.stdout.str() From debfbacce3db0224ff57bc5a2fbc766c4e380af5 Mon Sep 17 00:00:00 2001 From: Lucas van Bostelen Date: Mon, 31 Oct 2022 16:04:30 +0100 Subject: [PATCH 36/94] add content of test_stderr_bytes.py file to test_fail_messages.py --- tests/test_stderr_bytes.py | 45 -------------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 tests/test_stderr_bytes.py diff --git a/tests/test_stderr_bytes.py b/tests/test_stderr_bytes.py deleted file mode 100644 index e11c5d27..00000000 --- a/tests/test_stderr_bytes.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (C) 2018 Leiden University Medical Center -# This file is part of pytest-workflow -# -# pytest-workflow is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# pytest-workflow is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with pytest-workflow. If not, see &2 ; exit 12' - """, - "stderr: error") -] - - -@pytest.mark.parametrize(["test", "message"], EXITCODE_MESSAGE_TESTS) -def test_messages(test: str, message: str, pytester): - pytester.makefile(".yml", textwrap.dedent(test)) - # Ideally this should be run in a LC_ALL=C environment. But this is not - # possible due to multiple levels of process launching. - result = pytester.runpytest("-v", "--sb", "5") - assert message in result.stdout.str() From 7b97431312cc852c9e517d4a7adb0d7e346c24bf Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 31 Oct 2022 16:22:59 +0100 Subject: [PATCH 37/94] add content of test_stderr_bytes.py to test_fail_messages.py --- tests/test_fail_messages.py | 23 +++++++++++++++++++ tests/test_stderr_bytes.py | 45 ------------------------------------- 2 files changed, 23 insertions(+), 45 deletions(-) delete mode 100644 tests/test_stderr_bytes.py diff --git a/tests/test_fail_messages.py b/tests/test_fail_messages.py index 42507e78..878b4caf 100644 --- a/tests/test_fail_messages.py +++ b/tests/test_fail_messages.py @@ -175,3 +175,26 @@ def test_messages(test: str, message: str, pytester): # possible due to multiple levels of process launching. result = pytester.runpytest("-v") assert message in result.stdout.str() + + +EXITCODE_MESSAGE_TESTS: List[Tuple[str, str]] = [ + ("""\ + - name: stderr_bytes_stderr_test + command: bash -c 'printf "This code had an error" ; exit 12' + """, + "stdout: error"), + ("""\ + - name: stderr_bytes_stdout_test + command: bash -c 'printf "This code had an error" >&2 ; exit 12' + """, + "stderr: error") +] + + +@pytest.mark.parametrize(["test", "message"], EXITCODE_MESSAGE_TESTS) +def test_messages_exitcode(test: str, message: str, pytester): + pytester.makefile(".yml", textwrap.dedent(test)) + # Ideally this should be run in a LC_ALL=C environment. But this is not + # possible due to multiple levels of process launching. + result = pytester.runpytest("-v", "--sb", "5") + assert message in result.stdout.str() diff --git a/tests/test_stderr_bytes.py b/tests/test_stderr_bytes.py deleted file mode 100644 index e11c5d27..00000000 --- a/tests/test_stderr_bytes.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (C) 2018 Leiden University Medical Center -# This file is part of pytest-workflow -# -# pytest-workflow is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# pytest-workflow is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with pytest-workflow. If not, see &2 ; exit 12' - """, - "stderr: error") -] - - -@pytest.mark.parametrize(["test", "message"], EXITCODE_MESSAGE_TESTS) -def test_messages(test: str, message: str, pytester): - pytester.makefile(".yml", textwrap.dedent(test)) - # Ideally this should be run in a LC_ALL=C environment. But this is not - # possible due to multiple levels of process launching. - result = pytester.runpytest("-v", "--sb", "5") - assert message in result.stdout.str() From 3eb4ecdf340d1869e2500f0676eeb8537f986c86 Mon Sep 17 00:00:00 2001 From: Redmar van den Berg Date: Wed, 2 Nov 2022 09:50:34 +0100 Subject: [PATCH 38/94] Add examples for testing Bash scripts --- docs/examples.rst | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/examples.rst b/docs/examples.rst index cf918525..ce239250 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -158,3 +158,49 @@ below. command: nextflow run examplecode.nf --outdir test-output files: - path: "test-output/hello/HelloWorld.txt" + +Bash example +------------ + +The following is an example of a Bash file that can run directly as a script, or sourced to test each function separately: + +.. code-block:: bash + + #!/usr/bin/env bash + + function say_hello() { + local name="$1" + echo "Hello, ${name}!" + } + + function main() { + say_hello world + } + + # Only execute main when this file is run as a script + if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main + fi + +Save the bash file as ``script.sh``, and test it with the following pytest-workflow configuration: + + +.. code-block:: yml + + - name: test bash script + command: bash script.sh + stdout: + contains: + - "Hello, world!" + + - name: test bash function + command: > + bash -c " + source script.sh; + say_hello pytest-workflow + " + stdout: + contains: + - "Hello, pytest-workflow!" + must_not_contain: + - "Hello, world!" From 2b168955a8beace5a7638a1bcc9cef0ae475753b Mon Sep 17 00:00:00 2001 From: Redmar van den Berg Date: Wed, 2 Nov 2022 10:01:22 +0100 Subject: [PATCH 39/94] Update examples.rst --- docs/examples.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples.rst b/docs/examples.rst index ce239250..63bfed0d 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -185,7 +185,7 @@ The following is an example of a Bash file that can run directly as a script, or Save the bash file as ``script.sh``, and test it with the following pytest-workflow configuration: -.. code-block:: yml +.. code-block:: yaml - name: test bash script command: bash script.sh From 793bc878f82135fafb16a6b73abb0857571bb910 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 2 Nov 2022 17:10:27 +0100 Subject: [PATCH 40/94] Mention that there are examples in the docs And add nextflow to the list. --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index a70ead6c..39916b68 100644 --- a/README.rst +++ b/README.rst @@ -34,11 +34,11 @@ pytest-workflow pytest-workflow is a workflow-system agnostic testing framework that aims to make pipeline/workflow testing easy by using YAML files for the test -configuration. Whether you write your pipelines in WDL, snakemake, bash or -any other workflow framework, pytest-workflow makes testing easy. +configuration. Whether you write your pipelines in WDL, snakemake, nextflow, +bash or any other workflow framework, pytest-workflow makes testing easy. pytest-workflow is build on top of the pytest test framework. -For our complete documentation checkout our +For our complete documentation and examples checkout our `readthedocs page `_. From 0930ac02fe140f197a6e2fe82b03a680bb3b5ef2 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 8 Nov 2022 08:26:33 +0100 Subject: [PATCH 41/94] Enable tests on python 3.11 --- .github/workflows/ci.yml | 1 + HISTORY.rst | 1 + setup.py | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11d7c966..46a2bbb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,7 @@ jobs: - "3.8" - "3.9" - "3.10" + - "3.11" needs: lint steps: - uses: actions/checkout@v2.3.4 diff --git a/HISTORY.rst b/HISTORY.rst index a5efc2fb..c021c655 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ Changelog version 1.7.0-dev --------------------------- ++ Test and support for Python 3.11. + Add ``--stderr-bytes`` or ``--sb`` option to change the maximum number of bytes to display for the stderr and stdout on command failure. diff --git a/setup.py b/setup.py index 8eaa64df..a567ca91 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: " "GNU Affero General Public License v3 or later (AGPLv3+)", From 8180d78b296469e00490b8e79bbcf21958cd0447 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 9 Nov 2022 14:11:23 +0100 Subject: [PATCH 42/94] Remove underscore imports These were used for typing purposes only. Which made development easier. Since _pytest modules are not part of the public API, changes may occur. Since 7.0.0 all necessary objects are exposed trough the API so we can import them directly. --- src/pytest_workflow/plugin.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/pytest_workflow/plugin.py b/src/pytest_workflow/plugin.py index 8ff16693..3e811e6f 100644 --- a/src/pytest_workflow/plugin.py +++ b/src/pytest_workflow/plugin.py @@ -23,11 +23,6 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple -from _pytest.config import Config as PytestConfig -from _pytest.config.argparsing import Parser as PytestParser -from _pytest.mark import Mark -from _pytest.python import FunctionDefinition, Metafunc - import pytest import yaml @@ -39,7 +34,7 @@ from .workflow import Workflow, WorkflowQueue -def pytest_addoption(parser: PytestParser): +def pytest_addoption(parser: pytest.Parser): parser.addoption( "--kwd", "--keep-workflow-wd", action="store_true", @@ -130,7 +125,7 @@ def pytest_collect_file(file_path, path, parent): return None -def pytest_configure(config: PytestConfig): +def pytest_configure(config: pytest.Config): """This runs before tests start and adds values to the config.""" # Add marker to the config to prevent issues caused by: @@ -201,7 +196,8 @@ def pytest_collection(): print() -def get_workflow_names_from_workflow_marker(marker: Mark) -> Tuple[Any, ...]: +def get_workflow_names_from_workflow_marker(marker: pytest.Mark + ) -> Tuple[Any, ...]: if 'name' in marker.kwargs: raise DeprecationWarning( "Using pytest.mark.workflow(name='workflow name') is " @@ -210,7 +206,7 @@ def get_workflow_names_from_workflow_marker(marker: Mark) -> Tuple[Any, ...]: return marker.args -def pytest_generate_tests(metafunc: Metafunc): +def pytest_generate_tests(metafunc: pytest.Metafunc): """ This runs at the end of the collection phase. We use this hook to generate the workflow_dir fixtures for custom test functions. @@ -221,8 +217,11 @@ def pytest_generate_tests(metafunc: Metafunc): if "workflow_dir" not in metafunc.fixturenames: return - definition: FunctionDefinition = metafunc.definition - marker: Optional[Mark] = definition.get_closest_marker(name="workflow") + # Technically definition is of type FunctionDefinition, but that is a + # subclass of Function and FunctionDefinition cannot be accessed through + # the API. + definition: pytest.Function = metafunc.definition + marker: Optional[pytest.Mark] = definition.get_closest_marker("workflow") if marker is None: raise ValueError("workflow_dir can only be requested in tests marked" " with the workflow mark.") @@ -239,12 +238,12 @@ def pytest_generate_tests(metafunc: Metafunc): ids=workflow_names) -def pytest_collection_modifyitems(config: PytestConfig, +def pytest_collection_modifyitems(config: pytest.Config, items: List[pytest.Function]): """Here we skip all tests related to workflows that are not executed""" for item in items: - marker = item.get_closest_marker(name="workflow") + marker: Optional[pytest.Mark] = item.get_closest_marker("workflow") if marker is None: continue From c92ccc4e0a9f9b14c245aa4c118b18980dab4bf7 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 11 Nov 2022 09:42:21 +0100 Subject: [PATCH 43/94] Update requirements.txt to be correct --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index be8fac62..2c5d3bff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ pyyaml -pytest>=5.4.0 +pytest>=7.0.0 jsonschema \ No newline at end of file From e720d5f9f9ea66420300c1b236239d4f59b0fbbb Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 11 Nov 2022 11:18:49 +0100 Subject: [PATCH 44/94] add skip for tests searching non existing file. --- src/pytest_workflow/content_tests.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/src/pytest_workflow/content_tests.py b/src/pytest_workflow/content_tests.py index 589e6045..9b761129 100644 --- a/src/pytest_workflow/content_tests.py +++ b/src/pytest_workflow/content_tests.py @@ -243,8 +243,9 @@ def runtest(self): this makes content checking much faster on big files (NGS > 1 GB files) were we are looking for multiple words (variants / sequences). """ # Wait for thread to complete. + if self.parent.file_not_found: + pytest.skip(f"'{self.content_name}' not found so cant be searched") self.parent.thread.join() - assert not self.parent.file_not_found if self.regex: assert ((self.string in self.parent.found_patterns) == self.should_contain) @@ -253,17 +254,9 @@ def runtest(self): self.should_contain) def repr_failure(self, excinfo, style=None): - if self.parent.file_not_found: - containing = ("containing" if self.should_contain else - "not containing") - return ( - f"'{self.content_name}' does not exist and cannot be searched " - f"for {containing} '{self.string}'." - ) - else: - found = "not found" if self.should_contain else "found" - should = "should" if self.should_contain else "should not" - return ( - f"'{self.string}' was {found} in {self.content_name} " - f"while it {should} be there." - ) + found = "not found" if self.should_contain else "found" + should = "should" if self.should_contain else "should not" + return ( + f"'{self.string}' was {found} in {self.content_name} " + f"while it {should} be there." + ) From 58205700e308fb317c1e5fcd9a99be10a6721b7a Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 11 Nov 2022 11:35:47 +0100 Subject: [PATCH 45/94] Remove bandit as linting tool It only caused lots of nosec comments. --- src/pytest_workflow/util.py | 6 +++--- src/pytest_workflow/workflow.py | 4 ++-- tests/test_success_messages.py | 4 ++-- tests/test_utils.py | 11 +++++------ tests/test_workflow.py | 2 +- tox.ini | 3 --- 6 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/pytest_workflow/util.py b/src/pytest_workflow/util.py index 941fde88..e208dc4d 100644 --- a/src/pytest_workflow/util.py +++ b/src/pytest_workflow/util.py @@ -3,7 +3,7 @@ import os import re import shutil -import subprocess # nosec +import subprocess import sys import warnings from pathlib import Path @@ -50,7 +50,7 @@ def is_in_dir(child: Path, parent: Path, strict: bool = False) -> bool: def _run_command(*args): """Run an external command and return the output""" - result = subprocess.run(args, # nosec + result = subprocess.run(args, stdout=subprocess.PIPE, # Encoding to output as a string. encoding=sys.getdefaultencoding(), @@ -188,7 +188,7 @@ def file_md5sum(filepath: Path, block_size=64 * 1024) -> str: :param block_size: Block size in bytes :return: a md5sum as hexadecimal string. """ - hasher = hashlib.md5() # nosec: only used for file integrity + hasher = hashlib.md5() with filepath.open('rb') as file_handler: # Read the file in bytes for block in iter(lambda: file_handler.read(block_size), b''): hasher.update(block) diff --git a/src/pytest_workflow/workflow.py b/src/pytest_workflow/workflow.py index fbc9a63d..0101941c 100644 --- a/src/pytest_workflow/workflow.py +++ b/src/pytest_workflow/workflow.py @@ -22,7 +22,7 @@ """ import queue import shlex -import subprocess # nosec: security implications have been considered +import subprocess import tempfile import threading import time @@ -81,7 +81,7 @@ def start(self): stdout_h = self.stdout_file.open('wb') stderr_h = self.stderr_file.open('wb') sub_process_args = shlex.split(self.command) - self._popen = subprocess.Popen( # nosec: Shell is not enabled. # noqa + self._popen = subprocess.Popen( sub_process_args, stdout=stdout_h, stderr=stderr_h, cwd=str(self.cwd)) except Exception as error: diff --git a/tests/test_success_messages.py b/tests/test_success_messages.py index 209e5300..8157c8dd 100644 --- a/tests/test_success_messages.py +++ b/tests/test_success_messages.py @@ -16,7 +16,7 @@ """Tests the success messages""" import shutil -import subprocess # nosec +import subprocess import textwrap from pathlib import Path @@ -119,7 +119,7 @@ def succeeding_tests_output(tmp_path_factory: pytest.TempPathFactory): test_file = Path(tempdir, "test_succeeding.yml") with test_file.open("w") as file_handler: file_handler.write(SUCCEEDING_TESTS_YAML) - process_out = subprocess.run(args=["pytest", "-v"], # nosec + process_out = subprocess.run(args=["pytest", "-v"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=tempdir) diff --git a/tests/test_utils.py b/tests/test_utils.py index 95a02a73..5bbed249 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -16,7 +16,7 @@ import hashlib import os import shutil -import subprocess # nosec +import subprocess import tempfile from pathlib import Path @@ -100,9 +100,9 @@ def git_dir(): (git_dir / "test").mkdir() test_file = git_dir / "test" / "test.txt" test_file.touch() - subprocess.run(["git", "-C", str(git_dir), "init"]) # nosec - subprocess.run(["git", "-C", str(git_dir), "add", str(test_file)]) # nosec - subprocess.run(["git", "-C", str(git_dir), "commit", "-m", # nosec + subprocess.run(["git", "-C", str(git_dir), "init"]) + subprocess.run(["git", "-C", str(git_dir), "add", str(test_file)]) + subprocess.run(["git", "-C", str(git_dir), "commit", "-m", "initial commit"]) yield git_dir shutil.rmtree(git_dir) @@ -155,7 +155,6 @@ def test_git_root(git_dir): @pytest.mark.parametrize("hash_file", HASH_FILE_DIR.iterdir()) def test_file_md5sum(hash_file: Path): - # No sec added because this hash is only used for checking file integrity - whole_file_md5 = hashlib.md5(hash_file.read_bytes()).hexdigest() # nosec + whole_file_md5 = hashlib.md5(hash_file.read_bytes()).hexdigest() per_line_md5 = file_md5sum(hash_file) assert whole_file_md5 == per_line_md5 diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 83b637c5..7bf8d6f7 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -15,7 +15,7 @@ # along with pytest-workflow. If not, see Date: Fri, 11 Nov 2022 10:08:14 +0100 Subject: [PATCH 46/94] Add function for checking clned submodules --- src/pytest_workflow/util.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/pytest_workflow/util.py b/src/pytest_workflow/util.py index e208dc4d..2bd79e27 100644 --- a/src/pytest_workflow/util.py +++ b/src/pytest_workflow/util.py @@ -48,7 +48,7 @@ def is_in_dir(child: Path, parent: Path, strict: bool = False) -> bool: return False -def _run_command(*args): +def _run_command(*args) -> str: """Run an external command and return the output""" result = subprocess.run(args, stdout=subprocess.PIPE, @@ -64,6 +64,18 @@ def git_root(path: Filepath) -> str: return output.strip() # Remove trailing newline +def git_check_submodules_cloned(path: Filepath): + output = _run_command("git", "-C", os.fspath(path), "submodule", "status", + "--recursive") + for line in output.splitlines(): + commit, path, described_commit = line.strip().split() + if commit.startswith("-"): + raise RuntimeError( + f"Git submodule {path} was not cloned. Pytest-workflow cannot " + f"copy paths from non-existing submodules. Please clone all " + f"submodules using 'git submodule update --init --recursive'.") + + def git_ls_files(path: Filepath) -> List[str]: output = _run_command("git", "-C", os.fspath(path), "ls-files", # Make sure submodules are included. @@ -97,6 +109,7 @@ def _recurse_git_repository_tree(src: Filepath, dest: Filepath # A set of dirs we have already yielded. '' is the output of # os.path.dirname when the path is in the current directory. yielded_dirs: Set[str] = {''} + git_check_submodules_cloned(src) for path in git_ls_files(src): # git ls-files does not list directories. Yield parent first to prevent # creating files in non-existing directories. Also check if it is From 8e1114c2d392c9551bf5c1a955d2839f5ace0707 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 11 Nov 2022 10:41:48 +0100 Subject: [PATCH 47/94] Add test for checking git submodules --- src/pytest_workflow/util.py | 9 +++++---- tests/test_utils.py | 24 +++++++++++++++++++++++- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/pytest_workflow/util.py b/src/pytest_workflow/util.py index 2bd79e27..94e2ffbf 100644 --- a/src/pytest_workflow/util.py +++ b/src/pytest_workflow/util.py @@ -68,12 +68,13 @@ def git_check_submodules_cloned(path: Filepath): output = _run_command("git", "-C", os.fspath(path), "submodule", "status", "--recursive") for line in output.splitlines(): - commit, path, described_commit = line.strip().split() + commit, path = line.strip().split(maxsplit=1) if commit.startswith("-"): raise RuntimeError( - f"Git submodule {path} was not cloned. Pytest-workflow cannot " - f"copy paths from non-existing submodules. Please clone all " - f"submodules using 'git submodule update --init --recursive'.") + f"Git submodule '{path}' was not cloned. Pytest-workflow " + f"cannot copy paths from non-existing submodules. Please " + f"clone all submodules using 'git submodule update --init " + f"--recursive'.") def git_ls_files(path: Filepath) -> List[str]: diff --git a/tests/test_utils.py b/tests/test_utils.py index 5bbed249..4ac2926a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -22,7 +22,8 @@ import pytest -from pytest_workflow.util import duplicate_tree, file_md5sum, git_root, \ +from pytest_workflow.util import duplicate_tree, file_md5sum, \ + git_check_submodules_cloned, git_root, \ is_in_dir, link_tree, replace_whitespace WHITESPACE_TESTS = [ @@ -158,3 +159,24 @@ def test_file_md5sum(hash_file: Path): whole_file_md5 = hashlib.md5(hash_file.read_bytes()).hexdigest() per_line_md5 = file_md5sum(hash_file) assert whole_file_md5 == per_line_md5 + + +def test_git_submodule_check(tmp_path): + # Clone using biowdl/tasks as it is reasonably small and contains the + # scripts submodule. + subprocess.run( + # No recursive clone + ["git", "clone", "--depth=1", + "https://github.com/biowdl/tasks.git"], + cwd=tmp_path + ) + git_repo = tmp_path / "tasks" + with pytest.raises(RuntimeError) as error: + git_check_submodules_cloned(git_repo) + # Error message should allow user to resolve the issue. + error.match("'git submodule update --init --recursive'") + subprocess.run( + ["git", "-C", str(git_repo), "submodule", "update", "--init", + "--recursive"]) + # Check error does not occur when issue resolved. + git_check_submodules_cloned(git_repo) From 3a061a7c502b3ee67eae4f8fcfdbcf18365368e0 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 11 Nov 2022 11:20:38 +0100 Subject: [PATCH 48/94] Make sure test for submodule cloning does not require online access --- tests/test_utils.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 4ac2926a..1767d26a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -161,22 +161,36 @@ def test_file_md5sum(hash_file: Path): assert whole_file_md5 == per_line_md5 +def create_git_repo(path): + dir = Path(path) + os.mkdir(dir) + file = dir / "README.md" + file.write_text("# My new project\n\nHello this project is awesome!\n") + subprocess.run(["git", "init"], cwd=dir) + subprocess.run(["git", "add", "README.md"], cwd=dir) + subprocess.run(["git", "commit", "-m", "initial commit"], cwd=dir) + + def test_git_submodule_check(tmp_path): - # Clone using biowdl/tasks as it is reasonably small and contains the - # scripts submodule. + bird_repo = tmp_path / "bird" + nest_repo = tmp_path / "nest" + create_git_repo(bird_repo) + create_git_repo(nest_repo) + subprocess.run(["git", "submodule", "add", bird_repo.absolute()], + cwd=nest_repo.absolute()) + subprocess.run(["git", "commit", "-m", "add bird repo as a submodule"], + cwd=nest_repo.absolute()) + cloned_repo = tmp_path / "cloned" subprocess.run( # No recursive clone - ["git", "clone", "--depth=1", - "https://github.com/biowdl/tasks.git"], + ["git", "clone", nest_repo.absolute(), cloned_repo.absolute()], cwd=tmp_path ) - git_repo = tmp_path / "tasks" with pytest.raises(RuntimeError) as error: - git_check_submodules_cloned(git_repo) + git_check_submodules_cloned(cloned_repo) # Error message should allow user to resolve the issue. error.match("'git submodule update --init --recursive'") - subprocess.run( - ["git", "-C", str(git_repo), "submodule", "update", "--init", - "--recursive"]) + subprocess.run(["git", "submodule", "update", "--init", "--recursive"], + cwd=cloned_repo.absolute()) # Check error does not occur when issue resolved. - git_check_submodules_cloned(git_repo) + git_check_submodules_cloned(cloned_repo) From cf40b24e97b33b8a60b9628d292895411ac2b4c4 Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 11 Nov 2022 11:38:00 +0100 Subject: [PATCH 49/94] add test_skip_tests.py to test new functionality. --- tests/test_skip_tests.py | 45 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/test_skip_tests.py diff --git a/tests/test_skip_tests.py b/tests/test_skip_tests.py new file mode 100644 index 00000000..846850a5 --- /dev/null +++ b/tests/test_skip_tests.py @@ -0,0 +1,45 @@ +# Copyright (C) 2018 Leiden University Medical Center +# This file is part of pytest-workflow +# +# pytest-workflow is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# pytest-workflow is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with pytest-workflow. If not, see test3.txt" + files: + - path: test3.txt + contains: + - "kaas" + must_not_contain: + - "testing" +""") + +def test_skips(pytester): + pytester.makefile(".yml", test=SKIP_TESTS) + result = pytester.runpytest("-v").stdout.str() + assert "4 failed, 3 passed, 3 skipped" in result \ No newline at end of file From 7eede15b7b16bb79e7bbfffa5891524fe558b478 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 11 Nov 2022 11:38:06 +0100 Subject: [PATCH 50/94] Add submodule check to changelog --- HISTORY.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index c021c655..0852d39f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,9 @@ Changelog version 1.7.0-dev --------------------------- ++ When the ``--git-aware`` flag is used a submodule check is performed in other + to assert that all submodules are properly checked out. This prevents + unclear copying errors. + Test and support for Python 3.11. + Add ``--stderr-bytes`` or ``--sb`` option to change the maximum number of bytes to display for the stderr and stdout on From 9a628cb90ac807c5812a4e511853340da69342df Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 11 Nov 2022 11:40:00 +0100 Subject: [PATCH 51/94] Removed tests that no longer work --- tests/test_fail_messages.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/tests/test_fail_messages.py b/tests/test_fail_messages.py index 878b4caf..cf03b375 100644 --- a/tests/test_fail_messages.py +++ b/tests/test_fail_messages.py @@ -111,25 +111,6 @@ "'grep --help' was found in 'fail_test': stderr while " "it should not be there"), ("""\ - - name: file_not_exist - command: echo moo - files: - - path: moo.txt - contains: - - "moo" - """, - "moo.txt' does not exist and cannot be searched for containing 'moo'."), - ("""\ - - name: file_not_exist - command: echo moo - files: - - path: moo.txt - must_not_contain: - - "miaow" - """, - "moo.txt' does not exist and cannot be searched for " - "not containing 'miaow'."), - ("""\ - name: simple echo command: "echo Hello, world" stdout: From b89fdc409b2b7faa3abbd6f07c6f7546b8495561 Mon Sep 17 00:00:00 2001 From: LucasvanBostelen <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 11 Nov 2022 11:45:45 +0100 Subject: [PATCH 52/94] Update HISTORY.rst --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index c021c655..ca730cf3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ Changelog version 1.7.0-dev --------------------------- ++ Add skip for tests searching non existing file + Test and support for Python 3.11. + Add ``--stderr-bytes`` or ``--sb`` option to change the maximum number of bytes to display for the stderr and stdout on From a25b409630220f037e38e2debb91fa6b89961257 Mon Sep 17 00:00:00 2001 From: LucasvanBostelen <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 11 Nov 2022 11:52:29 +0100 Subject: [PATCH 53/94] fixed lint error --- tests/test_skip_tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_skip_tests.py b/tests/test_skip_tests.py index 846850a5..9387ccf0 100644 --- a/tests/test_skip_tests.py +++ b/tests/test_skip_tests.py @@ -36,10 +36,11 @@ contains: - "kaas" must_not_contain: - - "testing" + - "testing" """) + def test_skips(pytester): pytester.makefile(".yml", test=SKIP_TESTS) result = pytester.runpytest("-v").stdout.str() - assert "4 failed, 3 passed, 3 skipped" in result \ No newline at end of file + assert "4 failed, 3 passed, 3 skipped" in result From 756dde8560eeaac2b2ffa3f60df9fa9d0706edca Mon Sep 17 00:00:00 2001 From: LucasvanBostelen <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 11 Nov 2022 11:53:46 +0100 Subject: [PATCH 54/94] fixed lint error --- tests/test_skip_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_skip_tests.py b/tests/test_skip_tests.py index 9387ccf0..a469a172 100644 --- a/tests/test_skip_tests.py +++ b/tests/test_skip_tests.py @@ -36,7 +36,7 @@ contains: - "kaas" must_not_contain: - - "testing" + - "testing" """) From 78dc124fb818bcaacb9981cbb7743081fde179b2 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 11 Nov 2022 12:15:13 +0100 Subject: [PATCH 55/94] Attempt to work around CI limitations --- tests/test_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 1767d26a..1ff157d8 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -176,14 +176,17 @@ def test_git_submodule_check(tmp_path): nest_repo = tmp_path / "nest" create_git_repo(bird_repo) create_git_repo(nest_repo) - subprocess.run(["git", "submodule", "add", bird_repo.absolute()], + # https://bugs.launchpad.net/ubuntu/+source/git/+bug/1993586 + subprocess.run(["git", "-c", "protocol.file.allow=always", + "submodule", "add", bird_repo.absolute()], cwd=nest_repo.absolute()) subprocess.run(["git", "commit", "-m", "add bird repo as a submodule"], cwd=nest_repo.absolute()) cloned_repo = tmp_path / "cloned" subprocess.run( # No recursive clone - ["git", "clone", nest_repo.absolute(), cloned_repo.absolute()], + ["git", "-c", "protocol.file.allow=always", + "clone", nest_repo.absolute(), cloned_repo.absolute()], cwd=tmp_path ) with pytest.raises(RuntimeError) as error: From 078ed834f2b90d3f155826a0c099209d8d92357e Mon Sep 17 00:00:00 2001 From: LucasvanBostelen <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 11 Nov 2022 12:17:55 +0100 Subject: [PATCH 56/94] Changed stdout to parseoutcomes() to make assert more reliable --- tests/test_skip_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_skip_tests.py b/tests/test_skip_tests.py index a469a172..8e244d23 100644 --- a/tests/test_skip_tests.py +++ b/tests/test_skip_tests.py @@ -42,5 +42,5 @@ def test_skips(pytester): pytester.makefile(".yml", test=SKIP_TESTS) - result = pytester.runpytest("-v").stdout.str() - assert "4 failed, 3 passed, 3 skipped" in result + result = pytester.runpytest("-v").parseoutcomes() + assert {"failed": 4, "passed": 3, "skipped": 3} == result From 62ba94e0691fd21b245b74753328ba7bf9c91ee8 Mon Sep 17 00:00:00 2001 From: LucasvanBostelen <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 11 Nov 2022 12:20:07 +0100 Subject: [PATCH 57/94] Fixed typo --- src/pytest_workflow/content_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pytest_workflow/content_tests.py b/src/pytest_workflow/content_tests.py index 9b761129..5dcb4844 100644 --- a/src/pytest_workflow/content_tests.py +++ b/src/pytest_workflow/content_tests.py @@ -244,7 +244,7 @@ def runtest(self): were we are looking for multiple words (variants / sequences). """ # Wait for thread to complete. if self.parent.file_not_found: - pytest.skip(f"'{self.content_name}' not found so cant be searched") + pytest.skip(f"'{self.content_name}' not found so cannot be searched") self.parent.thread.join() if self.regex: assert ((self.string in self.parent.found_patterns) == From 4c0adce3f7ab400e0d03461f0f115041bac0691b Mon Sep 17 00:00:00 2001 From: LucasvanBostelen <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 11 Nov 2022 12:45:12 +0100 Subject: [PATCH 58/94] Update HISTORY.rst --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index ca730cf3..03210de7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,7 +9,7 @@ Changelog version 1.7.0-dev --------------------------- -+ Add skip for tests searching non existing file ++ Skip tests searching in non existing file + Test and support for Python 3.11. + Add ``--stderr-bytes`` or ``--sb`` option to change the maximum number of bytes to display for the stderr and stdout on From 2aede21f71f4ba12cc2684a56aa420ec0c813454 Mon Sep 17 00:00:00 2001 From: LucasvanBostelen <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 11 Nov 2022 12:54:57 +0100 Subject: [PATCH 59/94] fixed lint error --- src/pytest_workflow/content_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pytest_workflow/content_tests.py b/src/pytest_workflow/content_tests.py index 5dcb4844..6bebfc21 100644 --- a/src/pytest_workflow/content_tests.py +++ b/src/pytest_workflow/content_tests.py @@ -244,7 +244,8 @@ def runtest(self): were we are looking for multiple words (variants / sequences). """ # Wait for thread to complete. if self.parent.file_not_found: - pytest.skip(f"'{self.content_name}' not found so cannot be searched") + pytest.skip(f"'{self.content_name}' was not found so cannot be " + f"searched") self.parent.thread.join() if self.regex: assert ((self.string in self.parent.found_patterns) == From f5e27573c88bff48db802fdca7de6e1869818269 Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 11 Nov 2022 14:38:09 +0100 Subject: [PATCH 60/94] do check after thread.join to prevent error --- src/pytest_workflow/content_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pytest_workflow/content_tests.py b/src/pytest_workflow/content_tests.py index 6bebfc21..06c66a87 100644 --- a/src/pytest_workflow/content_tests.py +++ b/src/pytest_workflow/content_tests.py @@ -243,10 +243,10 @@ def runtest(self): this makes content checking much faster on big files (NGS > 1 GB files) were we are looking for multiple words (variants / sequences). """ # Wait for thread to complete. + self.parent.thread.join() if self.parent.file_not_found: pytest.skip(f"'{self.content_name}' was not found so cannot be " f"searched") - self.parent.thread.join() if self.regex: assert ((self.string in self.parent.found_patterns) == self.should_contain) From 08d7122c58abc123ae787735b8572da5c245a1d7 Mon Sep 17 00:00:00 2001 From: Redmar van den Berg Date: Mon, 14 Nov 2022 08:49:00 +0100 Subject: [PATCH 61/94] Fix typo --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0852d39f..fe1ba70b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,7 +9,7 @@ Changelog version 1.7.0-dev --------------------------- -+ When the ``--git-aware`` flag is used a submodule check is performed in other ++ When the ``--git-aware`` flag is used a submodule check is performed in order to assert that all submodules are properly checked out. This prevents unclear copying errors. + Test and support for Python 3.11. From b8c355f1c806c42fb639e6d699452fa6ece53410 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 14 Nov 2022 09:34:20 +0100 Subject: [PATCH 62/94] Make sure commit author is set --- tests/test_utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 1ff157d8..b57625a7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -168,7 +168,10 @@ def create_git_repo(path): file.write_text("# My new project\n\nHello this project is awesome!\n") subprocess.run(["git", "init"], cwd=dir) subprocess.run(["git", "add", "README.md"], cwd=dir) - subprocess.run(["git", "commit", "-m", "initial commit"], cwd=dir) + subprocess.run(["git", "branch", "-M", "main"]) + subprocess.run(["git", "commit", + "--author='A U Thor '", + "-m", "initial commit"], cwd=dir) def test_git_submodule_check(tmp_path): @@ -180,7 +183,9 @@ def test_git_submodule_check(tmp_path): subprocess.run(["git", "-c", "protocol.file.allow=always", "submodule", "add", bird_repo.absolute()], cwd=nest_repo.absolute()) - subprocess.run(["git", "commit", "-m", "add bird repo as a submodule"], + subprocess.run(["git", "commit", + "--author='A U Thor '", + "-m", "add bird repo as a submodule"], cwd=nest_repo.absolute()) cloned_repo = tmp_path / "cloned" subprocess.run( From 6d35bc001cd55bea4301aa6d5537963b9af9828e Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 14 Nov 2022 09:36:01 +0100 Subject: [PATCH 63/94] Run linting parallel with tests --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46a2bbb0..7c613953 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,6 @@ jobs: - "3.9" - "3.10" - "3.11" - needs: lint steps: - uses: actions/checkout@v2.3.4 - name: Set up Python ${{ matrix.python-version }} From 2dde7939080f85a2de47a2e2d65d2759fd2044ef Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 14 Nov 2022 09:37:08 +0100 Subject: [PATCH 64/94] Use lowercase m --- tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index b57625a7..daf0e36c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -168,7 +168,7 @@ def create_git_repo(path): file.write_text("# My new project\n\nHello this project is awesome!\n") subprocess.run(["git", "init"], cwd=dir) subprocess.run(["git", "add", "README.md"], cwd=dir) - subprocess.run(["git", "branch", "-M", "main"]) + subprocess.run(["git", "branch", "-m", "main"]) subprocess.run(["git", "commit", "--author='A U Thor '", "-m", "initial commit"], cwd=dir) From b8e722a564638a6d463b67722160278cfdc6d0ba Mon Sep 17 00:00:00 2001 From: A U Thor Date: Mon, 14 Nov 2022 09:55:20 +0100 Subject: [PATCH 65/94] Make sure author is set --- tests/test_utils.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index daf0e36c..81239626 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -166,12 +166,12 @@ def create_git_repo(path): os.mkdir(dir) file = dir / "README.md" file.write_text("# My new project\n\nHello this project is awesome!\n") - subprocess.run(["git", "init"], cwd=dir) + subprocess.run(["git", "init", "-b", "main"], cwd=dir) + subprocess.run(["git", "config", "user.name", "A U Thor"], cwd=dir) + subprocess.run(["git", "config", "user.email", "author@example.com"], + cwd=dir) subprocess.run(["git", "add", "README.md"], cwd=dir) - subprocess.run(["git", "branch", "-m", "main"]) - subprocess.run(["git", "commit", - "--author='A U Thor '", - "-m", "initial commit"], cwd=dir) + subprocess.run(["git", "commit", "-m", "initial commit"], cwd=dir) def test_git_submodule_check(tmp_path): @@ -183,9 +183,7 @@ def test_git_submodule_check(tmp_path): subprocess.run(["git", "-c", "protocol.file.allow=always", "submodule", "add", bird_repo.absolute()], cwd=nest_repo.absolute()) - subprocess.run(["git", "commit", - "--author='A U Thor '", - "-m", "add bird repo as a submodule"], + subprocess.run(["git", "commit", "-m", "add bird repo as a submodule"], cwd=nest_repo.absolute()) cloned_repo = tmp_path / "cloned" subprocess.run( From 757346d949c788d301e98515ff711862de3f480c Mon Sep 17 00:00:00 2001 From: A U Thor Date: Mon, 14 Nov 2022 10:01:17 +0100 Subject: [PATCH 66/94] Allow file protocol in submodule update --- tests/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 81239626..098f1e67 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -196,7 +196,8 @@ def test_git_submodule_check(tmp_path): git_check_submodules_cloned(cloned_repo) # Error message should allow user to resolve the issue. error.match("'git submodule update --init --recursive'") - subprocess.run(["git", "submodule", "update", "--init", "--recursive"], + subprocess.run(["git", "-c", "protocol.file.allow=always", "submodule", + "update", "--init", "--recursive"], cwd=cloned_repo.absolute()) # Check error does not occur when issue resolved. git_check_submodules_cloned(cloned_repo) From 7eb8d8ab3b7b545007ca6ca57ecf3cc693cfddb8 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 14 Nov 2022 11:08:53 +0100 Subject: [PATCH 67/94] Advertise python 3.11 support, remove Python 2 notice Python 2 hasn't been supported for nearly 2 years. It does not need to be explcitly mentioned now. --- README.rst | 2 +- docs/installation.rst | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 39916b68..739803e2 100644 --- a/README.rst +++ b/README.rst @@ -45,7 +45,7 @@ For our complete documentation and examples checkout our Installation ============ Pytest-workflow requires Python 3.6 or higher. It is tested on Python 3.6, 3.7, -3.8, 3.9 and 3.10. Python 2 is not supported. +3.8, 3.9, 3.10 and 3.11. - Make sure your virtual environment is activated. - Install using pip ``pip install pytest-workflow`` diff --git a/docs/installation.rst b/docs/installation.rst index 1a38d3bf..e7a18766 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -2,8 +2,7 @@ Installation ============ -Pytest-workflow is tested on python 3.6, 3.7, 3.8, 3.9 and 3.10. Python 2 is not -supported. +Pytest-workflow is tested on python 3.6, 3.7, 3.8, 3.9, 3.10 and 3.11. In a virtual environment ------------------------ From 3d4ad5b0d9bf04581e4263b15349e9988f01e0e9 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 14 Nov 2022 12:35:29 +0100 Subject: [PATCH 68/94] code-block must specify a code name otherwise readthedocs won't work --- docs/examples.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/examples.rst b/docs/examples.rst index 63bfed0d..307874ea 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -81,7 +81,7 @@ the running of miniwdl from the environment. Miniwdl will localize all the output files to an ``output_links`` directory inside the test output directory. If you have a workflow with the output: -.. code-block:: +.. code-block:: wdl output { File moo_file = moo_task.out @@ -120,7 +120,7 @@ it is possible to define the output directory when running the code. An example code defining a ``publishDir`` is listed below. -.. code-block:: +.. code-block:: nextflow process Hello { publishDir = [ @@ -141,7 +141,7 @@ An example code defining a ``publishDir`` is listed below. To run the code listed above the following command can be used in which ``examplecode.nf`` is the code listed above: -.. code-block:: +.. code-block:: bash nextflow run examplecode.nf --outdir test-output From 8b82416fe5214106d90c659adf5ce91f9dcaf6f4 Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 18 Nov 2022 13:06:16 +0100 Subject: [PATCH 69/94] add desired exit code to workflow object and skip tests if the exit code isn't equal to the desired exit code. --- src/pytest_workflow/content_tests.py | 3 +++ src/pytest_workflow/file_tests.py | 6 ++++++ src/pytest_workflow/plugin.py | 12 +++++++----- src/pytest_workflow/workflow.py | 4 ++++ 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/pytest_workflow/content_tests.py b/src/pytest_workflow/content_tests.py index 06c66a87..b27f2d1a 100644 --- a/src/pytest_workflow/content_tests.py +++ b/src/pytest_workflow/content_tests.py @@ -244,6 +244,9 @@ def runtest(self): were we are looking for multiple words (variants / sequences). """ # Wait for thread to complete. self.parent.thread.join() + if not self.parent.workflow.matching_exitcode(): + pytest.skip(f"'{self.parent.workflow.name}' did not exit with" + f"desired exit code.") if self.parent.file_not_found: pytest.skip(f"'{self.content_name}' was not found so cannot be " f"searched") diff --git a/src/pytest_workflow/file_tests.py b/src/pytest_workflow/file_tests.py index 8eca7151..17642fc8 100644 --- a/src/pytest_workflow/file_tests.py +++ b/src/pytest_workflow/file_tests.py @@ -102,6 +102,9 @@ def runtest(self): # Wait for the workflow process to finish before checking if the file # exists. self.workflow.wait() + if not self.workflow.matching_exitcode(): + pytest.skip(f"'{self.parent.workflow.name}' did not exit with" + f"desired exit code.") assert self.file.exists() == self.should_exist def repr_failure(self, excinfo, style=None): @@ -134,6 +137,9 @@ def __init__(self, parent: pytest.Collector, filepath: Path, def runtest(self): # Wait for the workflow to finish before we check the md5sum of a file. self.workflow.wait() + if not self.workflow.matching_exitcode(): + pytest.skip(f"'{self.parent.workflow.name}' did not exit with" + f"desired exit code.") self.observed_md5sum = file_md5sum(self.filepath) assert self.observed_md5sum == self.expected_md5sum diff --git a/src/pytest_workflow/plugin.py b/src/pytest_workflow/plugin.py index 8ff16693..496cd3d5 100644 --- a/src/pytest_workflow/plugin.py +++ b/src/pytest_workflow/plugin.py @@ -444,20 +444,22 @@ def collect(self): # collection phase. workflow = self.queue_workflow() + workflow.desired_exit_code = self.workflow_test.exit_code + # Below structure makes it easy to append tests tests = [] - tests += [ - FileTestCollector.from_parent( - parent=self, filetest=filetest, workflow=workflow) - for filetest in self.workflow_test.files] - tests += [ExitCodeTest.from_parent( parent=self, desired_exit_code=self.workflow_test.exit_code, workflow=workflow, stderr_bytes=self.config.getoption("stderr_bytes"))] + tests += [ + FileTestCollector.from_parent( + parent=self, filetest=filetest, workflow=workflow) + for filetest in self.workflow_test.files] + tests += [ContentTestCollector.from_parent( name="stdout", parent=self, filepath=workflow.stdout_file, diff --git a/src/pytest_workflow/workflow.py b/src/pytest_workflow/workflow.py index 0101941c..b85116e3 100644 --- a/src/pytest_workflow/workflow.py +++ b/src/pytest_workflow/workflow.py @@ -69,6 +69,7 @@ def __init__(self, self._started = False self.errors: List[Exception] = [] self.start_lock = threading.Lock() + self.desired_exit_code = 0 def start(self): """Runs the workflow in a subprocess in the background. @@ -137,6 +138,9 @@ def wait(self, timeout_secs: Optional[float] = None, # workflow pass + def matching_exitcode(self) -> bool: + return self.exit_code == self.desired_exit_code + @property def stdout(self) -> bytes: self.wait() From 5be526cf0d77d37f6e99e0b939a265e863e1f5bf Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 18 Nov 2022 13:12:34 +0100 Subject: [PATCH 70/94] removed redundant code from ExitCodeTest. --- src/pytest_workflow/plugin.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/pytest_workflow/plugin.py b/src/pytest_workflow/plugin.py index 496cd3d5..10ebf16b 100644 --- a/src/pytest_workflow/plugin.py +++ b/src/pytest_workflow/plugin.py @@ -451,7 +451,6 @@ def collect(self): tests += [ExitCodeTest.from_parent( parent=self, - desired_exit_code=self.workflow_test.exit_code, workflow=workflow, stderr_bytes=self.config.getoption("stderr_bytes"))] @@ -479,17 +478,15 @@ def collect(self): class ExitCodeTest(pytest.Item): def __init__(self, parent: pytest.Collector, - desired_exit_code: int, workflow: Workflow, stderr_bytes: int): - name = f"exit code should be {desired_exit_code}" + name = f"exit code should be {workflow.desired_exit_code}" super().__init__(name, parent=parent) self.stderr_bytes = stderr_bytes self.workflow = workflow - self.desired_exit_code = desired_exit_code def runtest(self): # workflow.exit_code waits for workflow to finish. - assert self.workflow.exit_code == self.desired_exit_code + assert self.workflow.matching_exitcode() def repr_failure(self, excinfo, style=None): standerr = self.workflow.stderr_file @@ -502,7 +499,7 @@ def repr_failure(self, excinfo, style=None): standout_file.seek(-self.stderr_bytes, os.SEEK_END) message = (f"'{self.workflow.name}' exited with exit code " + f"'{self.workflow.exit_code}' instead of " - f"'{self.desired_exit_code}'.\nstderr: " + f"'{self.workflow.desired_exit_code}'.\nstderr: " f"{standerr_file.read().strip().decode('utf-8')}" f"\nstdout: " f"{standout_file.read().strip().decode('utf-8')}") From 8e65cd5245c3f9db251d9a99623bf58bcc3522d2 Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 18 Nov 2022 13:19:58 +0100 Subject: [PATCH 71/94] add test to test_skip_tests.py --- tests/test_skip_tests.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/test_skip_tests.py b/tests/test_skip_tests.py index 8e244d23..a63387c7 100644 --- a/tests/test_skip_tests.py +++ b/tests/test_skip_tests.py @@ -29,7 +29,7 @@ must_not_contain: - "gorgonzola" -- name: wall3_test +- name: wall2_test command: bash -c "echo 'testing' > test3.txt" files: - path: test3.txt @@ -37,10 +37,20 @@ - "kaas" must_not_contain: - "testing" + +- name: wall3_test + command: bash -c "exit 12" + files: + - path: test4.txt + contains: + - "content" + must_not_contain: + - "no_content" + md5sum: e583af1f8b00b53cda87ae9ead880224 """) def test_skips(pytester): pytester.makefile(".yml", test=SKIP_TESTS) result = pytester.runpytest("-v").parseoutcomes() - assert {"failed": 4, "passed": 3, "skipped": 3} == result + assert {"failed": 5, "passed": 3, "skipped": 7} == result From 5a54ddca8e7c2f57aff071c7c921e67d83f47d50 Mon Sep 17 00:00:00 2001 From: LucasvanBostelen <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 18 Nov 2022 13:30:32 +0100 Subject: [PATCH 72/94] Update HISTORY.rst --- HISTORY.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index f51186a7..474e9bee 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,9 @@ version 1.7.0-dev + When the ``--git-aware`` flag is used a submodule check is performed in order to assert that all submodules are properly checked out. This prevents unclear copying errors. ++ Tests are now skipped if the workflow does not exit with the desired exit + code, except for the exit code tests, to reduce visual clutter when reporting + failing tests. + Tests for checking file content are now skipped when the file does not exist in order to reduce visual clutter when reporting failing tests. + Test and support for Python 3.11. From 1e6cb0b4e650f773425c34d9e4bc9dec37f67735 Mon Sep 17 00:00:00 2001 From: LucasvanBostelen <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 18 Nov 2022 13:36:52 +0100 Subject: [PATCH 73/94] removed obsolete tests --- tests/test_fail_messages.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/test_fail_messages.py b/tests/test_fail_messages.py index cf03b375..ea48da68 100644 --- a/tests/test_fail_messages.py +++ b/tests/test_fail_messages.py @@ -93,24 +93,6 @@ "moo' was found in 'echo does not contain moo': stdout while " "it should not be there"), ("""\ - - name: fail_test - command: grep - stderr: - contains: - - "No arguments?" - """, - "'No arguments?' was not found in 'fail_test': stderr " - "while it should be there"), - ("""\ - - name: fail_test - command: grep - stderr: - must_not_contain: - - "grep --help" - """, - "'grep --help' was found in 'fail_test': stderr while " - "it should not be there"), - ("""\ - name: simple echo command: "echo Hello, world" stdout: From e0cf667e26f48ef7039c10bb15a0399f44760cf5 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 18 Nov 2022 13:53:55 +0100 Subject: [PATCH 74/94] Use literal blocks instead of code-blocks for unsupported languages --- docs/examples.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/examples.rst b/docs/examples.rst index 307874ea..a2a5d6d4 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -81,7 +81,7 @@ the running of miniwdl from the environment. Miniwdl will localize all the output files to an ``output_links`` directory inside the test output directory. If you have a workflow with the output: -.. code-block:: wdl +:: output { File moo_file = moo_task.out @@ -120,7 +120,7 @@ it is possible to define the output directory when running the code. An example code defining a ``publishDir`` is listed below. -.. code-block:: nextflow +:: process Hello { publishDir = [ From d20033a5396e689defb36b4b92dbe48ec887038c Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 18 Nov 2022 13:56:07 +0100 Subject: [PATCH 75/94] Correctly define blocks --- docs/examples.rst | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/docs/examples.rst b/docs/examples.rst index a2a5d6d4..55947c55 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -79,14 +79,12 @@ information on the localization of output files as well as options to modify the running of miniwdl from the environment. Miniwdl will localize all the output files to an ``output_links`` directory -inside the test output directory. If you have a workflow with the output: +inside the test output directory. If you have a workflow with the output:: -:: - - output { - File moo_file = moo_task.out - Array[File] stats = moo_task.stats_files - } + output { + File moo_file = moo_task.out + Array[File] stats = moo_task.stats_files + } Inside the ``out`` directory the directories ``moo_file`` and ``stats`` will be created. Inside these directories will be the produced files. @@ -118,15 +116,13 @@ also be stored. Nextflow can output a copy of the output files to a separate wor directory. This can be achieved by defining a ``publishDir`` in the process. Through ``params.outdir`` it is possible to define the output directory when running the code. -An example code defining a ``publishDir`` is listed below. - -:: +An example code defining a ``publishDir`` is listed below. :: process Hello { publishDir = [ path: { "${params.outdir}/hello"} ] - + output: path "HelloWorld.txt" script: @@ -134,7 +130,7 @@ An example code defining a ``publishDir`` is listed below. echo "Hello World!" > HelloWorld.txt """ } - + workflow { Hello } From 90ffcd037172a0fd7556c926360f5029fb3ccd28 Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 21 Nov 2022 11:42:39 +0100 Subject: [PATCH 76/94] add desired_exit_code to init of workflow object --- src/pytest_workflow/plugin.py | 5 ++--- src/pytest_workflow/workflow.py | 8 ++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/pytest_workflow/plugin.py b/src/pytest_workflow/plugin.py index d0047a8e..85cb148c 100644 --- a/src/pytest_workflow/plugin.py +++ b/src/pytest_workflow/plugin.py @@ -406,7 +406,8 @@ def queue_workflow(self): # Create a workflow and make sure it runs in the tempdir workflow = Workflow(command=self.workflow_test.command, cwd=tempdir, - name=self.workflow_test.name) + name=self.workflow_test.name, + desired_exit_code=self.workflow_test.exit_code) # Add the workflow to the workflow queue. self.config.workflow_queue.put(workflow) @@ -443,8 +444,6 @@ def collect(self): # collection phase. workflow = self.queue_workflow() - workflow.desired_exit_code = self.workflow_test.exit_code - # Below structure makes it easy to append tests tests = [] diff --git a/src/pytest_workflow/workflow.py b/src/pytest_workflow/workflow.py index b85116e3..748b0244 100644 --- a/src/pytest_workflow/workflow.py +++ b/src/pytest_workflow/workflow.py @@ -35,7 +35,8 @@ class Workflow(object): def __init__(self, command: str, cwd: Optional[Path] = None, - name: Optional[str] = None): + name: Optional[str] = None, + desired_exit_code: int = 0): """ Initiates a workflow object :param command: The string that represents the command to be run @@ -69,7 +70,7 @@ def __init__(self, self._started = False self.errors: List[Exception] = [] self.start_lock = threading.Lock() - self.desired_exit_code = 0 + self.desired_exit_code = desired_exit_code def start(self): """Runs the workflow in a subprocess in the background. @@ -139,6 +140,9 @@ def wait(self, timeout_secs: Optional[float] = None, pass def matching_exitcode(self) -> bool: + """Checks if the workflow exited with the desired exit code""" + # This is done in the workflow object to reduce redundancy in the rest + # of the code. return self.exit_code == self.desired_exit_code @property From b6d77e2956853855d54929a3bbbf23d01aedd950 Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 21 Nov 2022 11:56:32 +0100 Subject: [PATCH 77/94] add test for matchin_exitcode function --- tests/test_workflow.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_workflow.py b/tests/test_workflow.py index 7bf8d6f7..46344278 100644 --- a/tests/test_workflow.py +++ b/tests/test_workflow.py @@ -96,3 +96,12 @@ def test_workflow_name(): def test_workflow_name_inferred(): workflow = Workflow("echo moo") assert workflow.name == "echo" + + +def test_workflow_matching_exit_code(): + workflow = Workflow("echo moo") + workflow.run() + assert workflow.matching_exitcode() + workflow2 = Workflow("grep", desired_exit_code=2) + workflow2.run() + assert workflow2.matching_exitcode() From 272e261ec594a186ad4d5dd60e176bf522d44d50 Mon Sep 17 00:00:00 2001 From: lucasvanb <95374055+lucasvanb@users.noreply.github.com> Date: Mon, 21 Nov 2022 12:08:35 +0100 Subject: [PATCH 78/94] split test_skip_tests.py into 3 tests --- tests/test_skip_tests.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/test_skip_tests.py b/tests/test_skip_tests.py index a63387c7..d95e0f2b 100644 --- a/tests/test_skip_tests.py +++ b/tests/test_skip_tests.py @@ -28,7 +28,8 @@ - "halloumi" must_not_contain: - "gorgonzola" - +""") +SKIP_TESTS2 = textwrap.dedent(""" - name: wall2_test command: bash -c "echo 'testing' > test3.txt" files: @@ -37,7 +38,8 @@ - "kaas" must_not_contain: - "testing" - +""") +SKIP_TESTS3 = textwrap.dedent(""" - name: wall3_test command: bash -c "exit 12" files: @@ -53,4 +55,16 @@ def test_skips(pytester): pytester.makefile(".yml", test=SKIP_TESTS) result = pytester.runpytest("-v").parseoutcomes() - assert {"failed": 5, "passed": 3, "skipped": 7} == result + assert {"failed": 2, "passed": 1, "skipped": 3} == result + + +def test_skips2(pytester): + pytester.makefile(".yml", test=SKIP_TESTS2) + result = pytester.runpytest("-v").parseoutcomes() + assert {"failed": 2, "passed": 2} == result + + +def test_skips3(pytester): + pytester.makefile(".yml", test=SKIP_TESTS3) + result = pytester.runpytest("-v").parseoutcomes() + assert {"failed": 1, "skipped": 4} == result From 9e88c05f846251cf36e5c6231cfd66c5442b2e3c Mon Sep 17 00:00:00 2001 From: LucasvanBostelen <95374055+lucasvanb@users.noreply.github.com> Date: Fri, 25 Nov 2022 12:31:24 +0100 Subject: [PATCH 79/94] restored and fixed fail tests --- tests/test_fail_messages.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_fail_messages.py b/tests/test_fail_messages.py index ea48da68..4640bd06 100644 --- a/tests/test_fail_messages.py +++ b/tests/test_fail_messages.py @@ -93,6 +93,24 @@ "moo' was found in 'echo does not contain moo': stdout while " "it should not be there"), ("""\ + - name: fail_test + command: bash -c 'echo "" >&2' + stderr: + contains: + - "No arguments?" + """, + "'No arguments?' was not found in 'fail_test': stderr " + "while it should be there"), + ("""\ + - name: fail_test + command: bash -c 'echo "grep --help" >&2' + stderr: + must_not_contain: + - "grep --help" + """, + "'grep --help' was found in 'fail_test': stderr while " + "it should not be there"), + ("""\ - name: simple echo command: "echo Hello, world" stdout: From 5b118ba84f3e6e6fdc6e9071a28097d197aaf925 Mon Sep 17 00:00:00 2001 From: Will Holtz Date: Tue, 6 Dec 2022 14:06:31 -0800 Subject: [PATCH 80/94] docs: pytest flags -k and -m are not supported --- docs/running_pytest_workflow.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/running_pytest_workflow.rst b/docs/running_pytest_workflow.rst index cbe704d0..0c6989f3 100644 --- a/docs/running_pytest_workflow.rst +++ b/docs/running_pytest_workflow.rst @@ -129,3 +129,9 @@ Internally names and tags are handled the same so if the following tests: - hello are run with ``pytest --tag hello`` then both ``hello`` and ``hello2`` are run. + +.. note:: + + The pytest flags ``-k`` and ``-m`` are not supported by pytest-workflow. + Rational for this design decision can be `found within GitHub issue #155 + `_. From 192da828f6028c4693502070a9f718269b8c1a92 Mon Sep 17 00:00:00 2001 From: Will Holtz Date: Tue, 6 Dec 2022 14:09:03 -0800 Subject: [PATCH 81/94] Update HISTORY.rst --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index 474e9bee..6d5e3d88 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -31,6 +31,7 @@ version 1.7.0-dev 7.0.0. + Throw a more descriptive error when a file copied with the --git-aware flag is not present on the filesystem anymore. ++ Document pytest flags ``-k`` and ``-m`` are not supported. version 1.6.0 --------------------------- From d7558f91488d8f7f2249619ed800be53f6e03b1a Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 7 Dec 2022 11:47:53 +0100 Subject: [PATCH 82/94] Drop python 3.6 support --- .github/workflows/ci.yml | 5 +---- HISTORY.rst | 5 ++++- README.rst | 4 ++-- docs/installation.rst | 2 +- docs/known_issues.rst | 2 +- docs/writing_tests.rst | 2 +- setup.py | 5 ++--- 7 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c613953..88eb8b00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: python-version: - - "3.6" + - "3.7" steps: - uses: actions/checkout@v2.3.4 - name: Set up Python ${{ matrix.python-version }} @@ -26,7 +26,6 @@ jobs: - name: Lint run: tox -e lint docs: - needs: lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v2.3.4 @@ -43,7 +42,6 @@ jobs: strategy: matrix: python-version: - - "3.6" - "3.7" - "3.8" - "3.9" @@ -60,7 +58,6 @@ jobs: - name: Run tests run: tox -e py3 - name: Upload coverage report - if: ${{ matrix.python-version == 3.6 }} # Only upload coverage once uses: codecov/codecov-action@v1 test-functional: diff --git a/HISTORY.rst b/HISTORY.rst index 474e9bee..82b5a0ac 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,8 +7,11 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. -version 1.7.0-dev +version 2.0.0-dev --------------------------- ++ Python 3.6 is no longer supported. It has been removed from github actions, + as such we can no longer guarantee that pytest-workflow works properly + with python 3.6. + When the ``--git-aware`` flag is used a submodule check is performed in order to assert that all submodules are properly checked out. This prevents unclear copying errors. diff --git a/README.rst b/README.rst index 739803e2..2ae92fb5 100644 --- a/README.rst +++ b/README.rst @@ -44,7 +44,7 @@ For our complete documentation and examples checkout our Installation ============ -Pytest-workflow requires Python 3.6 or higher. It is tested on Python 3.6, 3.7, +Pytest-workflow requires Python 3.7 or higher. It is tested on Python 3.7, 3.8, 3.9, 3.10 and 3.11. - Make sure your virtual environment is activated. @@ -143,7 +143,7 @@ predefined tests as well as custom tests are possible. - '^Hello .*' # Complex regexes will break yaml if double quotes are used For more information on how Python parses regular expressions, see the `Python -documentation `_. +documentation `_. Documentation for more advanced use cases including the custom tests can be found on our `readthedocs page `_. diff --git a/docs/installation.rst b/docs/installation.rst index e7a18766..d07b87db 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -2,7 +2,7 @@ Installation ============ -Pytest-workflow is tested on python 3.6, 3.7, 3.8, 3.9, 3.10 and 3.11. +Pytest-workflow is tested on python 3.7, 3.8, 3.9, 3.10 and 3.11. In a virtual environment ------------------------ diff --git a/docs/known_issues.rst b/docs/known_issues.rst index 42808670..c6ec3857 100644 --- a/docs/known_issues.rst +++ b/docs/known_issues.rst @@ -21,4 +21,4 @@ Known issues ``contains_regex`` and ``must_not_contain_regex``, since this collides with Python's usage of the same character to escape special characters in strings. Please see the `Python documentation on regular expressions - `_ for details. + `_ for details. diff --git a/docs/writing_tests.rst b/docs/writing_tests.rst index 62086d68..8e45958a 100644 --- a/docs/writing_tests.rst +++ b/docs/writing_tests.rst @@ -83,7 +83,7 @@ Test options The above YAML file contains all the possible options for a workflow test. Please see the `Python documentation on regular expressions -`_ to see how Python handles escape +`_ to see how Python handles escape sequences. .. note:: diff --git a/setup.py b/setup.py index a567ca91..a9431a3b 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,6 @@ classifiers=[ "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", @@ -50,8 +49,8 @@ "GNU Affero General Public License v3 or later (AGPLv3+)", "Framework :: Pytest", ], - # Because we use the resolve(strict=False) feature from pathlib. - python_requires=">=3.6", + # Because we cannot test anymore on Python 3.6. + python_requires=">=3.7", install_requires=[ "pytest>=7.0.0", # To use pathlib Path's in pytest "pyyaml", From b6d9cebf69a5d6b11dc8ae634ae427b57c33f777 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 21 Dec 2022 07:41:06 +0100 Subject: [PATCH 83/94] Ensure files are only read once for contain regexes and strings combined --- src/pytest_workflow/content_tests.py | 70 +++++++--------------------- tests/test_content_functions.py | 12 ++--- 2 files changed, 24 insertions(+), 58 deletions(-) diff --git a/src/pytest_workflow/content_tests.py b/src/pytest_workflow/content_tests.py index b27f2d1a..77f5d760 100644 --- a/src/pytest_workflow/content_tests.py +++ b/src/pytest_workflow/content_tests.py @@ -33,72 +33,42 @@ def check_content(strings: Iterable[str], - text_lines: Iterable[str]) -> Set[str]: + patterns: Iterable[str], + text_lines: Iterable[str]): """ - Checks whether any of the strings is present in the text lines + Checks whether any of the strings or patterns is present in the text lines It only reads the lines once and it stops reading when - everything is found. This makes searching for strings in large bodies of - text more efficient. - :param strings: A list of strings for which the present is checked + everything is found. This makes searching for strings and patterns in + large bodies of text more efficient. + :param strings: A list of strings to check for + :param patterns: A list of regex patterns to check for :param text_lines: The lines of text that need to be searched. - :return: A tuple with a set of found strings, and a set of not found - strings + :return: A tuple with a set of found strings, and a set of found patterns. """ - - # Create two sets. By default all strings are not found. strings_to_check = set(strings) found_strings: Set[str] = set() + regex_to_match: Set[re.Pattern] = {re.compile(pattern) + for pattern in patterns} + found_regexes: Set[re.Pattern] = set() for line in text_lines: # Break the loop if all strings are found # Python implements fast set equality checking by checking length first - if found_strings == strings_to_check: + if not strings_to_check and not regex_to_match: break for string in strings_to_check: if string in line: found_strings.add(string) - # Remove found strings for faster searching. This should be done - # outside of the loop above. - strings_to_check -= found_strings - return found_strings - - -def check_regex_content(patterns: Iterable[str], - text_lines: Iterable[str]) -> Set[str]: - """ - Checks whether any of the patterns is present in the text lines - It only reads the lines once and it stops reading when - everything is found. This makes searching for patterns in large bodies of - text more efficient. - :param patterns: A list of regexes which is matched - :param text_lines: The lines of text that need to be searched. - :return: A tuple with a set of found regexes, and a set of not found - regexes - """ - - # Create two sets. By default all strings are not found. - regex_to_match = {re.compile(pattern) for pattern in patterns} - found_patterns: Set[str] = set() - - for line in text_lines: - # Break the loop if all regexes have been matched - if not regex_to_match: - break - - # Regexes we don't have to check anymore - to_remove = list() for regex in regex_to_match: if re.search(regex, line): - found_patterns.add(regex.pattern) - to_remove.append(regex) + found_regexes.add(regex) - # Remove found patterns for faster searching. This should be done + # Remove found strings for faster searching. This should be done # outside of the loop above. - for regex in to_remove: - regex_to_match.remove(regex) - - return found_patterns + strings_to_check -= found_strings + regex_to_match -= found_regexes + return found_strings, {x.pattern for x in found_regexes} class ContentTestCollector(pytest.Collector): @@ -146,12 +116,8 @@ def find_strings(self): try: # Use 'rt' here explicitly as opposed to 'rb' with file_open(mode='rt') as file_handler: # type: ignore # mypy goes crazy here otherwise # noqa: E501 - self.found_strings = check_content( + self.found_strings, self.found_patterns = check_content( strings=strings_to_check, - text_lines=file_handler) - # Read the file again for the regex - with file_open(mode='rt') as file_handler: # type: ignore # mypy goes crazy here otherwise # noqa: E501 - self.found_patterns = check_regex_content( patterns=patterns_to_check, text_lines=file_handler) except FileNotFoundError: diff --git a/tests/test_content_functions.py b/tests/test_content_functions.py index 743d4d4c..67858be3 100644 --- a/tests/test_content_functions.py +++ b/tests/test_content_functions.py @@ -20,7 +20,7 @@ import pytest -from pytest_workflow.content_tests import check_content, check_regex_content +from pytest_workflow.content_tests import check_content LICENSE = Path(__file__).parent / "content_files" / "LICENSE" LICENSE_ZIPPED = LICENSE.parent / "LICENSE.gz" @@ -48,8 +48,8 @@ def test_check_content_succeeding(contains_strings, does_not_contain_strings): all_strings = set(contains_strings).union(set(does_not_contain_strings)) with LICENSE.open("rt") as license_h: - found_strings = check_content(list(all_strings), - license_h) + found_strings, _ = check_content( + list(all_strings), [], license_h) assert set(contains_strings) == found_strings assert set(does_not_contain_strings) == all_strings - found_strings @@ -60,8 +60,8 @@ def test_check_regex_content_succeeding(contains_regex, does_not_contain_regex): all_regex = set(contains_regex).union(set(does_not_contain_regex)) with LICENSE.open("rt") as license_h: - found_regex = check_regex_content(list(all_regex), - license_h) + _, found_regex = check_content( + [], list(all_regex), license_h) assert set(contains_regex) == found_regex assert set(does_not_contain_regex) == all_regex - found_regex @@ -72,5 +72,5 @@ def test_multiple_finds_one_line(): "the true meaning of its creed: \"We hold these truths to be", "self-evident: that all men are created equal.\""] contains = ["dream", "day", "nation", "creed", "truths"] - found_strings = check_content(contains, content) + found_strings, _ = check_content(contains, [], content) assert set(contains) == found_strings From 116ff4f8fb5533683e76af179a60b54705d90a40 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 21 Dec 2022 08:24:27 +0100 Subject: [PATCH 84/94] Create minimal reproducer for encoding issue --- tests/test_encodings.py | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tests/test_encodings.py diff --git a/tests/test_encodings.py b/tests/test_encodings.py new file mode 100644 index 00000000..f05f4aa9 --- /dev/null +++ b/tests/test_encodings.py @@ -0,0 +1,43 @@ +# Copyright (C) 2018 Leiden University Medical Center +# This file is part of pytest-workflow +# +# pytest-workflow is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# pytest-workflow is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with pytest-workflow. If not, see zeeën, +bacterie -> bacteriën. +Daarnaast worden veel diakritische tekens gebruikt in leenworden. Denk hierbij +aan woorden als: überhaupt, crème fraîche en Curaçao.""" + + +def test_encoding(pytester): + pytester.makefile(".yml", textwrap.dedent(""" + - name: test_encoding + command: "bash -c 'true'" + files: + - path: test.txt + contains: + - überhaupt + - crème fraîche + """)) + test_txt = pytester.path / "test.txt" + # UTF32 is not the default on windows and linux I believe + test_txt.write_text(TEST_TEXT, encoding="UTF32") + result = pytester.runpytest("-v") + assert result.ret == 0 From 2390cfd1151b2d93c0ae367107b0055209221325 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 21 Dec 2022 08:38:51 +0100 Subject: [PATCH 85/94] Add encoding keyword to file tests --- src/pytest_workflow/content_tests.py | 3 ++- src/pytest_workflow/schema.py | 10 +++++++--- src/pytest_workflow/schema/schema.json | 9 +++++++++ tests/test_encodings.py | 5 +++-- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/pytest_workflow/content_tests.py b/src/pytest_workflow/content_tests.py index 77f5d760..bd64afbc 100644 --- a/src/pytest_workflow/content_tests.py +++ b/src/pytest_workflow/content_tests.py @@ -115,7 +115,8 @@ def find_strings(self): self.filepath.open) try: # Use 'rt' here explicitly as opposed to 'rb' - with file_open(mode='rt') as file_handler: # type: ignore # mypy goes crazy here otherwise # noqa: E501 + with file_open(mode='rt', encoding=self.content_test.encoding) \ + as file_handler: # type: ignore # mypy goes crazy here otherwise # noqa: E501 self.found_strings, self.found_patterns = check_content( strings=strings_to_check, patterns=patterns_to_check, diff --git a/src/pytest_workflow/schema.py b/src/pytest_workflow/schema.py index df8084a8..c8c3e40f 100644 --- a/src/pytest_workflow/schema.py +++ b/src/pytest_workflow/schema.py @@ -113,11 +113,13 @@ class ContentTest(object): def __init__(self, contains: Optional[List[str]] = None, must_not_contain: Optional[List[str]] = None, contains_regex: Optional[List[str]] = None, - must_not_contain_regex: Optional[List[str]] = None): + must_not_contain_regex: Optional[List[str]] = None, + encoding: Optional[str] = None): self.contains: List[str] = contains or [] self.must_not_contain: List[str] = must_not_contain or [] self.contains_regex: List[str] = contains_regex or [] self.must_not_contain_regex: List[str] = must_not_contain_regex or [] + self.encoding: Optional[str] = encoding class FileTest(ContentTest): @@ -127,7 +129,8 @@ def __init__(self, path: str, md5sum: Optional[str] = None, contains: Optional[List[str]] = None, must_not_contain: Optional[List[str]] = None, contains_regex: Optional[List[str]] = None, - must_not_contain_regex: Optional[List[str]] = None): + must_not_contain_regex: Optional[List[str]] = None, + encoding: Optional[str] = None): """ A container object :param path: the path to the file @@ -143,7 +146,8 @@ def __init__(self, path: str, md5sum: Optional[str] = None, """ super().__init__(contains=contains, must_not_contain=must_not_contain, contains_regex=contains_regex, - must_not_contain_regex=must_not_contain_regex) + must_not_contain_regex=must_not_contain_regex, + encoding=encoding) self.path = Path(path) self.md5sum = md5sum self.should_exist = should_exist diff --git a/src/pytest_workflow/schema/schema.json b/src/pytest_workflow/schema/schema.json index c2356818..9ead66bb 100644 --- a/src/pytest_workflow/schema/schema.json +++ b/src/pytest_workflow/schema/schema.json @@ -67,6 +67,9 @@ "items": { "type": "string" } + }, + "encoding": { + "type": "string" } }, "additionalProperties": false @@ -97,6 +100,9 @@ "items": { "type": "string" } + }, + "encoding": { + "type": "string" } }, "additionalProperties": false @@ -140,6 +146,9 @@ "items": { "type": "string" } + }, + "encoding": { + "type": "string" } }, "required": [ diff --git a/tests/test_encodings.py b/tests/test_encodings.py index f05f4aa9..03f90469 100644 --- a/tests/test_encodings.py +++ b/tests/test_encodings.py @@ -31,12 +31,13 @@ def test_encoding(pytester): - name: test_encoding command: "bash -c 'true'" files: - - path: test.txt + - path: diakritische_tekens.txt + encoding: UTF32 contains: - überhaupt - crème fraîche """)) - test_txt = pytester.path / "test.txt" + test_txt = pytester.path / "diakritische_tekens.txt" # UTF32 is not the default on windows and linux I believe test_txt.write_text(TEST_TEXT, encoding="UTF32") result = pytester.runpytest("-v") From 6361e3371dd1bd83fa5a05b480a7be2a1499971d Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 21 Dec 2022 08:43:51 +0100 Subject: [PATCH 86/94] Add encoding to examples --- README.rst | 3 +++ docs/writing_tests.rst | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 2ae92fb5..c53eb866 100644 --- a/README.rst +++ b/README.rst @@ -103,6 +103,7 @@ predefined tests as well as custom tests are possible. must_not_contain: # A list of strings that should NOT be in the file (optional) - "Cock a doodle doo" md5sum: e583af1f8b00b53cda87ae9ead880224 # Md5sum of the file (optional) + encoding: UTF-8 # Encoding for the text file (optional). Defaults to system locale. - name: simple echo # A second workflow. Notice the starting `-` which means command: "echo moo" # that workflow items are in a list. You can add as much workflows as you want @@ -114,6 +115,7 @@ predefined tests as well as custom tests are possible. - "moo" must_not_contain: # List of strings that should NOT be in stout (optional) - "Cock a doodle doo" + encoding: ASCII # Encoding for stdout (optional). Defaults to system locale. - name: mission impossible # Also failing workflows can be tested tags: # A list of tags that can be used to select which test @@ -130,6 +132,7 @@ predefined tests as well as custom tests are possible. - "BSOD error, please contact the IT crowd" must_not_contain: # A list of strings which should NOT be in stderr (optional) - "Mission accomplished!" + encoding: UTF-16 # Encoding for stderr (optional). Defaults to system locale. - name: regex tests command: echo Hello, world diff --git a/docs/writing_tests.rst b/docs/writing_tests.rst index 8e45958a..9d75d435 100644 --- a/docs/writing_tests.rst +++ b/docs/writing_tests.rst @@ -40,6 +40,7 @@ Test options must_not_contain: # A list of strings that should NOT be in the file (optional) - "Cock a doodle doo" md5sum: e583af1f8b00b53cda87ae9ead880224 # Md5sum of the file (optional) + encoding: UTF-8 # Encoding for the text file (optional). Defaults to system locale. - name: simple echo # A second workflow. Notice the starting `-` which means command: "echo moo" # that workflow items are in a list. You can add as much workflows as you want @@ -51,6 +52,7 @@ Test options - "moo" must_not_contain: # List of strings that should NOT be in stout (optional) - "Cock a doodle doo" + encoding: ASCII # Encoding for stdout (optional). Defaults to system locale. - name: mission impossible # Also failing workflows can be tested tags: # A list of tags that can be used to select which test @@ -60,13 +62,14 @@ Test options files: - path: "fail.log" # Multiple files can be tested for each workflow - path: "TomCruise.txt.gz" # Gzipped files can also be searched, provided their extension is '.gz' - contains: + contains: - "starring" stderr: # Options for testing stderr (optional) contains: # A list of strings which should be in stderr (optional) - "BSOD error, please contact the IT crowd" must_not_contain: # A list of strings which should NOT be in stderr (optional) - "Mission accomplished!" + encoding: UTF-16 # Encoding for stderr (optional). Defaults to system locale. - name: regex tests command: echo Hello, world From 32bd39cdbb1a28364d737a02aa33958fffda0e88 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 21 Dec 2022 08:45:36 +0100 Subject: [PATCH 87/94] Also validate encoding in schema --- tests/yamls/valid/dream_file.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/yamls/valid/dream_file.yaml b/tests/yamls/valid/dream_file.yaml index 72b9d7e2..d5096d69 100644 --- a/tests/yamls/valid/dream_file.yaml +++ b/tests/yamls/valid/dream_file.yaml @@ -11,16 +11,19 @@ - "blabla" must_not_contain: - "stuff" + encoding: UTF8 stdout: contains: - "bla" must_not_contain: - "not_bla" + encoding: ASCII stderr: contains: - "bla" must_not_contain: - "not_bla" + encoding: UTF8 exit_code: 127 command: "the one string" - name: other test From 54f113c75af404102662e24dc56d218e1914572b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 21 Dec 2022 09:40:28 +0100 Subject: [PATCH 88/94] Update changelog with encoding and content test change --- HISTORY.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 8585ca66..4132e7a1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,10 @@ version 2.0.0-dev + Python 3.6 is no longer supported. It has been removed from github actions, as such we can no longer guarantee that pytest-workflow works properly with python 3.6. ++ Added an optional encoding key for files, stdout and stderr so the file can + be opened with the proper encoding. ++ Make content tests more efficient by reading each file only once instead of + twice when there are both strings and regexes to check for. + When the ``--git-aware`` flag is used a submodule check is performed in order to assert that all submodules are properly checked out. This prevents unclear copying errors. From 75de0664922ed7c9992deb7cde43e82f5991b037 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 21 Dec 2022 09:58:08 +0100 Subject: [PATCH 89/94] Use allowlist --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c8029aba..af66c289 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ commands = # Documentation should build on python version 3 [testenv:docs] deps=-r requirements-docs.txt -whitelist_externals=bash +allowlist_externals=bash mkdir rm commands= From c54d0573d83643a6707e1d2082b1b8e52dd2dd7f Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 21 Dec 2022 10:50:46 +0100 Subject: [PATCH 90/94] Add a reproducer for the issue --- tests/test_utils.py | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 098f1e67..b7cca37e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -166,17 +166,25 @@ def create_git_repo(path): os.mkdir(dir) file = dir / "README.md" file.write_text("# My new project\n\nHello this project is awesome!\n") + subdir = dir / "sub" + subdir.mkdir() + another_file = subdir / "subtext.md" + another_file.write_text("# Subtext\n\nSome other example text.\n") + subdir_link = dir / "gosub" + subdir_link.symlink_to(subdir, target_is_directory=True) subprocess.run(["git", "init", "-b", "main"], cwd=dir) subprocess.run(["git", "config", "user.name", "A U Thor"], cwd=dir) subprocess.run(["git", "config", "user.email", "author@example.com"], cwd=dir) - subprocess.run(["git", "add", "README.md"], cwd=dir) + subprocess.run(["git", "add", "."], cwd=dir) subprocess.run(["git", "commit", "-m", "initial commit"], cwd=dir) -def test_git_submodule_check(tmp_path): - bird_repo = tmp_path / "bird" - nest_repo = tmp_path / "nest" +@pytest.fixture() +def git_repo_with_submodules(): + repo_dir = Path(tempfile.mkdtemp()) + bird_repo = repo_dir / "bird" + nest_repo = repo_dir / "nest" create_git_repo(bird_repo) create_git_repo(nest_repo) # https://bugs.launchpad.net/ubuntu/+source/git/+bug/1993586 @@ -185,11 +193,16 @@ def test_git_submodule_check(tmp_path): cwd=nest_repo.absolute()) subprocess.run(["git", "commit", "-m", "add bird repo as a submodule"], cwd=nest_repo.absolute()) + yield nest_repo + shutil.rmtree(repo_dir) + + +def test_git_submodule_check(git_repo_with_submodules, tmp_path): cloned_repo = tmp_path / "cloned" subprocess.run( # No recursive clone ["git", "-c", "protocol.file.allow=always", - "clone", nest_repo.absolute(), cloned_repo.absolute()], + "clone", git_repo_with_submodules.absolute(), cloned_repo.absolute()], cwd=tmp_path ) with pytest.raises(RuntimeError) as error: @@ -201,3 +214,13 @@ def test_git_submodule_check(tmp_path): cwd=cloned_repo.absolute()) # Check error does not occur when issue resolved. git_check_submodules_cloned(cloned_repo) + + +# https://github.com/LUMC/pytest-workflow/issues/162 +def test_duplicate_git_tree_submodule_symlinks(git_repo_with_submodules): + assert (git_repo_with_submodules / ".git").exists() + dest = Path(tempfile.mkdtemp()) / "test" + duplicate_tree(git_repo_with_submodules, dest, git_aware=True) + assert dest.exists() + assert not (dest / ".git").exists() + assert (dest / "bird" / "gosub" / "subtext.md").exists() From 11ed0e313c9d4f307e67e73835abc0f40f28b083 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 21 Dec 2022 11:00:34 +0100 Subject: [PATCH 91/94] Fix submodule with links issue --- src/pytest_workflow/util.py | 4 +++- tests/test_utils.py | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/pytest_workflow/util.py b/src/pytest_workflow/util.py index 94e2ffbf..211e3ab3 100644 --- a/src/pytest_workflow/util.py +++ b/src/pytest_workflow/util.py @@ -173,7 +173,9 @@ def duplicate_tree(src: Filepath, dest: Filepath, copy: Callable[[Filepath, Filepath], None] = \ functools.partial(os.symlink, target_is_directory=False) else: - copy = shutil.copy2 # Preserves metadata, also used by shutil.copytree + # shutil.copy2 preserves metadata, also used by shutil.copytree + # follow_symlinks False to directly copy links + copy = functools.partial(shutil.copy2, follow_symlinks=False) os.makedirs(dest, exist_ok=False) for src_path, dest_path, is_dir in path_iter: diff --git a/tests/test_utils.py b/tests/test_utils.py index b7cca37e..005d5a72 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -171,7 +171,7 @@ def create_git_repo(path): another_file = subdir / "subtext.md" another_file.write_text("# Subtext\n\nSome other example text.\n") subdir_link = dir / "gosub" - subdir_link.symlink_to(subdir, target_is_directory=True) + subdir_link.symlink_to(subdir.relative_to(dir), target_is_directory=True) subprocess.run(["git", "init", "-b", "main"], cwd=dir) subprocess.run(["git", "config", "user.name", "A U Thor"], cwd=dir) subprocess.run(["git", "config", "user.email", "author@example.com"], @@ -223,4 +223,7 @@ def test_duplicate_git_tree_submodule_symlinks(git_repo_with_submodules): duplicate_tree(git_repo_with_submodules, dest, git_aware=True) assert dest.exists() assert not (dest / ".git").exists() - assert (dest / "bird" / "gosub" / "subtext.md").exists() + link = dest / "bird" / "gosub" + assert link.exists() + assert link.is_symlink() + assert link.resolve() == dest / "bird" / "sub" From b2f8516828e02f63d3267b1457d26f8a9f567d58 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 21 Dec 2022 11:02:28 +0100 Subject: [PATCH 92/94] Add symlink in git issue to changelog --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index 8585ca66..4c677022 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,7 @@ version 2.0.0-dev + Python 3.6 is no longer supported. It has been removed from github actions, as such we can no longer guarantee that pytest-workflow works properly with python 3.6. ++ Fix an issue where symlinks in git repositories could not be properly copied. + When the ``--git-aware`` flag is used a submodule check is performed in order to assert that all submodules are properly checked out. This prevents unclear copying errors. From 6c52eafbe14b556f5f4c03b85359364c66246336 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Fri, 23 Dec 2022 09:16:23 +0100 Subject: [PATCH 93/94] Remove redundant comment --- src/pytest_workflow/content_tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pytest_workflow/content_tests.py b/src/pytest_workflow/content_tests.py index bd64afbc..27701886 100644 --- a/src/pytest_workflow/content_tests.py +++ b/src/pytest_workflow/content_tests.py @@ -53,7 +53,6 @@ def check_content(strings: Iterable[str], for line in text_lines: # Break the loop if all strings are found - # Python implements fast set equality checking by checking length first if not strings_to_check and not regex_to_match: break From 2366451502b300990a5ec44de8073bac3e303851 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 4 Jan 2023 12:23:28 +0100 Subject: [PATCH 94/94] Version 2.0.0 --- HISTORY.rst | 7 ++++++- setup.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8d514cf4..79d9d35c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,8 +7,13 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. -version 2.0.0-dev +version 2.0.0 --------------------------- +This major release greatly cleans up the output of pytest-workflow in case of +an error as well as providing the stderr and stdout last bytes for debugging +purposes. When the exit code test fails all other tests from the workflow +are skipped. + + Python 3.6 is no longer supported. It has been removed from github actions, as such we can no longer guarantee that pytest-workflow works properly with python 3.6. diff --git a/setup.py b/setup.py index a9431a3b..50d383c0 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name="pytest-workflow", - version="1.7.0-dev", + version="2.0.0", description="A pytest plugin for configuring workflow/pipeline tests " "using YAML files", author="Leiden University Medical Center",