From 49cee1fe2ac0ec16fb72ae32ab962ba0220e7600 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 14:44:17 +0000 Subject: [PATCH 01/14] build(deps-dev): bump pytest from 7.4.1 to 7.4.2 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.1 to 7.4.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.1...7.4.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e4d649119..babc89199 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ "dev": [ "pre-commit==3.4.0", "pyyaml==6.0.1", - "pytest==7.4.1", + "pytest==7.4.2", "pytest-sugar==0.9.4", "pytest-instafail==0.5.0", "pytest-cov==4.1.0", From 52a4026ec457942ff18e4bc604e7cdf0c37216ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 14:43:53 +0000 Subject: [PATCH 02/14] build(deps-dev): bump pyinstaller from 5.13.2 to 6.0.0 Bumps [pyinstaller](https://github.com/pyinstaller/pyinstaller) from 5.13.2 to 6.0.0. - [Release notes](https://github.com/pyinstaller/pyinstaller/releases) - [Changelog](https://github.com/pyinstaller/pyinstaller/blob/develop/doc/CHANGES.rst) - [Commits](https://github.com/pyinstaller/pyinstaller/compare/v5.13.2...v6.0.0) --- updated-dependencies: - dependency-name: pyinstaller dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e4d649119..b9d964cb3 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ "types-tabulate==0.9.0.3", ], "build": [ - "pyinstaller==5.13.2", + "pyinstaller==6.0.0", ], }, zip_safe=False, From 912c26aaab95a5d5f2a4231170bb7dde079ba63a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:28:12 +0000 Subject: [PATCH 03/14] build(deps-dev): bump mypy from 1.5.1 to 1.6.0 Bumps [mypy](https://github.com/python/mypy) from 1.5.1 to 1.6.0. - [Commits](https://github.com/python/mypy/compare/v1.5.1...v1.6.0) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b9d964cb3..5e4f76000 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ "pycodestyle==2.11.0", "black==23.9.1", "isort==5.11.4", - "mypy==1.5.1", + "mypy==1.6.0", # type stubs for mypy "types-PyYAML==6.0.10", "types-tabulate==0.9.0.3", From a17009b43d4d14d5c06885e533759a66890f5384 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:28:17 +0000 Subject: [PATCH 04/14] build(deps-dev): bump pre-commit from 3.4.0 to 3.5.0 Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: pre-commit dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b9d964cb3..64b261432 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ install_requires=requirements, extras_require={ "dev": [ - "pre-commit==3.4.0", + "pre-commit==3.5.0", "pyyaml==6.0.1", "pytest==7.4.1", "pytest-sugar==0.9.4", From ec93fd5c5f02817a8eafb22960bc5e3498d37c75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:28:22 +0000 Subject: [PATCH 05/14] build(deps-dev): bump pycodestyle from 2.11.0 to 2.11.1 Bumps [pycodestyle](https://github.com/PyCQA/pycodestyle) from 2.11.0 to 2.11.1. - [Release notes](https://github.com/PyCQA/pycodestyle/releases) - [Changelog](https://github.com/PyCQA/pycodestyle/blob/main/CHANGES.txt) - [Commits](https://github.com/PyCQA/pycodestyle/compare/2.11.0...2.11.1) --- updated-dependencies: - dependency-name: pycodestyle dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b9d964cb3..16b0ae14d 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,7 @@ "pytest-sugar==0.9.4", "pytest-instafail==0.5.0", "pytest-cov==4.1.0", - "pycodestyle==2.11.0", + "pycodestyle==2.11.1", "black==23.9.1", "isort==5.11.4", "mypy==1.5.1", From c7f56cdc1a272600428d4382f7d3bcd5da75869b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:28:26 +0000 Subject: [PATCH 06/14] build(deps-dev): bump pyinstaller from 6.0.0 to 6.1.0 Bumps [pyinstaller](https://github.com/pyinstaller/pyinstaller) from 6.0.0 to 6.1.0. - [Release notes](https://github.com/pyinstaller/pyinstaller/releases) - [Changelog](https://github.com/pyinstaller/pyinstaller/blob/develop/doc/CHANGES.rst) - [Commits](https://github.com/pyinstaller/pyinstaller/compare/v6.0.0...v6.1.0) --- updated-dependencies: - dependency-name: pyinstaller dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b9d964cb3..0420c7a64 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ "types-tabulate==0.9.0.3", ], "build": [ - "pyinstaller==6.0.0", + "pyinstaller==6.1.0", ], }, zip_safe=False, From bbca88f988cd22b937c1b3838463178e55b835b9 Mon Sep 17 00:00:00 2001 From: Willi Ballenthin Date: Tue, 17 Oct 2023 15:28:55 +0200 Subject: [PATCH 07/14] installation: use releases/latest URL to avoid QS spam in release listing --- doc/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/installation.md b/doc/installation.md index 2f097e527..70924413c 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -3,7 +3,7 @@ ## Installation You can install FLOSS in a few different ways. First, if you simply want to use FLOSS to extract strings, just download - the [standalone binaries](https://github.com/mandiant/flare-floss/releases). + the [standalone binaries](https://github.com/mandiant/flare-floss/releases/latest). However, if you want to use FLOSS as a Python library, you can install the package directly from GitHub using `pip`. Finally, if you'd like to contribute patches or features to FLOSS, From 8b9aef9bdaddf120a5a9abe62df7697b1160469d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Oct 2023 14:57:17 +0000 Subject: [PATCH 08/14] build(deps-dev): bump black from 23.9.1 to 23.10.0 Bumps [black](https://github.com/psf/black) from 23.9.1 to 23.10.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/23.9.1...23.10.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 836853e85..dad3dedc0 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ "pytest-instafail==0.5.0", "pytest-cov==4.1.0", "pycodestyle==2.11.1", - "black==23.9.1", + "black==23.10.0", "isort==5.11.4", "mypy==1.6.0", # type stubs for mypy From cb5fdcbdda2c095b9866ee4194810e41d4211a6d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 17:59:55 +0000 Subject: [PATCH 09/14] build(deps-dev): bump mypy from 1.6.0 to 1.6.1 Bumps [mypy](https://github.com/python/mypy) from 1.6.0 to 1.6.1. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.6.0...v1.6.1) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dad3dedc0..233104720 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ "pycodestyle==2.11.1", "black==23.10.0", "isort==5.11.4", - "mypy==1.6.0", + "mypy==1.6.1", # type stubs for mypy "types-PyYAML==6.0.10", "types-tabulate==0.9.0.3", From 3e751f77b62b4b7a6e4e32c80f2326c5edd555aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 14:03:08 +0000 Subject: [PATCH 10/14] build(deps-dev): bump pytest from 7.4.2 to 7.4.3 Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.2 to 7.4.3. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.2...7.4.3) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index df7741cae..b7947681b 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ "dev": [ "pre-commit==3.5.0", "pyyaml==6.0.1", - "pytest==7.4.2", + "pytest==7.4.3", "pytest-sugar==0.9.4", "pytest-instafail==0.5.0", "pytest-cov==4.1.0", From 4ef1ba5c5ecd150fa68f695493b7f911be001873 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 14:03:20 +0000 Subject: [PATCH 11/14] build(deps-dev): bump black from 23.10.0 to 23.10.1 Bumps [black](https://github.com/psf/black) from 23.10.0 to 23.10.1. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/23.10.0...23.10.1) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index df7741cae..864e00fe0 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ "pytest-instafail==0.5.0", "pytest-cov==4.1.0", "pycodestyle==2.11.1", - "black==23.10.0", + "black==23.10.1", "isort==5.11.4", "mypy==1.6.1", # type stubs for mypy From 7dce9ad5c017335e68f4aa2e2c3a0ffde94aba06 Mon Sep 17 00:00:00 2001 From: mr-tz Date: Thu, 9 Nov 2023 10:46:01 +0100 Subject: [PATCH 12/14] simplify language configuration --- floss/language/go/extract.py | 4 +++- floss/main.py | 30 ++++++++++++++++++++---------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/floss/language/go/extract.py b/floss/language/go/extract.py index 03a07a02d..9aa0ffd27 100644 --- a/floss/language/go/extract.py +++ b/floss/language/go/extract.py @@ -276,7 +276,9 @@ def get_string_blob_strings(pe: pefile.PE, min_length) -> Iterable[StaticString] try: string_blob_start, string_blob_end = find_string_blob_range(pe, struct_strings) except ValueError: - logger.warning("Failed to find string blob range: Go version may be unsupported.") + logger.warning( + "Failed to find string blob range: Is this a Go binary? If so, the Go version may be unsupported." + ) return with floss.utils.timing("collect string blob strings"): diff --git a/floss/main.py b/floss/main.py index 6ef29e9cc..acd288acf 100644 --- a/floss/main.py +++ b/floss/main.py @@ -546,28 +546,38 @@ def main(argv=None) -> int: static_runtime = get_runtime_diff(interim) - lang_id = identify_language(sample, static_strings) - # set language configurations - if (lang_id == Language.GO and args.language == "") or args.language == Language.GO.value: + lang_id: Language + if args.language == Language.GO.value: + lang_id = Language.GO + elif args.language == Language.RUST.value: + lang_id = Language.RUST + elif args.language == Language.DOTNET.value: + lang_id = Language.DOTNET + elif args.language == "none": + lang_id = Language.UNKNOWN + else: + lang_id = identify_language(sample, static_strings) + + if lang_id == Language.GO: if analysis.enable_tight_strings or analysis.enable_stack_strings or analysis.enable_decoded_strings: logger.warning( "FLOSS handles Go static strings, but string deobfuscation may be inaccurate and take a long time" ) results.metadata.language = Language.GO.value - elif (lang_id == Language.RUST and args.language == "") or args.language == Language.RUST.value: + elif lang_id == Language.RUST: if analysis.enable_tight_strings or analysis.enable_stack_strings or analysis.enable_decoded_strings: logger.warning( "FLOSS handles Rust static strings, but string deobfuscation may be inaccurate and take a long time" ) results.metadata.language = Language.RUST.value - elif (lang_id == Language.DOTNET and args.language == "") or args.language == Language.DOTNET.value: - logger.warning(".NET language-specific string extraction is not supported") - logger.warning(" will NOT deobfuscate any .NET strings") + elif lang_id == Language.DOTNET: + logger.warning(".NET language-specific string extraction is not supported yet") + logger.warning("Furthermore, FLOSS does NOT attempt to deobfuscate any strings from .NET binaries") - # let's enable .NET strings after we can deobfuscate them + # enable .NET strings once we can extract them # results.metadata.language = Language.DOTNET.value # TODO for pure .NET binaries our deobfuscation algorithms do nothing, but for mixed-mode assemblies they may @@ -604,7 +614,7 @@ def main(argv=None) -> int: if not lang_id: logger.info("extracting static strings") else: - if (lang_id == Language.GO and args.language == "") or args.language == Language.GO.value: + if lang_id == Language.GO: logger.info("extracting language-specific Go strings") interim = time() @@ -615,7 +625,7 @@ def main(argv=None) -> int: static_strings, results.strings.language_strings, args.min_length ) - elif (lang_id == Language.RUST and args.language == "") or args.language == Language.RUST.value: + elif lang_id == Language.RUST: logger.info("extracting language-specific Rust strings") interim = time() From a7b7ea30ba8b916b20f67ca660e4bbc7425c032a Mon Sep 17 00:00:00 2001 From: mr-tz Date: Thu, 9 Nov 2023 12:53:06 +0100 Subject: [PATCH 13/14] fix b2s wide/utf-8 string handling via workaround --- floss/language/rust/extract.py | 55 +++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/floss/language/rust/extract.py b/floss/language/rust/extract.py index f67e57c82..3d3dc3960 100644 --- a/floss/language/rust/extract.py +++ b/floss/language/rust/extract.py @@ -4,7 +4,7 @@ import pathlib import argparse import itertools -from typing import List, Tuple, Iterable +from typing import List, Tuple, Iterable, Optional import pefile import binary2strings as b2s @@ -25,6 +25,41 @@ def get_rdata_section(pe: pefile.PE) -> pefile.SectionStructure: raise ValueError("no .rdata section found") +def fix_b2s_wide_strings( + strings: List[Tuple[str, str, Tuple[int, int], bool]], min_length: int, buffer: bytes +) -> List[Tuple[str, str, Tuple[int, int], bool]]: + # TODO(mr-tz): b2s may parse wide strings where there really should be utf-8 strings + # handle special cases here until fixed + # https://github.com/mandiant/flare-floss/issues/867 + fixed_strings: List[Tuple[str, str, Tuple[int, int], bool]] = list() + last_fixup: Optional[Tuple[str, str, Tuple[int, int], bool]] = None + for string in strings: + s = string[0] + string_type = string[1] + start = string[2][0] + + if string_type == "WIDE_STRING": + sd = s.encode("utf-16le", "ignore") + # utf-8 strings will not start with \x00 + if sd[0] == 0: + new_string = b2s.extract_string(buffer[start + 1 :]) + last_fixup = ( + new_string[0], + new_string[1], + (new_string[2][0] + start + 1, new_string[2][1] + start + 1), + new_string[3], + ) + if len(last_fixup[0]) < min_length: + last_fixup = None + else: + if last_fixup and s in last_fixup[0]: + fixed_strings.append(last_fixup) + else: + fixed_strings.append(string) + last_fixup = None + return fixed_strings + + def filter_and_transform_utf8_strings( strings: List[Tuple[str, str, Tuple[int, int], bool]], start_rdata: int, @@ -46,7 +81,7 @@ def filter_and_transform_utf8_strings( return transformed_strings -def split_strings(static_strings: List[StaticString], address: int) -> None: +def split_strings(static_strings: List[StaticString], address: int, min_length: int) -> None: """ if address is in between start and end of a string in ref data then split the string this modifies the elements of the static strings list directly @@ -57,8 +92,12 @@ def split_strings(static_strings: List[StaticString], address: int) -> None: rust_string = string.string[0 : address - string.offset] rest = string.string[address - string.offset :] - static_strings.append(StaticString(string=rust_string, offset=string.offset, encoding=StringEncoding.UTF8)) - static_strings.append(StaticString(string=rest, offset=address, encoding=StringEncoding.UTF8)) + if len(rust_string) >= min_length: + static_strings.append( + StaticString(string=rust_string, offset=string.offset, encoding=StringEncoding.UTF8) + ) + if len(rest) >= min_length: + static_strings.append(StaticString(string=rest, offset=address, encoding=StringEncoding.UTF8)) # remove string from static_strings for static_string in static_strings: @@ -97,12 +136,14 @@ def get_string_blob_strings(pe: pefile.PE, min_length: int) -> Iterable[StaticSt end_rdata = start_rdata + rdata_section.SizeOfRawData virtual_address = rdata_section.VirtualAddress pointer_to_raw_data = rdata_section.PointerToRawData + buffer_rdata = rdata_section.get_data() # extract utf-8 and wide strings, latter not needed here - strings = b2s.extract_all_strings(rdata_section.get_data(), min_length) + strings = b2s.extract_all_strings(buffer_rdata, min_length) + fixed_strings = fix_b2s_wide_strings(strings, min_length, buffer_rdata) # select only UTF-8 strings and adjust offset - static_strings = filter_and_transform_utf8_strings(strings, start_rdata) + static_strings = filter_and_transform_utf8_strings(fixed_strings, start_rdata) struct_string_addrs = map(lambda c: c.address, get_struct_string_candidates(pe)) @@ -126,7 +167,7 @@ def get_string_blob_strings(pe: pefile.PE, min_length: int) -> Iterable[StaticSt if not (start_rdata <= address < end_rdata): continue - split_strings(static_strings, address) + split_strings(static_strings, address, min_length) return static_strings From 98fbde05a9c657259229f8a2888bc7e19f2f147b Mon Sep 17 00:00:00 2001 From: mr-tz Date: Thu, 9 Nov 2023 13:57:14 +0100 Subject: [PATCH 14/14] enable test --- tests/test_language_extract_rust.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_language_extract_rust.py b/tests/test_language_extract_rust.py index f507efe75..9d210ce0c 100644 --- a/tests/test_language_extract_rust.py +++ b/tests/test_language_extract_rust.py @@ -30,12 +30,11 @@ def rust_strings64(): # .rdata:00000001400BD040 30 D0 0B 40 01 00 pieces ___str_ # .rdata:00000001400BD040 00 00 00 00 ; "Hello, world!\n" pytest.param("Hello, world!", 0xBB030, StringEncoding.UTF8, "rust_strings64"), - # TODO enable, see issue #867 # .rdata:00000001400BD050 69 6E 76 61 6C 69 aInvalidArgs db 'invalid args',0 # .rdata:00000001400BD05D 00 00 00 align 20h # .rdata:00000001400BD060 50 D0 0B 40 01 00 stru_1400BD060 ___str_ # .rdata:00000001400BD060 00 00 00 00 ; "invalid args" - # pytest.param("invalid args", 0xBB050, StringEncoding.UTF8, "rust_strings64"), + pytest.param("invalid args", 0xBB050, StringEncoding.UTF8, "rust_strings64"), ], ) def test_data_string_offset(request, string, offset, encoding, rust_strings):