Merge branch 'develop' into develop

.github/workflows/python/github.py

@@ -31,20 +31,20 @@ def __init__(self):
 
     def get_reviews(self):
         """ Get a list of reviews on a Github pull request as json object """
-        reviews = self.session.get(self.api + 'repos/{}/pulls/{}/reviews'.format(self.github_repository, self.pr_id))
+        reviews = self.session.get(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews')
         reviews.raise_for_status()
         return reviews.json()
 
     def update_review(self, review_id, body):
         """ Update a review given by `review_id` and set its body to `body` """
         payload = {'body': body}
-        resp = self.session.put(self.api + 'repos/{}/pulls/{}/reviews/{}'.format(self.github_repository, self.pr_id, review_id), json=payload)
+        resp = self.session.put(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews/{review_id}', json=payload)
         resp.raise_for_status()
         return resp.json()
 
     def post_review(self, body):
         """ Post a pull request review containing `body` and requesting changes """
         payload = {'body': body, 'event': "REQUEST_CHANGES"}
-        resp = self.session.post(self.api + 'repos/{}/pulls/{}/reviews'.format(self.github_repository, self.pr_id), json=payload)
+        resp = self.session.post(self.api + f'repos/{self.github_repository}/pulls/{self.pr_id}/reviews', json=payload)
         resp.raise_for_status()
         return resp.json()

.github/workflows/python/pycodestyle_comment.py

@@ -34,7 +34,7 @@ def style_error_format(style_error_list) -> str:
     """ Format the list of pycodestyle errors and return them a one string. """
     ret = ''
     for error in style_error_list:
-        ret += '* {}\n'.format(error)
+        ret += f'* {error}\n'
     return ret
 
 
@@ -45,7 +45,7 @@ def style_error_format(style_error_list) -> str:
         style_errors = list_style_errors()
 
         if style_errors:
-            print("Found {} errors.".format(len(style_errors)))
+            print(f"Found {len(style_errors)} errors.")
 
         gh = github.Github()
 

.github/workflows/scripts/setup-full.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# SPDX-FileCopyrightText: 2020 Birger Schacht
+# SPDX-FileCopyrightText: 2020 Birger Schacht, 2024 Institute for Common Good Technology
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 set -x
@@ -14,6 +14,14 @@ echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlo
 sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch
 sudo systemctl start elasticsearch
 
+sudo apt update
+if [ $python_version == '3.8' ]; then
+	# for pymssql there are no wheels for 3.8 https://github.com/certtools/intelmq/issues/2539
+	DEBIAN_FRONTEND="noninteractive" sudo -E apt install -y build-essential freetds-dev libssl-dev libkrb5-dev
+fi
+# for psql (used below)
+DEBIAN_FRONTEND="noninteractive" sudo -E apt install -y postgresql-client-14
+
 # Install the dependencies of all the bots
 pip install wheel
 for file in intelmq/bots/*/*/REQUIREMENTS.txt; do
@@ -30,7 +38,16 @@ done
 # Setup sudo and install intelmq
 sudo sed -i '/^Defaults\tsecure_path.*$/ d' /etc/sudoers
 sudo pip install .
-sudo intelmqsetup --skip-ownership
+
+intelmq_user_exists=$(getent passwd intelmq ||:)
+if [[ "$UID" -eq '0' && -z "$intelmq_user_exists" ]]; then
+	# create an unprivileged user, if currently running as root. Otherwise dropping privileges won't work
+	groupadd -r intelmq
+	useradd -r -d /var/lib/intelmq/ -c "user running intelmq" -g intelmq -s /bin/bash intelmq
+	sudo intelmqsetup
+else
+	sudo intelmqsetup --skip-ownership
+fi
 
 # Initialize the postgres database
 intelmq_psql_initdb

.github/workflows/unittests.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
         type: ['full', 'basic']
 
     services:
@@ -59,6 +59,7 @@ jobs:
         PGPORT: 5432
         PGUSER: intelmq
         PGPASSWORD: intelmq
+        python_version: ${{ matrix.python-version }}
       run: bash .github/workflows/scripts/setup-full.sh
 
     - name: Install test dependencies

CHANGELOG.md

@@ -12,6 +12,7 @@
 ### Configuration
 
 ### Core
+- Python 3.8 or newer is required (PR#2541 by Sebastian Wagner).
 
 ### Development
 
@@ -30,15 +31,25 @@
   - Fix to avoid schema download if not configured #2530.
 
 #### Experts
+- `intelmq.bots.experts.securitytxt`:
+  - Added new bot (PR#2538 by Frank Westers and Sebastian Wagner)
 - `intelmq.bots.experts.misp`: Use `PyMISP` class instead of deprecated `ExpandedPyMISP` (PR#2532 by Radek Vyhnal)
 
 #### Outputs
+- `intelmq.bots.outputs.cif3.output`:
+  - The requirement can only be installed on Python version < 3.12.
+  - Add a check on the Python version and exit if incompatible.
+  - Add a deprecation warning (PR#2544 by Sebastian Wagner)
 
 ### Documentation
 
 ### Packaging
 
 ### Tests
+- Install build dependencies for `pymssql` on Python 3.8 as there are no wheels available for this Python version (PR#2542 by Sebastian Wagner).
+- Install `psql` explicitly for workflow support on other platforms such as act (PR#2542 by Sebastian Wagner).
+- Create intelmq user & group if running privileged to allow dropping privileges (PR#2542 by Sebastian Wagner).
+- `intelmq.tests.lib.test_pipeline.TestAmqp.test_acknowledge`: Also skip on Python 3.11 besides on 3.8 when running on CI (PR#2542 by Sebastian Wagner).
 
 ### Tools
 

Makefile

@@ -19,3 +19,6 @@ codespell:
 
 test:
 	pytest --no-cov -v intelmq/tests/ && echo "Success!"
+
+codestyle:
+	pycodestyle intelmq/{bots,lib,bin}

NEWS.md

@@ -14,6 +14,12 @@ Please refer to the change log for a full list of changes.
 ---------------------------------
 
 ### Requirements
+Python 3.8 or newer is required.
+
+## Bots
+#### CIF 3 API
+The CIF 3 API Output bot is not compatible with Python version greater or equal to 3.12 and will be removed in the future due to lack of maintenance.
+See https://lists.cert.at/pipermail/intelmq-users/2024-December/000474.html for more information.
 
 ### Tools
 
@@ -36,7 +42,7 @@ No changes are required by administrators.
 ----------------------------------
 
 ### Documentation
-The documentation is now available at [docs.intelmq.org](https://docs.intelmq.org/). Documentation has been updated and restructured into User, Administrator and Developer Guide. It provides modern look with various quality of life improvements. Big thanks to to @gethvi. 
+The documentation is now available at [docs.intelmq.org](https://docs.intelmq.org/). Documentation has been updated and restructured into User, Administrator and Developer Guide. It provides modern look with various quality of life improvements. Big thanks to to @gethvi.
 We now have a slick, modern mkdocs based documentation. Please do check it out!
 
 
@@ -55,7 +61,7 @@ Shadowserver adds new scans on a nearly weekly basis. IntelMQ's release cycle an
 We therefore (thanks to @eslif2) move the shadowserver reports collector and parser to a new, dynamic system. It can:
 
  - fetch the shadowserver schema from shadowserver (https://interchange.shadowserver.org/intelmq/v1/schema)
- - dynamically collect new reports (see also https://docs.intelmq.org/latest/user/bots/?h=shadow#shadowserver-reports-api) 
+ - dynamically collect new reports (see also https://docs.intelmq.org/latest/user/bots/?h=shadow#shadowserver-reports-api)
  - parse the new reports
 
 **Note well**: if your IntelMQ system runs in an airgapped environment or if it may only reach out to specific IPs/sites, you should read the notes here:
@@ -86,7 +92,7 @@ Quite a few changes (thanks to Kamil, @gethvi) on AMQP
 ### General changes and bug fixes
 
 Digital Trust Center fixed a bug where the config was loaded twice in intelmqctl which created quite some speedups. Thanks!
-This speeds up IntelMQ API calls. 
+This speeds up IntelMQ API calls.
 
 ### Data Format
 

contrib/example-extension-package/setup.py

@@ -28,7 +28,7 @@
     entry_point = '.'.join(file.with_suffix('').parts)
     file = Path(str(file).replace('intelmq/bots', 'mybots/bots'))
     module = '.'.join(file.with_suffix('').parts)
-    BOTS.append('{0} = {1}:BOT.run'.format(entry_point, module))
+    BOTS.append(f'{entry_point} = {module}:BOT.run')
 
 setup(
     name='intelmq-example-extension',

docs/dev/release.md

@@ -30,8 +30,6 @@ These apply to all projects:
 
 - `intelmq/version.py`: Update the version.
 
-Eventually adapt the default log levels if necessary. Should be INFO for stable releases.
-
 ### IntelMQ API
 
 -   `intelmq_api/version.py`: Update the version.

docs/user/bots.md

@@ -3524,6 +3524,56 @@ to true.
 (optional, boolean) Query for IPs at `https://stat.ripe.net/data/maxmind-geo-lite/data.json?resource=%s`. Defaults to
 true.
 
+---
+
+### SecurityTXT <div id="intelmq.bots.experts.securitytxt.expert" />
+
+SecurityTXT is an initiative to standardize how websites publish their abuse contact information.
+It is standardized in [RFC 9116 "A File Format to Aid in Security Vulnerability Disclosure"](https://datatracker.ietf.org/doc/rfc9116/).
+Refer to the linked document RFC for more information on `security.txt`.
+This bot looks for `security.txt` files on a URL or IP, retrieves the primary contact information out of it and adds this to the event.
+
+**Requirements**
+
+To use this bot, you need to install the required dependencies:
+
+```bash
+pip3 install -r intelmq/bots/experts/securitytxt/REQUIREMENTS.txt
+```
+
+**Module:** `intelmq.bots.experts.securitytxt.expert`
+
+**Parameters**
+
+**`url_field`**
+
+The field in the event that contains the URL/IP on which to look for the the security.txt file. Default: `source.reverse_dns`
+
+**`contact_field`**
+
+The field in the event in which to put the found contact details. Default: `source.abuse_contact`
+
+**`only_email_address`** (bool)
+
+Contact details can be web URLs or email addresses. When this value is set to True, it only selects email addresses as contact information.
+Default: `true`
+
+**`overwrite`** (bool)
+
+Boolean indicating whether to override existing data in contact_field.
+Default: `true`
+
+**`check_expired`** (bool)
+
+Boolean indicating whether to check if the security.txt has expired according to its own expiry date.
+Default: `false`
+
+**`check_canonical`** (bool)
+
+Boolean indicating whether to check if the url is contained in the list of canonical urls.
+Default: `false`
+
+
 ---
 
 ### Sieve <div id="intelmq.bots.experts.sieve.expert" />

intelmq/__init__.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2014 Tomás Lima
+# SPDX-FileCopyrightText: 2014 Tomás Lima, 2015-2020 nic.at GmbH, 2024 Institute for Common Good Technology
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
@@ -29,7 +29,12 @@
     VAR_STATE_PATH = os.path.join(ROOT_DIR, "var/lib/bots/")
 
 
-DEFAULT_LOGGING_LEVEL = "INFO"
+if isinstance(__version_info__[-1], str) and __version_info__[-1][0].lower() in ('a', 'b', 'd'):
+    # for alpha, beta and dev instances, set default log level to DEBUG, for others, including RCs, use INFO
+    DEFAULT_LOGGING_LEVEL = "DEBUG"
+else:
+    DEFAULT_LOGGING_LEVEL = "INFO"
+
 HARMONIZATION_CONF_FILE = os.path.join(CONFIG_DIR, "harmonization.conf")
 RUNTIME_CONF_FILE = os.path.join(CONFIG_DIR, "runtime.yaml")
 old_runtime_conf_file = pathlib.Path(RUNTIME_CONF_FILE).with_suffix('.conf')

intelmq/bots/collectors/rsync/collector_rsync.py

@@ -13,7 +13,7 @@
 from intelmq.lib.bot import CollectorBot
 
 
-class Time(object):
+class Time:
     def __init__(self, delta=None):
         """ Delta is a datetime.timedelta JSON string, ex: '{days=-1}'. """
         self.time = datetime.now()

intelmq/bots/experts/modify/expert.py

@@ -17,10 +17,7 @@ def is_re_pattern(value):
     """
     Checks if the given value is a re compiled pattern
     """
-    if sys.version_info > (3, 7):
-        return isinstance(value, re.Pattern)
-    else:
-        return hasattr(value, "pattern")
+    return isinstance(value, re.Pattern)
 
 
 class MatchGroupMapping:

intelmq/bots/experts/securitytxt/REQUIREMENTS.txt

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: 2022 Frank Westers, 2024 Institute for Common Good Technology
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+wellknown-securitytxt