From a01cadbb1040aab886f80709530ba6253f2a3c9b Mon Sep 17 00:00:00 2001 From: Ian Leonard Date: Sat, 28 Jan 2023 05:20:05 -0500 Subject: [PATCH 1/9] buildsystem: copy files from DISTRIBUTION's filesystem This allows creating $DISTRO_DIR/$DISTRO/filesystem/ as a location to store files that should be copied to the image similar to how it is done for PROJECT or DEVICE filesystems. Signed-off-by: Ian Leonard --- scripts/image | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/image b/scripts/image index 3d6c6d1b447..52cb6613b0c 100755 --- a/scripts/image +++ b/scripts/image @@ -183,6 +183,17 @@ EOF ln -sf /etc/issue ${INSTALL}/etc/motd +# Copy DISTRIBUTION related files to filesystem +if [ -d "${DISTRO_DIR}/${DISTRO}/filesystem" ]; then + cp -PR --remove-destination ${DISTRO_DIR}/${DISTRO}/filesystem/* ${INSTALL} + #install distribution specific systemd services + for service in ${DISTRO_DIR}/${DISTRO}/filesystem/usr/lib/systemd/system/*.service; do + if [ -f "${service}" ]; then + enable_service ${service##*/} + fi + done +fi + # Copy PROJECT related files to filesystem if [ -d "${PROJECT_DIR}/${PROJECT}/filesystem" ]; then cp -PR --remove-destination ${PROJECT_DIR}/${PROJECT}/filesystem/* ${INSTALL} From ce6f91140d10c7d1bc362822972e882e03a8df68 Mon Sep 17 00:00:00 2001 From: Ian Leonard Date: Fri, 24 Feb 2023 04:02:41 -0500 Subject: [PATCH 2/9] show_config: use DISTRO_DIR Signed-off-by: Ian Leonard --- config/show_config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/show_config b/config/show_config index d1da4dac05c..e045dac82e8 100644 --- a/config/show_config +++ b/config/show_config @@ -147,8 +147,8 @@ show_config() { # $DISTRO/config/functions # $DISTRO/show_config - if [ -f distributions/${DISTRO}/show_config ]; then - . distributions/${DISTRO}/show_config + if [ -f "${DISTRO_DIR}/${DISTRO}/show_config" ]; then + . "${DISTRO_DIR}/${DISTRO}/show_config" fi if [ "$(type -t show_distro_config)" = "function" ]; then show_distro_config From ac1da2fea7ec23dc206ce5f2c0ebd77eb1214007 Mon Sep 17 00:00:00 2001 From: Ian Leonard Date: Fri, 24 Feb 2023 23:34:37 +0000 Subject: [PATCH 3/9] distro/update-system: add cli update script Signed-off-by: Ian Leonard --- .../filesystem/usr/bin/update-system | 371 ++++++++++++++++++ 1 file changed, 371 insertions(+) create mode 100755 distributions/LibreELEC/filesystem/usr/bin/update-system diff --git a/distributions/LibreELEC/filesystem/usr/bin/update-system b/distributions/LibreELEC/filesystem/usr/bin/update-system new file mode 100755 index 00000000000..7d9a526b911 --- /dev/null +++ b/distributions/LibreELEC/filesystem/usr/bin/update-system @@ -0,0 +1,371 @@ +#!/usr/bin/env python + +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (C) 2023-present Team LibreELEC (https://libreelec.tv) + +# disables pylint checks for: +# variable/function naming convention +# lack of docstring to start module +# line length check +# pylint: disable=C0103,C0114,C0301 + + +import argparse +import json +import os +import shutil +import tempfile +import urllib.request + +from datetime import datetime, timedelta +from hashlib import sha256 + + +# List of locations with JSON release data (may be a single string entry) +JSON_DOWNLOAD_URL = ['https://releases.libreelec.tv/releases_v2.json', 'https://releases.libreelec.tv/releases.json'] +# Where to put update files +LOCAL_UPDATE_DIR = '/storage/.update' + + +class UpdateSystem(): + '''Functions to check for, obtain, and validate system updates.''' + def __init__(self): + # information on running system + self.current = { + 'architecture': None, + 'distribution': None, + 'version': None, + 'version_id': None, + 'version_major': None, + 'version_minor': None, + 'version_bugfix': None, + 'timestamp': None + } + # information on proposed update + self.candidate = { + 'canarydate': None, + 'filename': None, + 'sha256': None, + 'subpath': None, + 'timestamp': None, + 'url': None, + 'version_major': None, + 'version_minor': None, + 'version_bugfix': None + } + # releases.json + self.update_json = None + # update results + self.abort_check = False + self.update_available = False + self.update_major = False + self.update_url = None + + + @staticmethod + def get_highest_value(values): + '''Review list of integers (that are internally strings, like releases) and return highest as string.''' + highest_value = 0 # releases start at 0 + for value in values: + value = int(value) + highest_value = max(value, highest_value) + return str(highest_value) + + + def parse_osrelease(self): + '''Read /etc/os-release and set corresponding variables.''' + with open('/etc/os-release', 'r', encoding='utf-8') as data: + content = data.read() + for line in content.splitlines(): + if line[0:15] == 'LIBREELEC_ARCH=': + self.current['architecture'] = line.split('=')[1].strip('\"') + elif line[0:5] == 'NAME=': + self.current['distribution'] = line.split('=')[1].strip('\"') + elif line[0:8] == 'VERSION=': + self.current['version'] = line.split('=')[1].strip('\"') + elif line[0:11] == 'VERSION_ID=': + self.current['version_id'] = line.split('=')[1].strip('\"') + if self.current['architecture'] and self.current['distribution'] and self.current['version'] and self.current['version_id']: + break + # If debugging other devices, change self.current[architecture, distribution, version, version_id] here + self.current['version_major'], self.current['version_minor'] = [int(i) for i in self.current['version_id'].split('.')] + self.current['version_bugfix'] = int(self.current['version'].split('.')[2]) if not self.current['version'].startswith(('devel', 'nightly')) else None + self.current['timestamp'] = datetime.strptime(self.current['version'].split('-')[1], '%Y%m%d') if 'nightly' in self.current['version'] else None + + + def fetch_update_json(self): + '''Downloads releases.json file and readies for parsing.''' + if args.json: + if args.json.startswith(('http://', 'https://')): + releases_json = args.json + elif os.path.isfile(os.path.join(os.getcwd(), args.json)): + releases_json = f'file://{os.path.join(os.getcwd(), args.json)}' + else: + print(f'ERROR: Unable to locate: {args.json}.') + return + else: + releases_json = JSON_DOWNLOAD_URL + if isinstance(releases_json, list): + for url in releases_json: + try: + data = urllib.request.urlopen(url) + except urllib.error.HTTPError as err: + if err.code == 404: + print(f'ERROR: HTTP 404: Failed to download from: {url}') + continue + else: + print(f'Unhandled HTTPError: {err=}') + raise + else: + break + else: + data = urllib.request.urlopen(releases_json) + self.update_json = json.loads(data.read().decode('utf-8').strip()) if data else None + + + def parse_device_json(self, device_json, canary=None): + '''Parse json fields of a device's release entry.''' + self.candidate['filename'] = device_json['file']['name'] + # Assumes filename format is 'distribution-device.arch-version.tar' + version = tuple(int(i) for i in self.candidate['filename'].split('-')[-1].removesuffix('.tar').split('.')) + self.candidate['version_major'] = version[0] + self.candidate['version_minor'] = version[1] + self.candidate['version_bugfix'] = version[2] + self.candidate['sha256'] = device_json['file']['sha256'] + # old releases.json without subpath field + self.candidate['subpath'] = device_json['file']['subpath'] if 'subpath' in device_json['file'] else None + # old releases.json without timestamp field + self.candidate['timestamp'] = datetime.strptime(device_json['file']['timestamp'], '%Y-%m-%d %H:%M:%S') if 'timestamp' in device_json['file'] else None + # set if parts are known + self.candidate['canarydate'] = self.candidate['timestamp'] + timedelta(days=canary) if self.candidate['timestamp'] and canary else None + + + def fetch_update_file(self): + '''Download update_url to a temporary directory. Copy to LOCAL_UPDATE_DIR when finished.''' + def get_sha256_hash(file_name): + '''Calculate sha256 sum of file_name in 8KiB chunks.''' + with open(file_name, mode='rb') as file: + sha256hash = sha256() + while True: + data_block = file.read(8192) + if not data_block: + break + sha256hash.update(data_block) + return sha256hash.hexdigest() + + + download_sha256sum = None + with tempfile.TemporaryDirectory() as update_temp_dir: + try: + with urllib.request.urlopen(self.update_url) as download, open(f'{update_temp_dir}/update.file', mode='wb') as file_in_progress: + shutil.copyfileobj(download, file_in_progress) + except Exception as e: + print(e) + # delete partial download + if os.path.isfile(f'{update_temp_dir}/update.file'): + print('ERROR: Download failure. Deleting partially downloaded file.') + os.remove(f'{update_temp_dir}/update.file') + else: + print('Download finished.') + if os.path.isfile(f'{update_temp_dir}/update.file'): + download_sha256sum = get_sha256_hash(f'{update_temp_dir}/update.file') + if args.verbose: + print(f'{self.candidate["sha256"]=}\n{download_sha256sum=}') + if self.candidate['sha256'] == download_sha256sum: + print(f'Copying {self.candidate["filename"]} to {LOCAL_UPDATE_DIR}') + shutil.copy2(f'{update_temp_dir}/update.file', f'{LOCAL_UPDATE_DIR}/{self.candidate["filename"]}') + else: + print(f'ERROR: sha256 checksum failure.\n Wanted: {self.candidate["sha256"]}\n Downloaded: {download_sha256sum}') + print(' Deleting downloaded file.') + os.remove(f'{update_temp_dir}/update.file') + + + def precheck_update(self): + '''Gather and validate information needed for update check.''' + self.parse_osrelease() + if args.verbose: + print(f'{self.current["architecture"]=}\n{self.current["distribution"]=}\n{self.current["version"]=}\n{self.current["version_id"]=}') + if not (self.current['architecture'] and self.current['distribution'] and self.current['version'] and self.current['version_id']): + print('ERROR: parse_osrelease failed. Unable to determine running device or version.') + self.abort_check = True + self.update_available = False + self.update_major = False + return + if self.current['version'].startswith('devel'): + print('ERROR: Automatic updates from development builds are unsupported.') + self.abort_check = True + self.update_available = False + self.update_major = False + return + + # Retrieve json with release data + if not self.update_json: + self.fetch_update_json() + if not self.update_json: + print('ERROR: Failed to load json release data.') + self.abort_check = True + self.update_available = False + self.update_major = False + return + + + def check_for_bugfix(self): + '''Review releases.json for bugfix update.''' + self.precheck_update() + if self.abort_check: + return + + release_branch = f'{self.current["distribution"]}-{self.current["version_id"]}' + if release_branch not in self.update_json: + print('Running release branch not in json file.') + self.update_available = False + self.update_major = False + self.update_url = None + return + device_release = UpdateSystem.get_highest_value(self.update_json[release_branch]['project'][self.current['architecture']]['releases']) + # Old releases.json without canary field + release_canary = self.update_json[release_branch]['canary'] if 'canary' in self.update_json[release_branch] else None + + # Parses highest (most recent) release of device within releases.json file + self.parse_device_json(self.update_json[release_branch]['project'][self.current['architecture']]['releases'][device_release], release_canary) + + # Higher bugfix release or stable release after running nightly available + # 'is not None' is on purpose - '0' is valid but evals to Falsey + if (self.current['version_bugfix'] is not None and self.candidate['version_bugfix'] > self.current['version_bugfix']) or (self.candidate['timestamp'] and self.current['timestamp'] and self.candidate['timestamp'] > self.current['timestamp']): + self.candidate['url'] = self.update_json[release_branch]['url'] + self.update_url = f'{self.candidate["url"]}{self.candidate["subpath"]}/{self.candidate["filename"]}' if self.candidate['subpath'] else f'{self.candidate["url"]}{self.candidate["filename"]}' + if args.force or (self.candidate['canarydate'] and datetime.now() > self.candidate['canarydate']): + self.update_available = True + return + if self.candidate['canarydate']: + print(f'Update held until {self.candidate["canarydate"]} for general availability.') + else: + print('Unable to determine testing period of update. Holding update back. Use --force to proceed anyway.') + self.update_available = False + self.update_major = False + self.update_url = None + return + + + def check_for_major(self): + '''Review releases.json for major update.''' + self.precheck_update() + if self.abort_check: + return + + # Check for major system updates + highest_version_major = self.current['version_major'] + highest_version_minor = self.current['version_minor'] + # Highest release branch for architecture + for os_branch in self.update_json: + key_version_major, key_version_minor = [int(i) for i in os_branch.removeprefix(f'{self.current["distribution"]}-').split('.')] + if ((key_version_major > highest_version_major or + (key_version_major == highest_version_major and key_version_minor > highest_version_minor)) and + self.current['architecture'] in self.update_json[os_branch]['project']): + highest_version_major = key_version_major + highest_version_minor = key_version_minor + # Test if release series changed + if highest_version_major == self.current['version_major'] and highest_version_minor == self.current['version_minor']: + self.update_available = False + self.update_major = False + return + + release_branch = f'{self.current["distribution"]}-{highest_version_major}.{highest_version_minor}' + # determine latest release for device + device_release = UpdateSystem.get_highest_value(self.update_json[release_branch]['project'][self.current['architecture']]['releases']) + # old releases.json without canary field + release_canary = self.update_json[release_branch]['canary'] if 'canary' in self.update_json[release_branch] else None + self.parse_device_json(self.update_json[release_branch]['project'][self.current['architecture']]['releases'][device_release], release_canary) + + # Major or minor version update + if self.candidate['version_major'] > self.current['version_major'] or \ + (self.candidate['version_major'] == self.current['version_major'] and self.candidate['version_minor'] > self.current['version_minor']): + + self.candidate['url'] = self.update_json[release_branch]['url'] + self.update_url = f'{self.candidate["url"]}{self.candidate["subpath"]}/{self.candidate["filename"]}' if self.candidate['subpath'] else f'{self.candidate["url"]}{self.candidate["filename"]}' + self.update_available = True + self.update_major = True + return + + + def check_for_update(self): + '''Function to run when invoked by settings addon: checks for bugfix then major updates. + Returns four values: + True/False if update is found + True/False if update is major (11.x -> 12.x) + URL to download update file, if available + sha256sum for update file according to the json + ''' + self.check_for_bugfix() + if self.update_available: + return self.update_available, self.update_major, self.update_url, self.candidate['sha256'] + + self.check_for_major() + if self.update_available and self.update_major: + return self.update_available, self.update_major, self.update_Url, self.candidate['sha256'] + + # Both checks negative; report nothing available. + return False, False, None, None + + + def standalone_update(self): + '''Function to run when being invoked as standalone script: handles update check and downloading update.''' + if args.bugfix: + self.check_for_bugfix() + if self.update_available: + print(f'Found update file: {self.update_url}') + else: + print('No bugfix release found.') + elif args.major: + self.check_for_major() + if self.update_available: + print(f'Found update file: {self.update_url}') + else: + print('No newer release series found.') + else: + self.check_for_bugfix() + if not self.update_available: + self.check_for_major() + if args.verbose: + print(f'{self.update_available=}\n{self.update_major=}\n{self.update_url=}\n{self.candidate["sha256"]=}') + + if self.update_available: + if self.update_major: + print('Major system update found. Check https://libreelec.tv for release notes that may affect this device.') + else: + if args.update: + print(f'Downloading: {self.update_url}') + self.fetch_update_file() + if os.path.isfile(f'{LOCAL_UPDATE_DIR}/{self.candidate["filename"]}'): + print('Download complete. Reboot to continue the update.') + else: + print('Download failed. Please try again later.') + else: + print('System update found. Use --update to apply.') + else: + print('No eligible system updates found to apply.') + + + +if __name__ == '__main__': + # parse CLI arguments + parser = argparse.ArgumentParser(description='Parse releases.json for suitable update files.', argument_default=False) + parser.add_argument('-v', '--verbose', + help = 'Verbose output', action = 'store_true') + parser.add_argument('-b', '--bugfix', + help = 'Check for bugfix updates only (ex 11.0.x -> 11.0.y).', action = 'store_true') + parser.add_argument('-m', '--major', + help = 'Check for major/minor updates only (ex 11.x -> 12.x).', action = 'store_true') + parser.add_argument('-f', '--force', + help = 'Ignore testing periods for updates.', action = 'store_true') + parser.add_argument('-j', '--json', + help = 'http or full file path to an alternative releases.json file.', action = 'store') + parser.add_argument('-u', '--update', + help = 'Update system to latest minor bugfix release.', action = 'store_true') + args = parser.parse_args() + + UpdateSystem().standalone_update() + +else: + # Set default CLI args values when importing + args = argparse.Namespace(verbose=False, bugfix=False, major=False, force=False, json=False, update=False) From 37a139c7bd26bfe17ff072930d30543e981de6e7 Mon Sep 17 00:00:00 2001 From: Ian Leonard Date: Fri, 1 Mar 2024 01:47:18 -0500 Subject: [PATCH 4/9] update-system: allow providing a json instead of downloading Signed-off-by: Ian Leonard --- distributions/LibreELEC/filesystem/usr/bin/update-system | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distributions/LibreELEC/filesystem/usr/bin/update-system b/distributions/LibreELEC/filesystem/usr/bin/update-system index 7d9a526b911..5cafcfd4e35 100755 --- a/distributions/LibreELEC/filesystem/usr/bin/update-system +++ b/distributions/LibreELEC/filesystem/usr/bin/update-system @@ -29,7 +29,7 @@ LOCAL_UPDATE_DIR = '/storage/.update' class UpdateSystem(): '''Functions to check for, obtain, and validate system updates.''' - def __init__(self): + def __init__(self, given_json=None): # information on running system self.current = { 'architecture': None, @@ -54,7 +54,7 @@ class UpdateSystem(): 'version_bugfix': None } # releases.json - self.update_json = None + self.update_json = given_json # update results self.abort_check = False self.update_available = False From d4ff6f3a5d597cddf6deb299b47538f9b74c8a8f Mon Sep 17 00:00:00 2001 From: Ian Leonard Date: Thu, 21 Mar 2024 05:15:09 -0400 Subject: [PATCH 5/9] update-system: remove ability to take lists for releases.json location Signed-off-by: Ian Leonard --- .../filesystem/usr/bin/update-system | 26 +++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/distributions/LibreELEC/filesystem/usr/bin/update-system b/distributions/LibreELEC/filesystem/usr/bin/update-system index 5cafcfd4e35..c487832437e 100755 --- a/distributions/LibreELEC/filesystem/usr/bin/update-system +++ b/distributions/LibreELEC/filesystem/usr/bin/update-system @@ -21,8 +21,8 @@ from datetime import datetime, timedelta from hashlib import sha256 -# List of locations with JSON release data (may be a single string entry) -JSON_DOWNLOAD_URL = ['https://releases.libreelec.tv/releases_v2.json', 'https://releases.libreelec.tv/releases.json'] +# Location of JSON release data +JSON_DOWNLOAD_URL = 'https://releases.libreelec.tv/releases.json' # Where to put update files LOCAL_UPDATE_DIR = '/storage/.update' @@ -105,21 +105,15 @@ class UpdateSystem(): return else: releases_json = JSON_DOWNLOAD_URL - if isinstance(releases_json, list): - for url in releases_json: - try: - data = urllib.request.urlopen(url) - except urllib.error.HTTPError as err: - if err.code == 404: - print(f'ERROR: HTTP 404: Failed to download from: {url}') - continue - else: - print(f'Unhandled HTTPError: {err=}') - raise - else: - break - else: + try: data = urllib.request.urlopen(releases_json) + except urllib.error.HTTPError as err: + if err.code == 404: + print(f'ERROR: HTTP 404: Failed to download from: {url}') + raise + else: + print(f'Unhandled HTTPError: {err=}') + raise self.update_json = json.loads(data.read().decode('utf-8').strip()) if data else None From 48acb5121afc4e7a8fb1770bcb177d001d687f9f Mon Sep 17 00:00:00 2001 From: Ian Leonard Date: Sat, 23 Mar 2024 05:09:20 -0400 Subject: [PATCH 6/9] update-system: add option for updating to latest nightly build Signed-off-by: Ian Leonard --- .../filesystem/usr/bin/update-system | 100 +++++++++++++++--- 1 file changed, 85 insertions(+), 15 deletions(-) diff --git a/distributions/LibreELEC/filesystem/usr/bin/update-system b/distributions/LibreELEC/filesystem/usr/bin/update-system index c487832437e..d0eaaca3c17 100755 --- a/distributions/LibreELEC/filesystem/usr/bin/update-system +++ b/distributions/LibreELEC/filesystem/usr/bin/update-system @@ -22,7 +22,8 @@ from hashlib import sha256 # Location of JSON release data -JSON_DOWNLOAD_URL = 'https://releases.libreelec.tv/releases.json' +STABLE_JSON_URL = 'https://releases.libreelec.tv/releases.json' +TEST_JSON_URL = 'https://test.libreelec.tv/releases.json' # Where to put update files LOCAL_UPDATE_DIR = '/storage/.update' @@ -90,7 +91,12 @@ class UpdateSystem(): # If debugging other devices, change self.current[architecture, distribution, version, version_id] here self.current['version_major'], self.current['version_minor'] = [int(i) for i in self.current['version_id'].split('.')] self.current['version_bugfix'] = int(self.current['version'].split('.')[2]) if not self.current['version'].startswith(('devel', 'nightly')) else None - self.current['timestamp'] = datetime.strptime(self.current['version'].split('-')[1], '%Y%m%d') if 'nightly' in self.current['version'] else None + if self.current['version'].startswith('nightly'): + self.current['timestamp'] = datetime.strptime(self.current['version'].split('-')[1], '%Y%m%d') + elif self.current['version'].startswith('devel'): + self.current['timestamp'] = datetime.strptime(self.current['version'].split('-')[1], '%Y%m%d%H%M%S') + else: + self.current['timestamp'] = None def fetch_update_json(self): @@ -103,8 +109,11 @@ class UpdateSystem(): else: print(f'ERROR: Unable to locate: {args.json}.') return + elif args.nightly: + releases_json = TEST_JSON_URL else: - releases_json = JSON_DOWNLOAD_URL + releases_json = STABLE_JSON_URL + try: data = urllib.request.urlopen(releases_json) except urllib.error.HTTPError as err: @@ -119,19 +128,35 @@ class UpdateSystem(): def parse_device_json(self, device_json, canary=None): '''Parse json fields of a device's release entry.''' - self.candidate['filename'] = device_json['file']['name'] - # Assumes filename format is 'distribution-device.arch-version.tar' - version = tuple(int(i) for i in self.candidate['filename'].split('-')[-1].removesuffix('.tar').split('.')) + if args.nightly: + if 'image' in device_json: # image entry + self.candidate['filename'] = device_json['image']['name'] + self.candidate['sha256'] = device_json['image']['sha256'] + # old releases.json without subpath field + self.candidate['subpath'] = device_json['image']['subpath'] if 'subpath' in device_json['image'] else None + # old releases.json without timestamp field + self.candidate['timestamp'] = datetime.strptime(device_json['image']['timestamp'], '%Y-%m-%d %H:%M:%S') if 'timestamp' in device_json['image'] else None + else: # uboot entry + self.candidate['filename'] = device_json['uboot'][0]['name'] + self.candidate['sha256'] = device_json['uboot'][0]['sha256'] + # old releases.json without subpath field + self.candidate['subpath'] = device_json['uboot'][0]['subpath'] if 'subpath' in device_json['uboot'][0] else None + # old releases.json without timestamp field + self.candidate['timestamp'] = datetime.strptime(device_json['uboot'][0]['timestamp'], '%Y-%m-%d %H:%M:%S') if 'timestamp' in device_json['uboot'][0] else None + else: # file entry + self.candidate['filename'] = device_json['file']['name'] + # Assumes filename format is 'distribution-device.arch-version.tar' + self.candidate['sha256'] = device_json['file']['sha256'] + # old releases.json without subpath field + self.candidate['subpath'] = device_json['file']['subpath'] if 'subpath' in device_json['file'] else None + # old releases.json without timestamp field + self.candidate['timestamp'] = datetime.strptime(device_json['file']['timestamp'], '%Y-%m-%d %H:%M:%S') if 'timestamp' in device_json['file'] else None + version = tuple(int(i) for i in self.candidate['filename'].split('-')[-1].removesuffix('.tar').split('.')) if not 'devel' in self.candidate['filename'] and not 'nightly' in self.candidate['filename'] else (None, None, None) self.candidate['version_major'] = version[0] self.candidate['version_minor'] = version[1] self.candidate['version_bugfix'] = version[2] - self.candidate['sha256'] = device_json['file']['sha256'] - # old releases.json without subpath field - self.candidate['subpath'] = device_json['file']['subpath'] if 'subpath' in device_json['file'] else None - # old releases.json without timestamp field - self.candidate['timestamp'] = datetime.strptime(device_json['file']['timestamp'], '%Y-%m-%d %H:%M:%S') if 'timestamp' in device_json['file'] else None - # set if parts are known - self.candidate['canarydate'] = self.candidate['timestamp'] + timedelta(days=canary) if self.candidate['timestamp'] and canary else None + # set if parts are known and not nightly check + self.candidate['canarydate'] = self.candidate['timestamp'] + timedelta(days=canary) if self.candidate['timestamp'] and canary and not args.nightly else None def fetch_update_file(self): @@ -282,6 +307,37 @@ class UpdateSystem(): return + def check_for_nightly(self): + '''Review test build's releases.json for updates to running release series.''' + self.precheck_update() + if self.abort_check: + return + + release_branch = f'{self.current["distribution"]}-{self.current["version_id"]}' + if release_branch not in self.update_json: + print('Running release branch not in json file.') + self.update_available = False + self.update_major = False + self.update_url = None + return + + device_release = UpdateSystem.get_highest_value(self.update_json[release_branch]['project'][self.current['architecture']]['releases']) + # Parses highest (most recent) release of device within releases.json file + self.parse_device_json(self.update_json[release_branch]['project'][self.current['architecture']]['releases'][device_release]) + + # compare timestamps to determine newer + if self.candidate['timestamp'] and self.current['timestamp'] and self.candidate['timestamp'] > self.current['timestamp']: + self.candidate['url'] = self.update_json[release_branch]['url'] + self.update_url = f'{self.candidate["url"]}{self.candidate["subpath"]}/{self.candidate["filename"]}' if self.candidate['subpath'] else f'{self.candidate["url"]}{self.candidate["filename"]}' + self.update_available = True + return + else: + self.update_available = False + self.update_major = False + self.update_url = None + return + + def check_for_update(self): '''Function to run when invoked by settings addon: checks for bugfix then major updates. Returns four values: @@ -316,12 +372,18 @@ class UpdateSystem(): print(f'Found update file: {self.update_url}') else: print('No newer release series found.') + elif args.nightly: + self.check_for_nightly() + if self.update_available: + print(f'Found update file: {self.update_url}') + else: + print('No newer nightly test build found.') else: self.check_for_bugfix() if not self.update_available: self.check_for_major() if args.verbose: - print(f'{self.update_available=}\n{self.update_major=}\n{self.update_url=}\n{self.candidate["sha256"]=}') + print(f'{self.update_available=}\n{self.update_major=}\n{self.update_url=}') if self.update_available: if self.update_major: @@ -350,6 +412,8 @@ if __name__ == '__main__': help = 'Check for bugfix updates only (ex 11.0.x -> 11.0.y).', action = 'store_true') parser.add_argument('-m', '--major', help = 'Check for major/minor updates only (ex 11.x -> 12.x).', action = 'store_true') + parser.add_argument('-n', '--nightly', + help = 'Check for current nightly test build.', action = 'store_true') parser.add_argument('-f', '--force', help = 'Ignore testing periods for updates.', action = 'store_true') parser.add_argument('-j', '--json', @@ -358,8 +422,14 @@ if __name__ == '__main__': help = 'Update system to latest minor bugfix release.', action = 'store_true') args = parser.parse_args() + # sanity check + if args.nightly and (args.bugfix or args.major): + print('Error: --nightly may not be combined with --bugfix or --major. Assuming --nightly intended.') + args.bugfix = False + args.major = False + UpdateSystem().standalone_update() else: # Set default CLI args values when importing - args = argparse.Namespace(verbose=False, bugfix=False, major=False, force=False, json=False, update=False) + args = argparse.Namespace(verbose=False, bugfix=False, major=False, nightly=False, force=False, json=False, update=False) From b0bd687aa6644debd568abfdef73e36cc28cfa1f Mon Sep 17 00:00:00 2001 From: Ian Leonard Date: Sat, 23 Mar 2024 05:21:40 -0400 Subject: [PATCH 7/9] update-system: permit updating from devel builds Signed-off-by: Ian Leonard --- distributions/LibreELEC/filesystem/usr/bin/update-system | 6 ------ 1 file changed, 6 deletions(-) diff --git a/distributions/LibreELEC/filesystem/usr/bin/update-system b/distributions/LibreELEC/filesystem/usr/bin/update-system index d0eaaca3c17..95b2f4a28b3 100755 --- a/distributions/LibreELEC/filesystem/usr/bin/update-system +++ b/distributions/LibreELEC/filesystem/usr/bin/update-system @@ -210,12 +210,6 @@ class UpdateSystem(): self.update_available = False self.update_major = False return - if self.current['version'].startswith('devel'): - print('ERROR: Automatic updates from development builds are unsupported.') - self.abort_check = True - self.update_available = False - self.update_major = False - return # Retrieve json with release data if not self.update_json: From 9bac12b220026647208eac08f9289560cf0b6b2a Mon Sep 17 00:00:00 2001 From: Ian Leonard Date: Sat, 23 Mar 2024 18:12:10 -0400 Subject: [PATCH 8/9] update-system: misc cleanups Signed-off-by: Ian Leonard --- .../filesystem/usr/bin/update-system | 63 +++++++------------ 1 file changed, 23 insertions(+), 40 deletions(-) diff --git a/distributions/LibreELEC/filesystem/usr/bin/update-system b/distributions/LibreELEC/filesystem/usr/bin/update-system index 95b2f4a28b3..6b89e5a3278 100755 --- a/distributions/LibreELEC/filesystem/usr/bin/update-system +++ b/distributions/LibreELEC/filesystem/usr/bin/update-system @@ -115,15 +115,15 @@ class UpdateSystem(): releases_json = STABLE_JSON_URL try: - data = urllib.request.urlopen(releases_json) + with urllib.request.urlopen(releases_json) as data: + self.update_json = json.loads(data.read().decode('utf-8').strip()) if data else None except urllib.error.HTTPError as err: if err.code == 404: - print(f'ERROR: HTTP 404: Failed to download from: {url}') + print(f'ERROR: HTTP 404: Failed to download from: {releases_json}') raise else: - print(f'Unhandled HTTPError: {err=}') + print(f'ERROR: Unhandled HTTPError: {err=}') raise - self.update_json = json.loads(data.read().decode('utf-8').strip()) if data else None def parse_device_json(self, device_json, canary=None): @@ -199,26 +199,30 @@ class UpdateSystem(): os.remove(f'{update_temp_dir}/update.file') + def abort_update_check(self, msg=None): + '''Reset update result flags to None.''' + print(msg) + self.update_available = False + self.update_major = False + self.update_url = None + + def precheck_update(self): '''Gather and validate information needed for update check.''' self.parse_osrelease() if args.verbose: print(f'{self.current["architecture"]=}\n{self.current["distribution"]=}\n{self.current["version"]=}\n{self.current["version_id"]=}') if not (self.current['architecture'] and self.current['distribution'] and self.current['version'] and self.current['version_id']): - print('ERROR: parse_osrelease failed. Unable to determine running device or version.') + self.abort_update_check('ERROR: parse_osrelease failed. Unable to determine running device or version.') self.abort_check = True - self.update_available = False - self.update_major = False return # Retrieve json with release data if not self.update_json: self.fetch_update_json() if not self.update_json: - print('ERROR: Failed to load json release data.') + self.abort_update_check('ERROR: Failed to load json release data.') self.abort_check = True - self.update_available = False - self.update_major = False return @@ -230,10 +234,7 @@ class UpdateSystem(): release_branch = f'{self.current["distribution"]}-{self.current["version_id"]}' if release_branch not in self.update_json: - print('Running release branch not in json file.') - self.update_available = False - self.update_major = False - self.update_url = None + self.abort_update_check('Running release branch not in json file.') return device_release = UpdateSystem.get_highest_value(self.update_json[release_branch]['project'][self.current['architecture']]['releases']) # Old releases.json without canary field @@ -254,9 +255,7 @@ class UpdateSystem(): print(f'Update held until {self.candidate["canarydate"]} for general availability.') else: print('Unable to determine testing period of update. Holding update back. Use --force to proceed anyway.') - self.update_available = False - self.update_major = False - self.update_url = None + self.abort_update_check() return @@ -309,10 +308,7 @@ class UpdateSystem(): release_branch = f'{self.current["distribution"]}-{self.current["version_id"]}' if release_branch not in self.update_json: - print('Running release branch not in json file.') - self.update_available = False - self.update_major = False - self.update_url = None + self.abort_update_check('Running release branch not in json file.') return device_release = UpdateSystem.get_highest_value(self.update_json[release_branch]['project'][self.current['architecture']]['releases']) @@ -326,9 +322,7 @@ class UpdateSystem(): self.update_available = True return else: - self.update_available = False - self.update_major = False - self.update_url = None + self.abort_update_check() return @@ -346,40 +340,30 @@ class UpdateSystem(): self.check_for_major() if self.update_available and self.update_major: - return self.update_available, self.update_major, self.update_Url, self.candidate['sha256'] + return self.update_available, self.update_major, self.update_url, self.candidate['sha256'] # Both checks negative; report nothing available. return False, False, None, None def standalone_update(self): - '''Function to run when being invoked as standalone script: handles update check and downloading update.''' + '''Function to run when being invoked as standalone script: calls update check and downloads update.''' if args.bugfix: self.check_for_bugfix() - if self.update_available: - print(f'Found update file: {self.update_url}') - else: - print('No bugfix release found.') elif args.major: self.check_for_major() - if self.update_available: - print(f'Found update file: {self.update_url}') - else: - print('No newer release series found.') elif args.nightly: self.check_for_nightly() - if self.update_available: - print(f'Found update file: {self.update_url}') - else: - print('No newer nightly test build found.') else: self.check_for_bugfix() if not self.update_available: self.check_for_major() + if args.verbose: print(f'{self.update_available=}\n{self.update_major=}\n{self.update_url=}') if self.update_available: + print(f'Found update file: {self.update_url}') if self.update_major: print('Major system update found. Check https://libreelec.tv for release notes that may affect this device.') else: @@ -391,12 +375,11 @@ class UpdateSystem(): else: print('Download failed. Please try again later.') else: - print('System update found. Use --update to apply.') + print('System update found. Run command again with --update to apply.') else: print('No eligible system updates found to apply.') - if __name__ == '__main__': # parse CLI arguments parser = argparse.ArgumentParser(description='Parse releases.json for suitable update files.', argument_default=False) From 30547835f96d4d1283989a7fcddaf02e307e3299 Mon Sep 17 00:00:00 2001 From: Ian Leonard Date: Sun, 24 Mar 2024 18:07:24 -0400 Subject: [PATCH 9/9] update-system: remove checks for old releases.json format Signed-off-by: Ian Leonard --- .../LibreELEC/filesystem/usr/bin/update-system | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/distributions/LibreELEC/filesystem/usr/bin/update-system b/distributions/LibreELEC/filesystem/usr/bin/update-system index 6b89e5a3278..b9ec89955fd 100755 --- a/distributions/LibreELEC/filesystem/usr/bin/update-system +++ b/distributions/LibreELEC/filesystem/usr/bin/update-system @@ -132,25 +132,19 @@ class UpdateSystem(): if 'image' in device_json: # image entry self.candidate['filename'] = device_json['image']['name'] self.candidate['sha256'] = device_json['image']['sha256'] - # old releases.json without subpath field - self.candidate['subpath'] = device_json['image']['subpath'] if 'subpath' in device_json['image'] else None - # old releases.json without timestamp field - self.candidate['timestamp'] = datetime.strptime(device_json['image']['timestamp'], '%Y-%m-%d %H:%M:%S') if 'timestamp' in device_json['image'] else None + self.candidate['subpath'] = device_json['image']['subpath'] + self.candidate['timestamp'] = datetime.strptime(device_json['image']['timestamp'], '%Y-%m-%d %H:%M:%S') else: # uboot entry self.candidate['filename'] = device_json['uboot'][0]['name'] self.candidate['sha256'] = device_json['uboot'][0]['sha256'] - # old releases.json without subpath field - self.candidate['subpath'] = device_json['uboot'][0]['subpath'] if 'subpath' in device_json['uboot'][0] else None - # old releases.json without timestamp field - self.candidate['timestamp'] = datetime.strptime(device_json['uboot'][0]['timestamp'], '%Y-%m-%d %H:%M:%S') if 'timestamp' in device_json['uboot'][0] else None + self.candidate['subpath'] = device_json['uboot'][0]['subpath'] + self.candidate['timestamp'] = datetime.strptime(device_json['uboot'][0]['timestamp'], '%Y-%m-%d %H:%M:%S') else: # file entry self.candidate['filename'] = device_json['file']['name'] # Assumes filename format is 'distribution-device.arch-version.tar' self.candidate['sha256'] = device_json['file']['sha256'] - # old releases.json without subpath field - self.candidate['subpath'] = device_json['file']['subpath'] if 'subpath' in device_json['file'] else None - # old releases.json without timestamp field - self.candidate['timestamp'] = datetime.strptime(device_json['file']['timestamp'], '%Y-%m-%d %H:%M:%S') if 'timestamp' in device_json['file'] else None + self.candidate['subpath'] = device_json['file']['subpath'] + self.candidate['timestamp'] = datetime.strptime(device_json['file']['timestamp'], '%Y-%m-%d %H:%M:%S') version = tuple(int(i) for i in self.candidate['filename'].split('-')[-1].removesuffix('.tar').split('.')) if not 'devel' in self.candidate['filename'] and not 'nightly' in self.candidate['filename'] else (None, None, None) self.candidate['version_major'] = version[0] self.candidate['version_minor'] = version[1]