diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 851f48537..21ef02b7a 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -2,7 +2,7 @@ name: Helm Chart CI on: workflow_call: pull_request: - types: ['opened', 'edited', 'reopened', 'synchronize'] + types: [ 'opened', 'edited', 'reopened', 'synchronize' ] env: CARGO_TERM_COLOR: always @@ -24,10 +24,8 @@ jobs: echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV nix-shell --run "echo" ./scripts/helm/shell.nix - name: HelmChart publish test - run: | - nix-shell --pure --run "./scripts/helm/test-publish-chart-yaml.sh" ./scripts/helm/shell.nix + run: nix-shell --pure --run "./scripts/helm/test-publish-chart-yaml.sh" ./scripts/helm/shell.nix - name: HelmChart Readme run: nix-shell --run "./scripts/helm/generate-readme.sh" ./scripts/helm/shell.nix - name: HelmChart Template run: nix-shell --pure --run "./scripts/helm/test-template.sh" ./scripts/helm/shell.nix - diff --git a/.github/workflows/image-pr.yml b/.github/workflows/image-pr.yml index 7d4ffb991..503874b76 100644 --- a/.github/workflows/image-pr.yml +++ b/.github/workflows/image-pr.yml @@ -17,7 +17,6 @@ jobs: - uses: DeterminateSystems/nix-installer-action@v14 - uses: DeterminateSystems/magic-nix-cache-action@v8 - name: Test building the release images - run: ./scripts/release.sh --skip-publish --build-bins + run: ./scripts/release.sh --skip-publish --build-bins --debug - name: Test building the static binaries - run: nix-build -A utils.release.x86_64.linux-musl.kubectl-plugin --arg incremental false - + run: nix-build -A utils.debug.x86_64.linux-musl.kubectl-plugin --arg incremental false diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index b5af768d3..0a655920e 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -28,4 +28,3 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push the release images run: ./scripts/release.sh - diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml index f642e5b9a..368819cce 100644 --- a/.github/workflows/k8s-ci.yml +++ b/.github/workflows/k8s-ci.yml @@ -4,7 +4,7 @@ on: jobs: k8s-ci: - runs-on: ubuntu-latest + runs-on: ubuntu-latest-16-cores steps: - name: Bind mount /dev/sda1 to /nix run: | @@ -18,39 +18,52 @@ jobs: with: kvm: true - uses: DeterminateSystems/magic-nix-cache-action@v6 - - name: Pre-populate nix-shell + - name: Setup Nix Path run: | export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r) echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV - nix-shell ./scripts/k8s/shell.nix --run "echo" - - name: Build binaries and images - id: build - run: | - TAG=$(nix-shell ./shell.nix --run './scripts/python/generate-test-tag.sh') - TEST_DIR=$(realpath $(mktemp -d ./test-dir-XXXXXX)) - nix-shell ./shell.nix --run "./scripts/python/tag-chart.sh $TAG" - RUSTFLAGS="-C debuginfo=0 -C strip=debuginfo" ./scripts/release.sh --tag $TAG --build-binary-out $TEST_DIR --no-static-linking --skip-publish --debug - echo "tag=$TAG" >> $GITHUB_OUTPUT - echo "bin=$TEST_DIR" >> $GITHUB_OUTPUT + - name: Pre-populate K8s nix-shell + run: nix-shell ./scripts/k8s/shell.nix --run "echo" + - name: Pre-populate helm nix-shell + run: nix-shell ./scripts/helm/shell.nix --run "echo" + - name: Pre-populate pytest nix-shell + run: nix-shell ./scripts/python/shell.nix --run "echo" - name: BootStrap k8s cluster + run: nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label" + - name: Prepare v-next images and binary + run: nix-shell ./scripts/python/shell.nix --run "./scripts/python/upgrade-test-helper.sh --build --chart-tag --chart ./chart" + - name: Load images to kind cluster + run: nix-shell ./scripts/python/shell.nix --run "./scripts/python/upgrade-test-helper.sh --load" + - name: Run pytest run: | - nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label" - - name: Load images to Kind cluster - run: nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/load-images-to-kind.sh --tag ${{ steps.build.outputs.tag }} --trim-debug-suffix" - - name: Run Pytests - run: | - export UPGRADE_TARGET_VERSION=${{ steps.build.outputs.tag }} - export TEST_DIR=${{ steps.build.outputs.bin }} - nix-shell ./shell.nix --run "./scripts/python/test.sh" - - name: The job has failed + export REUSE_CLUSTER=1 + export CHART_VNEXT_SKIP=1 + export CLEAN=0 + nix-shell ./scripts/python/shell.nix --run "./scripts/python/test.sh" + - name: Collect logs if: ${{ failure() }} run: | nix-shell ./scripts/k8s/shell.nix --run "kubectl get pods -A -o wide" nix-shell ./scripts/k8s/shell.nix --run "kubectl -n mayastor logs -l openebs.io/release=mayastor --all-containers=true" nix-shell ./scripts/k8s/shell.nix --run "kubectl -n mayastor logs -l app=upgrade --all-containers=true" + - name: Upload pytest logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: pytest.log + path: pytest.log + - name: Report pytest results + if: always() + uses: pmeier/pytest-results-action@main + with: + path: report.xml + summary: true + display-options: a + fail-on-empty: true + title: Test results k8s-ci-vm: - runs-on: ubuntu-latest + runs-on: ubuntu-latest-16-cores steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@v11 @@ -61,8 +74,8 @@ jobs: run: | export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r) echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV - nix-shell ./scripts/k8s/shell.nix --run "echo" + nix-shell ./scripts/helm/shell.nix --run "echo" - name: Test on VM run: | - nix-shell ./scripts/k8s/shell.nix --run "cd chart; helm dependency update" + nix-shell ./scripts/helm/shell.nix --run "cd chart; helm dependency update" nix-build ./tests/helm/test.nix --option sandbox false diff --git a/.github/workflows/release-chart.yml b/.github/workflows/release-chart.yml index fd811c47e..f25270925 100644 --- a/.github/workflows/release-chart.yml +++ b/.github/workflows/release-chart.yml @@ -55,7 +55,13 @@ jobs: run: | tag="${{ github.ref_name }}" BASE_REF="${{ github.event.base_ref }}" - BRANCH="${BASE_REF#refs/heads/}" + if [ -n "$BASE_REF" ]; then + BRANCH="${BASE_REF#refs/heads/}" + else + BRANCH="$(nix-shell --pure --run "./scripts/helm/find-released-branch.sh "$tag"" ./scripts/helm/shell.nix)" + fi + echo "BASE_BRANCH=$BRANCH" >> $GITHUB_ENV + nix-shell --pure --run "./scripts/helm/publish-chart-yaml.sh --released "$tag" --released-branch "$BRANCH"" ./scripts/helm/shell.nix nix-shell --pure --run "SKIP_GIT=1 ./scripts/helm/generate-readme.sh" ./scripts/helm/shell.nix - name: Create Pull Request @@ -73,7 +79,7 @@ jobs: signoff: true delete-branch: true branch-suffix: "random" - base: ${{ github.event.base_ref }} + base: ${{ env.BASE_BRANCH }} token: ${{ secrets.ORG_CI_GITHUB }} - name: Approve Pull Request by CI Bot if: ${{ steps.cpr.outputs.pull-request-number }} diff --git a/.gitignore b/.gitignore index 6398c18f6..8e0cd2f42 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,6 @@ __pycache__ # Pytest assets /test-dir-* tests/bdd/venv -pytest.log \ No newline at end of file +pytest.log +#/tests/bdd/chart-vnext/ +/chart/kubectl-plugin diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e18326cae..8c6f9aa59 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,6 +24,13 @@ repos: pass_filenames: false types: [file, rust] language: system + - id: python-fmt + name: Python fmt + description: Run python fmt on files included in the commit. + entry: nix-shell --pure --run './scripts/python/fmt.sh' + pass_filenames: false + types: [file, python] + language: system - id: commit-lint name: Commit Lint description: Runs commitlint against the commit message. diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index d9746c57c..000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/env groovy - -// Searches previous builds to find first non aborted one -def getLastNonAbortedBuild(build) { - if (build == null) { - return null; - } - - if(build.result.toString().equals("ABORTED")) { - return getLastNonAbortedBuild(build.getPreviousBuild()); - } else { - return build; - } -} - -def mainBranches() { - return BRANCH_NAME == "develop" || BRANCH_NAME.startsWith("release/"); -} - -def cronSchedule() { - node { - if ( env.CRON_SCHEDULE_URL == null ) { - println "ERROR: No cron schedule url in the environment" - return "" - } - println "DEBUG: Fetching Cron Schedule from ${env.CRON_SCHEDULE_URL}" - def YAML = sh (script: "curl ${env.CRON_SCHEDULE_URL}", returnStdout: true).trim() - def schedules = readYaml text: YAML - def branch_cfg = schedules[BRANCH_NAME] ? schedules[BRANCH_NAME] : schedules["default"] - if ( branch_cfg == null ) { - println "ERROR: Failed to retrieve the cron schedule for the branch: ${BRANCH_NAME}" - return "" - } - - def job_name_split = env.JOB_NAME.tokenize('/') as String[]; - def project_name = job_name_split[0] - if ( project_name == null ) { - println "ERROR: No project name for: ${env.JOB_NAME}" - return "" - } - - def project_cfg = branch_cfg[project_name] ? branch_cfg[project_name] : branch_cfg["default"] - if ( project_cfg == null ) { - println "ERROR: No cron schedule for: ${project_name}" - return "" - } - - println "INFO: Cron Schedule for ${project_name}/${BRANCH_NAME}: ${project_cfg}" - return project_cfg - } -} -def dockerId() { - script { - dockerId = sh (script: "./dependencies/control-plane/utils/dependencies/scripts/git-org-name.sh --case upper", returnStdout: true).trim() + "_DOCKERHUB" - println "Using docker id: ${dockerId}" - return dockerId - } -} - -// TODO: Use multiple choices -run_linter = true -rust_test = true -helm_test = true -run_tests = params.run_tests -build_images = params.build_images - -// Will skip steps for cases when we don't want to build -if (currentBuild.getBuildCauses('jenkins.branch.BranchIndexingCause') && mainBranches()) { - print "INFO: Branch Indexing, skip tests and push the new images." - run_tests = false - build_images = true -} - -pipeline { - agent none - options { - timeout(time: 2, unit: 'HOURS') - } - parameters { - booleanParam(defaultValue: false, name: 'build_images') - booleanParam(defaultValue: true, name: 'run_tests') - } - triggers { - cron(cronSchedule()) - } - - stages { - stage('init') { - agent { label 'nixos-mayastor' } - steps { - step([ - $class: 'GitHubSetCommitStatusBuilder', - contextSource: [ - $class: 'ManuallyEnteredCommitContextSource', - context: 'continuous-integration/jenkins/branch' - ], - statusMessage: [ content: 'Pipeline started' ] - ]) - } - } - stage('linter') { - agent { label 'nixos-mayastor' } - when { - beforeAgent true - not { - anyOf { - branch 'master' - branch 'release/*' - expression { run_linter == false } - } - } - } - steps { - sh 'printenv' - sh 'nix-shell --run "./dependencies/control-plane/scripts/rust/generate-openapi-bindings.sh"' - sh 'nix-shell --run "./scripts/rust/linter.sh"' - sh 'nix-shell --run "./scripts/git/check-submodule-branches.sh"' - } - } - stage('test') { - when { - beforeAgent true - not { - anyOf { - branch 'master' - } - } - expression { run_tests == true } - } - parallel { - stage('rust unit tests') { - when{ - expression { rust_test == true } - } - agent { label 'nixos-mayastor' } - steps { - withCredentials([usernamePassword(credentialsId: dockerId(), usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh 'echo $PASSWORD | docker login -u $USERNAME --password-stdin' - } - sh 'printenv' - sh 'nix-shell --run "cargo test"' - } - } - stage('chart publish test') { - when { - expression { helm_test == true } - } - agent { label 'nixos-mayastor' } - steps { - sh 'printenv' - sh 'nix-shell --pure --run "./scripts/helm/test-publish-chart-yaml.sh" ./scripts/helm/shell.nix' - } - } - stage('chart doc test') { - when { - expression { helm_test == true } - } - agent { label 'nixos-mayastor' } - steps { - sh 'printenv' - sh 'nix-shell --pure --run "./scripts/helm/generate-readme.sh" ./scripts/helm/shell.nix' - } - } - stage('chart template and install test') { - when { - expression { helm_test == true } - } - agent { label 'nixos-mayastor' } - steps { - sh 'printenv' - sh 'nix-shell --pure --run "./scripts/helm/test-template.sh" ./scripts/helm/shell.nix' - sh 'nix-shell --pure --run "./scripts/k8s/deployer.sh start --label" ./scripts/k8s/shell.nix' - sh 'nix-shell --pure --run "./scripts/helm/install.sh --wait" ./scripts/k8s/shell.nix' - } - post { - failure { - sh 'nix-shell --pure --run "kubectl get pods -A -o wide" ./scripts/k8s/shell.nix' - sh 'nix-shell --pure --run "kubectl -n mayastor logs -lopenebs.io/release=mayastor --all-containers=true" ./scripts/k8s/shell.nix' - } - always { - sh 'nix-shell --pure --run "./scripts/k8s/deployer.sh stop" ./scripts/k8s/shell.nix' - } - } - } - stage('image build test') { - when { - branch 'staging' - } - agent { label 'nixos-mayastor' } - steps { - sh 'printenv' - sh './scripts/nix/git-submodule-init.sh --force' - sh './scripts/release.sh --skip-publish --debug --build-bins' - } - } - }// parallel stages block - }// end of test stage - stage('build and push images') { - agent { label 'nixos-mayastor' } - when { - beforeAgent true - anyOf { - expression { build_images == true } - anyOf { - branch 'master' - branch 'release/*' - branch 'develop' - } - } - } - steps { - withCredentials([usernamePassword(credentialsId: dockerId(), usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh 'echo $PASSWORD | docker login -u $USERNAME --password-stdin' - } - sh 'printenv' - sh './scripts/nix/reclaim-space.sh 10' - sh './scripts/nix/git-submodule-init.sh --force' // (Jenkins now does the submodule update - may remove?) - sh './scripts/release.sh' - } - post { - always { - sh 'docker image prune --all --force' - } - } - } - } - - // The main motivation for post block is that if all stages were skipped - // (which happens when running cron job and branch != develop) then we don't - // want to set commit status in github (jenkins will implicitly set it to - // success). - post { - always { - node(null) { - script { - // If no tests were run then we should neither be updating commit - // status in github nor send any slack messages - if (currentBuild.result != null) { - step([ - $class : 'GitHubCommitStatusSetter', - errorHandlers : [[$class: "ChangingBuildStatusErrorHandler", result: "UNSTABLE"]], - contextSource : [ - $class : 'ManuallyEnteredCommitContextSource', - context: 'continuous-integration/jenkins/branch' - ], - statusResultSource: [ - $class : 'ConditionalStatusResultSource', - results: [ - [$class: 'AnyBuildResult', message: 'Pipeline result', state: currentBuild.getResult()] - ] - ] - ]) - } - } - } - } - } -} diff --git a/chart/.helmignore b/chart/.helmignore index d3403136e..eaba7b26b 100644 --- a/chart/.helmignore +++ b/chart/.helmignore @@ -24,3 +24,4 @@ README.md.tmpl # Nix Shell *.nix +kubectl-plugin/ \ No newline at end of file diff --git a/chart/shell.nix b/chart/shell.nix index bc228e1fd..6df345629 100644 --- a/chart/shell.nix +++ b/chart/shell.nix @@ -1,16 +1,10 @@ -{}: -let - sources = import ../nix/sources.nix; - pkgs = import sources.nixpkgs { - overlays = [ (_: _: { inherit sources; }) (import ../nix/overlay.nix { }) ]; - }; -in -with pkgs; -let -in -mkShell { +{ pkgs ? import (import ../nix/sources.nix).nixpkgs { + overlays = [ (_: _: { inherit (import ../nix/sources.nix); }) (import ../nix/overlay.nix { }) ]; + } +}: +pkgs.mkShell { name = "helm-scripts-shell"; - buildInputs = [ + buildInputs = with pkgs; [ coreutils git helm-docs diff --git a/nix/pkgs/images/default.nix b/nix/pkgs/images/default.nix index f614faeeb..02a033dda 100644 --- a/nix/pkgs/images/default.nix +++ b/nix/pkgs/images/default.nix @@ -26,7 +26,7 @@ let } // config; }; build-exporter-image = { buildType }: { - io-engine = build-extensions-image rec{ + io-engine = build-extensions-image rec { inherit buildType; package = extensions.${buildType}.metrics.exporter.io-engine; pname = package.pname; @@ -49,6 +49,9 @@ let patchShebangs build/scripts/helm/publish-chart-yaml.sh patchShebangs build/scripts/helm/generate-consolidated-values.sh patchShebangs build/scripts/utils/log.sh + if [ -L build/chart/kubectl-plugin ]; then + rm build/chart/kubectl-plugin + fi # if tag is not semver just keep whatever is checked-in # todo: handle this properly? diff --git a/pytest.ini b/pytest.ini index f9405c8c8..706787d0d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,3 +5,4 @@ log_cli_format = %(asctime)s [%(levelname)s] %(message)s log_cli_date_format = %Y-%m-%d %H:%M:%S log_file = pytest.log log_file_level = DEBUG +addopts = --log-disable kubernetes.client.rest diff --git a/scripts/helm/find-released-branch.sh b/scripts/helm/find-released-branch.sh new file mode 100755 index 000000000..2a65e541c --- /dev/null +++ b/scripts/helm/find-released-branch.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +SCRIPTDIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" +ROOTDIR="$SCRIPTDIR/../.." + +source "$ROOTDIR/scripts/utils/log.sh" + +set -euo pipefail + +# Check if the given branch matches the tag +# Example: +# release/2.7 matches tag v2.7.2, v2.7.2-rc.4, etc.. +# release/2.7 does not match tag v2.6.0, etc... +tag_matches_branch() { + local tag="${1#v}" + local release_branch="$2" + + branch_version="${release_branch#release/}" + if ! [[ "$branch_version" =~ ^[0-9]+.[0-9]+$ ]]; then + return 1 + fi + + if ! [[ "$tag" = "$branch_version"* ]]; then + return 1 + fi +} + +# For the given tag, find the branch which is compatible +# See tag_matches_branch for more information. +find_released_branch() { + local tag="$1" + local branches + branches=$(git branch -r --contains "$TAG" --points-at "$TAG" --format "%(refname:short)" 2>/dev/null) + local branch="" + + for release_branch in $branches; do + release_branch=${release_branch#origin/} + if tag_matches_branch "$TAG" "$release_branch"; then + if [ -n "$branch" ]; then + log_fatal "Multiple branches matched!" + fi + branch="$release_branch" + fi + done + + echo "$branch" +} + +TAG="$1" +BRANCH="$(find_released_branch "$TAG")" + +if [ -z "$BRANCH" ]; then + log_fatal "Failed to find matching released branch for tag '$TAG'" +fi + +echo "$BRANCH" diff --git a/scripts/helm/install.sh b/scripts/helm/install.sh index dd1ae4b30..52c3be3c3 100755 --- a/scripts/helm/install.sh +++ b/scripts/helm/install.sh @@ -7,14 +7,14 @@ repo_add() { local -r preferred_name=$2 local repo - if [ "$(helm repo ls -o yaml | yq "contains([{\"url\": \"$url\"}])")" = "true" ]; then - repo=$(helm repo ls -o yaml | yq ".[] | select(.url == \"$url\") | .name") + if [ -z "$DRY_RUN" ] && [ "$($HELM repo ls -o yaml | yq "contains([{\"url\": \"$url\"}])")" = "true" ]; then + repo=$($HELM repo ls -o yaml | yq ".[] | select(.url == \"$url\") | .name") else - helm repo add "$preferred_name" "$url" > /dev/null + $HELM repo add "$preferred_name" "$url" > /dev/null repo=$preferred_name fi - helm repo update > /dev/null || true + $HELM repo update > /dev/null || true echo "$repo" } @@ -22,8 +22,8 @@ repo_add() { TIMEOUT="5m" WAIT= -DRY_RUN="" -CHART= +DRY_RUN= +HELM_DRY_RUN="" SCRIPT_DIR="$(dirname "$0")" CHART_DIR="$SCRIPT_DIR"/../../chart CHART_SOURCE=$CHART_DIR @@ -35,6 +35,8 @@ HOSTED= VERSION= REGISTRY= DEFAULT_REGISTRY="https://openebs.github.io/mayastor-extensions" +HELM="helm" +KUBECTL="kubectl" help() { cat < How long to wait for helm to complete install (Default: $TIMEOUT). --wait Wait for helm to complete install. - --dry-run Install helm with --dry-run. + --dry-run Don't run any commands, output them only. + --helm-dry-run Install helm with --dry-run. --dep-update Run helm dependency update. --fail-if-installed Fail with a status code 1 if the helm release '$RELEASE_NAME' already exists in the $K8S_NAMESPACE namespace. --hosted-chart Install a hosted chart instead of the local chart. @@ -80,8 +83,13 @@ while [ "$#" -gt 0 ]; do --wait) WAIT="yes" shift;; + --helm-dry-run) + HELM_DRY_RUN=" --dry-run" + shift;; --dry-run) - DRY_RUN=" --dry-run" + DRY_RUN="yes" + HELM="echo $HELM" + KUBECTL="echo $KUBECTL" shift;; --dep-update) DEP_UPDATE="y" @@ -137,8 +145,8 @@ if [ -n "$HOSTED" ]; then DEP_UPDATE_ARG= fi -if [ "$(helm ls -n "$K8S_NAMESPACE" -o yaml | yq "contains([{\"name\": \"$RELEASE_NAME\"}])")" = "true" ]; then - already_exists_log= "Helm release $RELEASE_NAME already exists in namespace $K8S_NAMESPACE" +if [ -z "$DRY_RUN" ] && [ "$($HELM ls -n "$K8S_NAMESPACE" -o yaml | yq "contains([{\"name\": \"$RELEASE_NAME\"}])")" = "true" ]; then + already_exists_log="Helm release $RELEASE_NAME already exists in namespace $K8S_NAMESPACE" if [ -n "$FAIL_IF_INSTALLED" ]; then die "ERROR: $already_exists_log" 1 fi @@ -146,12 +154,12 @@ if [ "$(helm ls -n "$K8S_NAMESPACE" -o yaml | yq "contains([{\"name\": \"$RELEAS else echo "Installing Mayastor Chart" set -x - helm install "$RELEASE_NAME" "$CHART_SOURCE" -n "$K8S_NAMESPACE" --create-namespace \ + $HELM install "$RELEASE_NAME" "$CHART_SOURCE" -n "$K8S_NAMESPACE" --create-namespace \ --set="etcd.livenessProbe.initialDelaySeconds=5,etcd.readinessProbe.initialDelaySeconds=5,etcd.replicaCount=1" \ --set="obs.callhome.enabled=true,obs.callhome.sendReport=false,localpv-provisioner.analytics.enabled=false" \ --set="eventing.enabled=false" \ - $DRY_RUN $WAIT_ARG $DEP_UPDATE_ARG $VERSION_ARG + $HELM_DRY_RUN $WAIT_ARG $DEP_UPDATE_ARG $VERSION_ARG set +x fi -kubectl get pods -n "$K8S_NAMESPACE" -o wide +$KUBECTL get pods -n "$K8S_NAMESPACE" -o wide diff --git a/scripts/k8s/deployer.sh b/scripts/k8s/deployer.sh index a08bc423a..c97bbff2a 100755 --- a/scripts/k8s/deployer.sh +++ b/scripts/k8s/deployer.sh @@ -16,6 +16,7 @@ KUBECTL="kubectl" DOCKER="docker" HUGE_PAGES=1800 LABEL= +CLEANUP="false" SUDO=${SUDO:-"sudo"} help() { @@ -31,6 +32,7 @@ Options: --dry-run Don't do anything, just output steps. --hugepages Add 2MiB hugepages (Default: $HUGE_PAGES). --label Label worker nodes with the io-engine selector. + --cleanup Prior to starting, stops the running instance of the deployer. Command: start Start the k8s cluster. @@ -90,6 +92,9 @@ while [ "$#" -gt 0 ]; do --label) LABEL="true" shift;; + --cleanup) + CLEANUP="true" + shift;; --hugepages) shift test $# -lt 1 && die "Missing hugepage number" @@ -115,12 +120,14 @@ if [ -z "$COMMAND" ]; then die "No command specified!" fi -if [ "$COMMAND" = "stop" ]; then - if command -v nvme 2>dev/null; then +if [ "$COMMAND" = "stop" ] || [ "$CLEANUP" = "true" ]; then + if command -v nvme &>/dev/null; then $SUDO nvme disconnect-all fi $KIND delete cluster - exit 0 + if [ "$COMMAND" = "stop" ]; then + exit 0 + fi fi "$SCRIPT_DIR"/setup-io-prereq.sh --hugepages "$HUGE_PAGES" --nvme-tcp $DRY_RUN diff --git a/scripts/k8s/setup-io-prereq.sh b/scripts/k8s/setup-io-prereq.sh index 6bba221d8..b1fb60df9 100755 --- a/scripts/k8s/setup-io-prereq.sh +++ b/scripts/k8s/setup-io-prereq.sh @@ -35,7 +35,13 @@ die() { } setup_hugepages() { - $SYSCTL -w vm.nr_hugepages="$1" + local wanted="$1" + $SYSCTL -w vm.nr_hugepages="$wanted" + local actual + actual=$($SYSCTL -b vm.nr_hugepages) + if [ "$actual" != "$wanted" ]; then + die "Wanted $wanted hugepages, but only allocated $actual hugepages" + fi } modprobe_nvme_tcp() { @@ -99,7 +105,7 @@ while [ "$#" -gt 0 ]; do done if [ -n "$HUGE_PAGES" ]; then - pages=$(sysctl -b vm.nr_hugepages) + pages=$($SYSCTL -b vm.nr_hugepages) if [ "$HUGE_PAGES" -gt "$pages" ]; then setup_hugepages "$HUGE_PAGES" diff --git a/scripts/k8s/shell.nix b/scripts/k8s/shell.nix index a7bb34f55..c129b5f86 100644 --- a/scripts/k8s/shell.nix +++ b/scripts/k8s/shell.nix @@ -1,8 +1,8 @@ +{ pkgs ? import (import ../../nix/sources.nix).nixpkgs { + overlays = [ (_: _: { inherit (import ../../nix/sources.nix); }) (import ../../nix/overlay.nix { }) ]; + } +}: let - sources = import ../../nix/sources.nix; - pkgs = import sources.nixpkgs { - overlays = [ (_: _: { inherit sources; }) (import ../../nix/overlay.nix { }) ]; - }; inPureNixShell = builtins.getEnv "IN_NIX_SHELL" == "pure"; in pkgs.mkShell { diff --git a/scripts/python/shell.nix b/scripts/python/shell.nix new file mode 120000 index 000000000..7bef29a76 --- /dev/null +++ b/scripts/python/shell.nix @@ -0,0 +1 @@ +../../tests/bdd/shell.nix \ No newline at end of file diff --git a/scripts/python/tag-chart.sh b/scripts/python/tag-chart.sh index 63dd74bff..4d158abf6 100755 --- a/scripts/python/tag-chart.sh +++ b/scripts/python/tag-chart.sh @@ -15,7 +15,7 @@ fi CHART_VERSION=${1#v} IMAGE_TAG="v$CHART_VERSION" -CHART_DIR="$ROOT_DIR/chart" +CHART_DIR=${CHART_DIR:-"$ROOT_DIR/chart"} # TODO: tests should work with its own copy of the chart. Shouldn't modify the chart. # chart/Chart.yaml yq_ibl " diff --git a/scripts/python/test.sh b/scripts/python/test.sh index dcdc88e64..824cfed58 100755 --- a/scripts/python/test.sh +++ b/scripts/python/test.sh @@ -2,20 +2,13 @@ SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" ROOT_DIR="$SCRIPT_DIR/../.." +REPORT="$ROOT_DIR/report.xml" + # Imports source "$ROOT_DIR/scripts/utils/log.sh" set -e -_pytest() { - pytest "$1" || pytest_return=$? - # Exit code 5 denotes no tests were run, which is something we're ok with. - if [ "$pytest_return" = 5 ]; then - exit 0 - fi - exit $pytest_return -} - # Print usage options for this script. print_help() { cat </dev/null; then + git restore "$CHART_VNEXT" || : + fi + fi + if [ $ERROR != 0 ]; then + exit $ERROR + fi +} + +# Print usage options for this script. +print_help() { + cat < The tag for the vnext chart. + Default: Automatically figured out. + --chart The path where the chart will be copied to and modified for the vnext tag. + Default: \$workspace_root/tests/python/chart_vnext. + --fork Forks the vnext chart from "$CHART" into CHART_VNEXT. + --chart-tag Tag the vnext chart. + --build Builds the container images and the plugin binary. + --load Loads the images into the kind cluster. + +Examples: + $(basename "${0}") --fork + +The kubectl-mayastor binary will be built at CHART_VNEXT/kubectl-plugin/bin/kubectl-mayastor +EOF +} + +# Parse args. +while test $# -gt 0; do + arg="$1" + case "$arg" in + --tag) + shift + TAG="$1" + ;; + --chart) + shift + CHART_VNEXT="$1" + ;; + --fork) + CHART_FORK="true" + ;; + --chart-tag) + CHART_TAG="true" + ;; + --build) + IMAGE_BUILD="true" + ;; + --load) + IMAGE_LOAD="true" + ;; + -h* | --help*) + print_help + exit 0 + ;; + *) + print_help + log_fatal "unexpected argument '$arg'" 1 + ;; + esac + shift +done + +if [ "$(kubectl config current-context)" != "kind-kind" ]; then + log_fatal "Only Supported on Kind Clusters!" +fi + +if [ -z "$TAG" ]; then + TAG="$("$SCRIPT_DIR"/generate-test-tag.sh)" +fi + +if [ -z "$CHART_VNEXT" ]; then + CHART_VNEXT="$ROOT_DIR/tests/bdd/chart-vnext" +fi +KUBECTL_MAYASTOR="$CHART_VNEXT/kubectl-plugin/bin/kubectl-mayastor" + +# Ensure the chart vnext is created, copied from the original +if [ "$CHART_FORK" = "true" ]; then + mkdir -p "$CHART_VNEXT" + rm -r "${CHART_VNEXT:?}"/* + cp -r "$CHART/." "${CHART_VNEXT:?}" +fi + +if [ "$CHART_TAG" = "true" ]; then + # Tag the vnext chart + CHART_DIR="$CHART_VNEXT" "$SCRIPT_DIR"/tag-chart.sh "$TAG" + trap cleanup_handler INT QUIT TERM HUP EXIT +fi + +# Build the vnext images and kubectl-binary (in debug mode) +if [ "$IMAGE_BUILD" = "true" ]; then + RUSTFLAGS="-C debuginfo=0 -C strip=debuginfo" "$ROOT_DIR"/scripts/release.sh --tag "$TAG" --build-binary-out "$CHART_VNEXT" --no-static-linking --skip-publish --debug + + # Ensure binary is on the correct version + PLUGIN_VERSION="$($KUBECTL_MAYASTOR --version)" + if [[ ! "$PLUGIN_VERSION" =~ ^Kubectl\ Plugin\ \(kubectl-mayastor\).*\($TAG\+0\)$ ]]; then + log_fatal "The built kubectl-plugin reports version $PLUGIN_VERSION but we want $TAG" + fi +fi + +cleanup_handler + +# Load the images into the kind cluster +if [ "$IMAGE_LOAD" = "true" ]; then + "$ROOT_DIR"/scripts/k8s/load-images-to-kind.sh --tag "$TAG" --trim-debug-suffix +fi diff --git a/scripts/release.sh b/scripts/release.sh index 5d2dc722d..d76737cda 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -17,4 +17,11 @@ BUILD_BINARIES="kubectl-plugin" PROJECT="extensions" . "$SOURCE_REL" +# Sadly helm ignore does not work on symlinks: https://github.com/helm/helm/issues/13284 +# So we must cleanup to ensure the upgrade image is built correctly +CHART_DIR="$(dirname "$0")/../chart" +if [ -L "$CHART_DIR"/kubectl-plugin ]; then + rm "$CHART_DIR"/kubectl-plugin +fi + common_run $@ diff --git a/shell.nix b/shell.nix index dafbd1ea3..c8b2b830f 100644 --- a/shell.nix +++ b/shell.nix @@ -4,8 +4,8 @@ let pkgs = import sources.nixpkgs { overlays = [ (_: _: { inherit sources; }) (import ./nix/overlay.nix { }) (import sources.rust-overlay) ]; }; + lib = pkgs.lib; in -with pkgs; let norust_moth = "You have requested an environment without rust, you should provide it!"; @@ -14,12 +14,10 @@ let channel = import ./nix/lib/rust.nix { inherit pkgs; }; rust_chan = channel.default_src; rust = rust_chan.${rust-profile}; -in -mkShell { - name = "extensions-shell"; - buildInputs = [ - autoflake - black + k8sShellAttrs = import ./scripts/k8s/shell.nix { inherit pkgs; }; + helmShellAttrs = import ./chart/shell.nix { inherit pkgs; }; + bddShellAttrs = import ./tests/bdd/shell.nix { inherit pkgs; }; + buildInputs = with pkgs; [ cacert cargo-expand cargo-udeps @@ -28,10 +26,6 @@ mkShell { coreutils cowsay git - helm-docs - isort - kubectl - kubernetes-helm-wrapped llvmPackages.libclang niv nixpkgs-fmt @@ -39,19 +33,18 @@ mkShell { openssl pkg-config pre-commit - python3 - semver-tool utillinux - virtualenv - which - yq-go - kind - ] ++ pkgs.lib.optional (!norust) rust - ++ pkgs.lib.optional (system == "aarch64-darwin") darwin.apple_sdk.frameworks.Security; + ]; +in +pkgs.mkShell { + name = "extensions-shell"; + buildInputs = buildInputs ++ pkgs.lib.optional (!norust) rust + ++ k8sShellAttrs.buildInputs ++ helmShellAttrs.buildInputs ++ bddShellAttrs.buildInputs + ++ pkgs.lib.optional (pkgs.system == "aarch64-darwin") pkgs.darwin.apple_sdk.frameworks.Security; - PROTOC = "${protobuf}/bin/protoc"; - PROTOC_INCLUDE = "${protobuf}/include"; - NODE_PATH = "${nodePackages."@commitlint/config-conventional"}/lib/node_modules"; + PROTOC = "${pkgs.protobuf}/bin/protoc"; + PROTOC_INCLUDE = "${pkgs.protobuf}/include"; + NODE_PATH = "${pkgs.nodePackages."@commitlint/config-conventional"}/lib/node_modules"; # using the nix rust toolchain USE_NIX_RUST = "${toString (!norust)}"; @@ -71,8 +64,8 @@ mkShell { export CTRL_SRC="$EXTENSIONS_SRC"/dependencies/control-plane export PATH="$PATH:$(pwd)/target/debug" - ${pkgs.lib.optionalString (norust) "cowsay ${norust_moth}"} - ${pkgs.lib.optionalString (norust) "echo"} + ${lib.optionalString (norust) "cowsay ${norust_moth}"} + ${lib.optionalString (norust) "echo"} rust_version="${rust.version}" rustup_channel="${lib.strings.concatMapStringsSep "-" (x: x) (lib.lists.drop 1 (lib.strings.splitString "-" rust.version))}" \ dev_rustup="${toString (devrustup)}" devrustup_moth="${devrustup_moth}" . "$CTRL_SRC"/scripts/rust/env-setup.sh diff --git a/tests/bdd/README.md b/tests/bdd/README.md new file mode 100644 index 000000000..7b478b9ae --- /dev/null +++ b/tests/bdd/README.md @@ -0,0 +1,58 @@ +# BDD Tests + +The BDD tests are written in Python and make use of the pytest-bdd library. + +The feature files in the `features` directory define the behaviour expected of mayastor. These behaviours are +described using the [Gherkin](https://cucumber.io/docs/gherkin/) syntax. + +The feature files can be used to auto-generate the test file. For example +running `pytest-bdd generate upgrade.feature > test_upgrade.py` +generates the `test_upgrade.py` test file from the `upgrade.feature` file. +When updating the feature file, you can also get some helpe updating the python code. +Example: `pytest --generate-missing --feature upgrade.feature test_upgrade.py` + +**:warning: Note: Running pytest-bdd generate will overwrite any existing files with the same name** + +## Running the Tests by entering the python virtual environment + +Before running any tests run the `setup.sh` script. This sets up the necessary environment to run the tests: + +```bash +# NOTE: you should be inside the nix-shell to begin +source ./setup.sh +``` + +To run all the tests: + +```bash +pytest . +``` + +To run individual test files: + +```bash +pytest features/test_upgrade.py +``` + +To run an individual test within a test file use the `-k` option followed by the test name: + +```bash +pytest features/test_upgrade.py -k test_upgrade_to_vnext +``` + +## Running the Tests + +The script in `../../scripts/python/test.sh` can be used to run the tests without entering the venv. +This script will implicitly enter and exit the venv during test execution. + +To run all the tests: + +```bash +../../scripts/python/test.sh +``` + +Arguments will be passed directly to pytest. Example running individual tests: + +```bash +../../scripts/python/test.sh features/test_upgrade.py -k test_upgrade_to_vnext +``` diff --git a/tests/bdd/common/__init__.py b/tests/bdd/common/__init__.py new file mode 100644 index 000000000..84de90b5d --- /dev/null +++ b/tests/bdd/common/__init__.py @@ -0,0 +1,70 @@ +import logging +import os +import subprocess + +logger = logging.getLogger(__name__) + + +def root_dir(): + file_path = os.path.abspath(__file__) + return file_path.split("tests/bdd")[0] + + +def chart_vnext(): + vnext = os.getenv("CHART_VNEXT") + if vnext is not None: + return vnext + # return os.path.join(root_dir(), "./tests/bdd/chart-vnext") + return os.path.join(root_dir(), "./chart") + + +def chart_vnext_skip(): + skip = os.getenv("CHART_VNEXT_SKIP") + if skip is not None and skip.lower() in ("yes", "true", "1", "0"): + return True + return False + + +def run( + command: str, + args: list[str] = None, + absolute=False, + capture_output=True, + log_run=True, + **kwargs, +): + if absolute: + command = [command] + else: + command = [os.path.join(root_dir(), command)] + if args is not None: + command.extend(args) + if log_run: + logger.info(f"Running '{command}'") + else: + logger.debug(f"Running '{command}'") + try: + result = subprocess.run( + command, capture_output=capture_output, check=True, text=True, **kwargs + ) + logger.debug( + f"Command '{command}' completed with:\nStdErr Output: {result.stderr}\nStdOut Output: {result.stdout}" + ) + return result.stdout.strip() + + except subprocess.CalledProcessError as e: + logger.error( + f"Command '{command}' failed with exit code {e.returncode}\nStdErr Output: {e.stderr}\nStdOut Output: {e.stdout}" + ) + raise e + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + raise e + + +def env_cleanup(): + clean = os.getenv("CLEAN") + if clean is not None and clean.lower() in ("no", "false", "f", "0"): + return False + return True diff --git a/tests/bdd/common/environment.py b/tests/bdd/common/environment.py index f0d0b1257..6cea9f882 100644 --- a/tests/bdd/common/environment.py +++ b/tests/bdd/common/environment.py @@ -4,14 +4,17 @@ logger = logging.getLogger(__name__) -def get_env(variable: str): - try: - value = os.getenv(variable) - if len(value) == 0: - raise ValueError("Env {variable} is empty") - logger.info(f"Found env {variable}={value}") - return value +def get_env(variable: str, warn=False): + value = os.getenv(variable) + if value is None: + if warn: + logger.warning(f"The env {variable} does not exist") + return None - except Exception as e: - logger.error(f"Failed to get env {variable}: {e}") + if len(value) == 0: + if warn: + logger.warning(f"The env {variable} is an empty string") return None + + logger.info(f"Found env {variable}={value}") + return value diff --git a/tests/bdd/common/helm.py b/tests/bdd/common/helm.py index b116de3db..f1ab544d3 100644 --- a/tests/bdd/common/helm.py +++ b/tests/bdd/common/helm.py @@ -5,8 +5,9 @@ from enum import Enum from shutil import which +import common +from common import root_dir, run from common.environment import get_env -from common.repo import root_dir, run_script logger = logging.getLogger(__name__) @@ -14,25 +15,11 @@ def repo_ls(): - try: - result = subprocess.run( - [helm_bin, "repo", "ls", "-o", "json"], - capture_output=True, - check=True, - text=True, - ) - return json.loads(result.stdout.strip()) - - except subprocess.CalledProcessError as e: - logger.error( - f"Error: command 'helm repo ls -o json' failed with exit code {e.returncode}" - ) - logger.error(f"Error Output: {e.stderr}") - return None - - except Exception as e: - logger.error(f"An unexpected error occurred: {e}") - return None + result = common.run( + helm_bin, + ["repo", "ls", "-o", "json"], + ) + return json.loads(result) def repo_add_mayastor(): @@ -42,43 +29,35 @@ def repo_add_mayastor(): if r["url"] == "https://openebs.github.io/mayastor-extensions": return r["name"] - try: - repo_name = "mayastor" - subprocess.run( - [ - helm_bin, - "repo", - "add", - repo_name, - "https://openebs.github.io/mayastor-extensions", - ], - capture_output=True, - check=True, - text=True, - ) - - subprocess.run( - [ - helm_bin, - "repo", - "update", - ], - capture_output=True, - check=True, - text=True, - ) - return repo_name - - except subprocess.CalledProcessError as e: - logger.error( - f"Error: command 'helm repo add mayastor https://openebs.github.io/mayastor-extensions' failed with exit code {e.returncode}" - ) - logger.error(f"Error Output: {e.stderr}") - return None - - except Exception as e: - logger.error(f"An unexpected error occurred: {e}") - return None + repo_name = "mayastor" + common.run( + helm_bin, + [ + "repo", + "add", + repo_name, + "https://openebs.github.io/mayastor-extensions", + ], + ) + + subprocess.run( + [ + helm_bin, + "repo", + "update", + ], + capture_output=True, + check=True, + text=True, + ) + common.run( + helm_bin, + [ + "repo", + "update", + ], + ) + return repo_name def latest_chart_so_far(version=None): @@ -90,48 +69,31 @@ def latest_chart_so_far(version=None): version = v repo_name = repo_add_mayastor() - assert repo_name is not None - helm_search_command = [ + stdout = common.run( helm_bin, - "search", - "repo", - repo_name + "/mayastor", - "--version", - "<" + version, - "-o", - "json", - ] - try: - result = subprocess.run( - helm_search_command, - capture_output=True, - check=True, - text=True, - ) - result_chart_info = json.loads(result.stdout.strip()) - return result_chart_info[0]["version"] - - except subprocess.CalledProcessError as e: - logger.error( - f"Error: command {helm_search_command} failed with exit code {e.returncode}" - ) - logger.error(f"Error Output: {e.stderr}") - return None - - except Exception as e: - logger.error(f"An unexpected error occurred: {e}") - return None + [ + "search", + "repo", + repo_name + "/mayastor", + "--version", + "<" + version, + "-o", + "json", + ], + ) + result_chart_info = json.loads(stdout) + return result_chart_info[0]["version"] class ChartSource(Enum): HOSTED = [ - "/bin/bash", + "bash", "-c", os.path.join(root_dir(), "scripts/helm/install.sh") + " --hosted-chart --wait", ] LOCAL = [ - "/bin/bash", + "bash", "-c", os.path.join(root_dir(), "scripts/helm/install.sh") + " --dep-update --wait", ] @@ -152,8 +114,7 @@ def __init__(self): self.namespace = "mayastor" def get_metadata_mayastor(self): - command = [ - helm_bin, + args = [ "get", "metadata", "mayastor", @@ -162,75 +123,37 @@ def get_metadata_mayastor(self): "-o", "json", ] - try: - result = subprocess.run( - command, - capture_output=True, - check=True, - text=True, - ) - return json.loads(result.stdout.strip()) + return json.loads(common.run(helm_bin, args, log_run=False)) - except subprocess.CalledProcessError as e: - logger.error( - f"Error: command '{command}' failed with exit code {e.returncode}" - ) - logger.error(f"Error Output: {e.stderr}") - return None - - except Exception as e: - logger.error(f"An unexpected error occurred: {e}") - return None - - def list(self): + def get_deployed(self, release: str): """ - Lists the deployed Helm releases in the specified namespace. + Get the deployed Helm release in the specified namespace as json Executes the 'helm ls' command to retrieve a list of deployed releases. Returns: str: A newline-separated string of deployed release names, or None if an error occurs. """ - try: - result = subprocess.run( - [ - helm_bin, - "ls", - "-n", - self.namespace, - "--deployed", - "--short", - ], - capture_output=True, - check=True, - text=True, - ) - return result.stdout.strip() - - except subprocess.CalledProcessError as e: - logger.error( - f"Error: command 'helm ls -n {self.namespace} --deployed --short' failed with exit code {e.returncode}" - ) - logger.error(f"Error Output: {e.stderr}") - return None - - except Exception as e: - logger.error(f"An unexpected error occurred: {e}") - return None - - def release_is_deployed(self, release_name: str): - releases = self.list() - if releases is not None: - for release in releases: - if release == release_name: - return True - return False + args = [ + "ls", + "-n", + self.namespace, + "--deployed", + f"--filter=^{release}$", + "-o=json", + ] + return common.run(helm_bin, args) def install_mayastor(self, source: ChartSource, version=None): - if self.release_is_deployed("mayastor"): - logger.error( - f"WARN: Helm release 'mayastor' already exists in the 'mayastor' namespace." + output_json = json.loads(self.get_deployed("mayastor")) + if len(output_json) == 1: + current_version = output_json[0]["app_version"] + logger.warning( + f"Helm release 'mayastor' already exists in the 'mayastor' namespace @ v{current_version}." ) + assert ( + current_version == version + ), f"Wanted to install {version}, but {current_version} already installed" return install_command = [] @@ -261,12 +184,12 @@ def install_mayastor(self, source: ChartSource, version=None): f"Error: command {install_command} failed with exit code {e.returncode}" ) logger.error(f"Error Output: {e.stderr}") - return None + raise e except Exception as e: logger.error(f"An unexpected error occurred: {e}") - return None + raise e def generate_test_tag(): - return run_script("scripts/python/generate-test-tag.sh") + return run("scripts/python/generate-test-tag.sh") diff --git a/tests/bdd/common/k8s_deployer.py b/tests/bdd/common/k8s_deployer.py new file mode 100644 index 000000000..d4bdac7da --- /dev/null +++ b/tests/bdd/common/k8s_deployer.py @@ -0,0 +1,48 @@ +import logging +import os + +import common +from common import run + +logger = logging.getLogger(__name__) + + +def deployer(): + return "./scripts/k8s/deployer.sh" + + +def start(workers: int): + if carry_on(): + try: + common.run( + "helm", + [ + "uninstall", + "mayastor", + "-n=mayastor", + "--ignore-not-found", + "--wait", + ], + absolute=True, + ) + common.run( + "kubectl", ["delete", "jobs", "-n=mayastor", "--all"], absolute=True + ) + return + except: + pass + + run(deployer(), ["start", "--label", "--cleanup", f"--workers={workers}"]) + + +def stop(): + if common.env_cleanup(): + run(deployer(), ["stop"]) + + +def carry_on(): + clean = os.getenv("REUSE_CLUSTER") + if clean is not None and clean.lower() in ("yes", "true", "y", "1"): + cluster = common.run("kind", ["get", "clusters"], absolute=True, log_run=False) + return cluster == "kind" + return False diff --git a/tests/bdd/common/kubectl_mayastor.py b/tests/bdd/common/kubectl_mayastor.py index 8ca4d2d8b..ad655ef03 100644 --- a/tests/bdd/common/kubectl_mayastor.py +++ b/tests/bdd/common/kubectl_mayastor.py @@ -1,41 +1,19 @@ import logging import os -import subprocess -from shutil import which -from common.environment import get_env +import common logger = logging.getLogger(__name__) -def get_bin_path(): - bins = get_env("TEST_DIR") - if bins: - return os.path.join(bins, "kubectl-plugin/bin/kubectl-mayastor") - logging.warning(f"Environmental variable 'BIN' is not set") - return which("kubectl-mayastor") - - -def kubectl_mayastor(args: list[str]): - command = [get_bin_path()] - command.extend(args) - logger.info(f"Running kubectl-mayastor command: {command}") - - try: - result = subprocess.run( - command, - capture_output=True, - check=True, - text=True, - ) - logger.info(f"kubectl-mayastor command succeeded") - return result.stdout.strip() - - except subprocess.CalledProcessError as e: - logger.error(f"Error: command '{command}' failed with exit code {e.returncode}") - logger.error(f"Error Output: {e.stderr}") - return None - - except Exception as e: - logger.error(f"An unexpected error occurred: {e}") - return None +def plugin_vnext(): + chart_vnext = common.chart_vnext() + return os.path.join(chart_vnext, "kubectl-plugin/bin/kubectl-mayastor") + + +def upgrade_vnext(): + run(["upgrade"], log_run=True) + + +def run(args: list[str], log_run=False): + return common.run(plugin_vnext(), args, log_run=log_run) diff --git a/tests/bdd/common/repo.py b/tests/bdd/common/repo.py index 47da706b4..00bfe35aa 100644 --- a/tests/bdd/common/repo.py +++ b/tests/bdd/common/repo.py @@ -11,9 +11,8 @@ def root_dir(): def run_script(script: str): - script = os.path.join(root_dir(), script) - logger.info(f"Running script '{script}'") - command = ["/bin/bash", "-c", script] + command = os.path.join(root_dir(), script) + logger.info(f"Running script '{command}'") try: result = subprocess.run( command, @@ -28,8 +27,8 @@ def run_script(script: str): except subprocess.CalledProcessError as e: logger.error(f"Error: command {command} failed with exit code {e.returncode}") logger.error(f"Error Output: {e.stderr}") - return None + raise e except Exception as e: logger.error(f"An unexpected error occurred: {e}") - return None + raise e diff --git a/tests/bdd/features/test_upgrade.py b/tests/bdd/features/test_upgrade.py index 1061e48e1..69b6fc1f5 100644 --- a/tests/bdd/features/test_upgrade.py +++ b/tests/bdd/features/test_upgrade.py @@ -1,10 +1,15 @@ """Upgrade feature tests.""" +import json import logging +import os +import common +import pytest +from common import k8s_deployer from common.environment import get_env from common.helm import ChartSource, HelmReleaseClient, latest_chart_so_far -from common.kubectl_mayastor import kubectl_mayastor +from common.kubectl_mayastor import upgrade_vnext from common.repo import run_script from kubernetes import client, config from pytest_bdd import given, scenario, then, when @@ -15,25 +20,72 @@ helm = HelmReleaseClient() -@scenario("upgrade.feature", "upgrade command is issued") -def test_upgrade_command_is_issued(): - """upgrade command is issued.""" +@scenario("upgrade.feature", "Upgrading to the local chart as v-next") +def test_upgrade_to_vnext(): + """Upgrading to the local chart as v-next.""" -@given("an installed mayastor helm chart") -def an_installed_mayastor_helm_chart(): - """an installed mayastor helm chart.""" - assert helm.install_mayastor(ChartSource.HOSTED, latest_chart_so_far()) is not None +@given("a 2-worker node kind kubernetes cluster") +def _(): + """a 2-worker node kind kubernetes cluster.""" + k8s_deployer.start(workers=2) + yield + k8s_deployer.stop() + + +@given("the latest mayastor helm chart is installed") +def the_latest_mayastor_is_installed(latest_chart_version): + """the latest mayastor helm chart is installed.""" + helm.install_mayastor(ChartSource.HOSTED, latest_chart_version) + + +@given("all io-engine nodes shall be listed by kubectl-mayastor") +def all_io_engine_nodes_shall_be_listed(latest_chart_version): + """all io-engine nodes shall be listed by kubectl-mayastor.""" + wait_rest_nodes_version(latest_chart_version) + + +@given("a v-next chart is prepared") +def _(): + """a v-next chart is prepared.""" + if common.chart_vnext_skip(): + return + # todo: fork once build system supports alternate chart path + # common.run("./scripts/python/upgrade-test-helper.sh", ["--fork", "--tag"]) + + +@given("the images and plugin are built for v-next") +def _(): + """the images and plugin are built for v-next.""" + if common.chart_vnext_skip(): + return + chart = os.path.join(common.root_dir(), "./chart") + common.run( + "./scripts/python/upgrade-test-helper.sh", + ["--build", "--chart-tag", "--chart", chart], + ) + + +@given("the images are loadable from the cluster") +def _(): + """the images are loadable from the cluster.""" + if common.chart_vnext_skip(): + return + common.run("./scripts/python/upgrade-test-helper.sh", ["--load"]) @when("a kubectl mayastor upgrade command is issued") def a_kubectl_mayastor_upgrade_command_is_issued(): """a kubectl mayastor upgrade command is issued.""" - assert kubectl_mayastor(["upgrade"]) is not None + upgrade_vnext() -@then("the installed chart should be upgraded to the kubectl mayastor plugin's version") -def the_installed_chart_should_be_upgraded_to_the_kubectl_mayastor_plugins_version(): +@then( + "eventually the installed chart should be upgraded to the kubectl mayastor plugin's version" +) +def eventually_the_installed_chart_should_be_upgraded_to_the_kubectl_mayastor_plugins_version( + latest_chart_version, +): """the installed chart should be upgraded to the kubectl mayastor plugin's version.""" upgrade_target_version = get_env("UPGRADE_TARGET_VERSION") @@ -42,15 +94,22 @@ def the_installed_chart_should_be_upgraded_to_the_kubectl_mayastor_plugins_versi upgrade_target_version = upgrade_target_version.lstrip("v") logger.info(f"Value of upgrade_target_version={upgrade_target_version}") + def log_it(): + log = (pytest.attempts % 10) == 0 + pytest.attempts += 1 + return log + @retry( - stop_max_attempt_number=450, + stop_max_attempt_number=60, wait_fixed=2000, ) def helm_upgrade_succeeded(): - logger.info("Checking if helm upgrade succeeded...") + log = log_it() + if log: + logger.info("Checking if helm upgrade succeeded...") metadata = helm.get_metadata_mayastor() - logger.debug(f"helm get metadata output={metadata}") - logger.debug(f"upgrade_target_version={upgrade_target_version}") + if log: + logger.info(f"helm get metadata output={metadata}") if metadata: assert metadata["version"] == upgrade_target_version return @@ -60,25 +119,99 @@ def helm_upgrade_succeeded(): stop_max_attempt_number=600, wait_fixed=2000, ) - def data_plane_upgrade_succeeded(): - logger.info("Checking if data-plane upgrade succeeded...") + def data_plane_upgrade_succeeded(not_target_tag): + log = log_it() + if log: + logger.info("Checking if data-plane upgrade succeeded...") config.load_kube_config() v1 = client.CoreV1Api() label_selector = "app=io-engine" pods = v1.list_namespaced_pod( namespace="mayastor", label_selector=label_selector ) - switch = True - for pod in pods.items: + io_engines = list( + filter( + lambda pod: any( + container.name == "io-engine" for container in pod.spec.containers + ), + pods.items, + ) + ) + if len(io_engines) == 0: + return + + all_done = True + for pod in io_engines: for i, container in enumerate(pod.spec.containers): if container.name == "io-engine": - logger.info( - f"pod.metadata.name={pod.metadata.name}, pod.spec.containers[{i}].image={container.image}" - ) - switch = switch and container.image.endswith(":develop") - logger.info(f"Value of 'switch' after the AND={switch}") + # Not straightforward to know which version to expect here, so let's check that + # the version is not the latest instead? + if container.image.endswith(f":v{not_target_tag.strip('v')}"): + all_done = False + if log: + logger.info( + f"pod.metadata.name={pod.metadata.name}, pod.spec.containers[{i}].image={container.image}" + ) break - assert switch + assert all_done is True + + nodes = client.CoreV1Api().list_node( + label_selector="openebs.io/engine=mayastor" + ) + + assert len(nodes.items) == len(io_engines) + pytest.attempts = 0 helm_upgrade_succeeded() - data_plane_upgrade_succeeded() + pytest.attempts = 0 + + data_plane_upgrade_succeeded(latest_chart_version) + + # Not straightforward to know which version to expect here, so let's check that + # the version is not the latest instead? + wait_rest_nodes_version(latest_chart_version, match=False) + + +@pytest.fixture(scope="module") +def latest_chart_version(): + yield latest_chart_so_far() + + +@retry( + stop_max_attempt_number=60, + wait_fixed=1000, +) +def wait_rest_nodes_version(version, match=True): + config.load_kube_config() + nodes = client.CoreV1Api().list_node(label_selector="openebs.io/engine=mayastor") + k8s_nodes = len(nodes.items) + + rest_nodes = json.loads( + common.kubectl_mayastor.run(["get", "nodes", "-o=json"], log_run=True) + ) + rest_io_engines = len(rest_nodes) + logger.info(f"Mayastor Nodes: {rest_nodes}") + + assert ( + k8s_nodes == rest_io_engines + ), f"Found {k8s_nodes} k8s nodes with the io-engine label, but only {rest_io_engines} nodes from kubectl-mayastor" + + assert all( + node["spec"]["version"] == node["state"]["version"] for node in rest_nodes + ) + + version_stripped = version.strip("v") + if match: + all_on_version = all( + node["spec"]["version"].strip("v") == version_stripped + for node in rest_nodes + ) + assert all_on_version, f"Not all nodes on the version v{version_stripped}" + else: + all_not_on_version = all( + node["spec"]["version"].strip("v") != version_stripped + for node in rest_nodes + ) + assert ( + all_not_on_version + ), f"Some of the nodes are still on the version v{version_stripped}" diff --git a/tests/bdd/features/upgrade.feature b/tests/bdd/features/upgrade.feature index 8d4d9b61f..42945847e 100644 --- a/tests/bdd/features/upgrade.feature +++ b/tests/bdd/features/upgrade.feature @@ -1,8 +1,13 @@ Feature: Upgrade Background: - Given an installed mayastor helm chart + Given a 2-worker node kind kubernetes cluster + And a v-next chart is prepared + And the images and plugin are built for v-next + And the images are loadable from the cluster + And the latest mayastor helm chart is installed + And all io-engine nodes shall be listed by kubectl-mayastor - Scenario: upgrade command is issued + Scenario: Upgrading to the local chart as v-next When a kubectl mayastor upgrade command is issued - Then the installed chart should be upgraded to the kubectl mayastor plugin's version + Then eventually the installed chart should be upgraded to the kubectl mayastor plugin's version diff --git a/tests/bdd/shell.nix b/tests/bdd/shell.nix new file mode 100644 index 000000000..21f066a7d --- /dev/null +++ b/tests/bdd/shell.nix @@ -0,0 +1,22 @@ +{ pkgs ? import (import ../../nix/sources.nix).nixpkgs { + overlays = [ (_: _: { inherit (import ../nix/../sources.nix); }) (import ../../nix/overlay.nix { }) ]; + } +}: +let + k8sShellAttrs = import ../../scripts/k8s/shell.nix { inherit pkgs; }; + helmShellAttrs = import ../../chart/shell.nix { inherit pkgs; }; + bddBuildInputs = with pkgs; [ + autoflake + black + isort + python3 + utillinux + virtualenv + which + ]; +in +pkgs.mkShell { + name = "pytest-shell"; + buildInputs = k8sShellAttrs.buildInputs ++ helmShellAttrs.buildInputs ++ + bddBuildInputs; +}