[DO NOT MERGE] Feature/test harness

.github/workflows/integration.yml

@@ -0,0 +1,68 @@
+# are working against known samples
+name: "OSCAL Validations: Integration tests"
+
+# Triggered when code is pushed to master and on pull requests
+on:
+  push:
+    branches:
+      - master
+      - develop
+      - 'feature/**'  # This will match any branch starting with "feature"
+
+  pull_request:
+
+# the job requires some dependencies to be installed (including submodules), runs the tests, and then reports results
+jobs:
+  # one job that runs tests
+  run-tests:
+    name: "Integration Tests with ${{ matrix.cli_type }} CLI"
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        cli_type: ['default', 'enhanced']
+        include:
+          - cli_type: default
+            cli_url: https://repo1.maven.org/maven2/gov/nist/secauto/oscal/tools/oscal-cli/cli-core/1.0.3/cli-core-1.0.3-oscal-cli.zip
+          - cli_type: enhanced
+            cli_url: https://repo1.maven.org/maven2/dev/metaschema/oscal/oscal-cli-enhanced/2.4.0/oscal-cli-enhanced-2.4.0-oscal-cli.zip
+
+    defaults:
+        run:
+          working-directory: ./build
+    steps:
+      # Check-out the repository under $GITHUB_WORKSPACE
+      - name: Checkout repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+
+      - name: Set up Java
+        uses: actions/setup-java@67fbd726daaf08212a7b021c1c4d117f94a81dd3
+        with:
+          distribution: 'adopt'
+          java-version: '11'
+
+      - name: Read node version from `.nvmrc` file
+        id: nvmrc
+        shell: bash
+        run: echo "NODE_VERSION=$(cat .nvmrc)" >> $GITHUB_ENV
+
+      - name: Install required node.js version
+        uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+      - name: Setup Environment
+        run: |
+          export OSCAL_CLI_URL=${{ matrix.cli_url }}
+      - name: Install Dependencies
+        run: |
+          make configure
+      - name: Run tests
+        shell: bash
+        run: |
+          make test
+      - name : Publish all Junit XML tests results in Github Summary
+        uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86
+        if: always()
+        with:
+            paths: |
+              **/reports/junit-*.xml
+      

.gitignore

@@ -57,3 +57,6 @@ node_modules/
 /.runbuild
 
 /summary.csv
+/build/reports
+build/@rerun.txt
+/build/sarif

build/Makefile

@@ -1,5 +1,7 @@
 SHELL:=/usr/bin/env bash
 
+# Default OSCAL CLI URL
+OSCAL_CLI_URL ?= https://repo1.maven.org/maven2/dev/metaschema/oscal/oscal-cli-enhanced/2.4.0/oscal-cli-enhanced-2.4.0-oscal-cli.zip
 .PHONY: help
 # Run "make" or "make help" to get a list of user targets
 # Adapted from https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
@@ -25,6 +27,19 @@ dependencies: node_modules ## Ensure dependencies have been installed
 node_modules: package.json package-lock.json
 	npm ci
 
+.PHONY: test
+test: 
+	npm run test
+
+.PHONY: configure
+configure: 
+	npm install
+	wget $(OSCAL_CLI_URL)
+	unzip -o *-oscal-cli.zip -d node_modules/.bin/oscal-cli-enhanced
+	rm *-oscal-cli.zip
+	ln -sf oscal-cli-enhanced/bin/oscal-cli node_modules/.bin/oscal-cli
+	chmod +x node_modules/.bin/oscal-cli-enhanced/bin/oscal-cli
+
 .PHONY: clean
 clean: clean-schemas clean-linkcheck clean-converters clean-archives clean-resolved-metaschemas ## Remove all generated content
 

build/cucumber.json

@@ -0,0 +1,29 @@
+{
+    "default": {
+      "import": [
+        "import { register } from 'ts-node'",
+        "register({ esm: true, experimentalSpecifierResolution: 'node' })",
+        "features/**/*.ts"
+      ],
+      "format": [
+        ["junit", "reports/junit-oscal.xml"],
+        ["html", "reports/oscal.html"],
+        ["rerun","@rerun.txt"]
+      ],
+      "retry": 2,
+      "retryTagFilter": "@flaky"
+    },
+    "rerun": {
+      "import": [
+        "import { register } from 'ts-node'",
+        "register({ esm: true, experimentalSpecifierResolution: 'node' })",
+        "features/**/*.ts"
+      ],
+      "format": [
+        ["junit", "reports/junit-oscal-rerun.xml"],
+        ["html", "reports/oscal-rerun.html"]
+      ],
+      "retry": 0,
+      "paths": ["@rerun.txt"]
+    }
+  }

build/features/oscal.feature

@@ -0,0 +1,36 @@
+Feature: Validate OSCAL Content
+  As a developer
+  I want to validate OSCAL content against appropriate metaschemas
+
+  Background: 
+    Given the OSCAL CLI tool is installed
+    And the metaschema directory is "../src/metaschema"
+
+  Scenario Outline: Validate OSCAL content
+    When I validate "<type>" content in "<path>"
+    Then all validations should pass without errors
+
+    Examples:
+      | type    | path |
+      | profile | ../src/specifications/profile-resolution/profile-resolution-examples/full-test_profile.xml|
+      | catalog | ../src/specifications/valid-content/catalog.xml|
+      | ssp | ../src/specifications/valid-content/ssp.xml|
+      | poam | ../src/specifications/valid-content/poam.xml|
+      | assessment-plan | ../src/specifications/valid-content/ap.xml|
+      | assessment-results | ../src/specifications/valid-content/ar.xml|
+
+@style
+  Scenario Outline: Validate OSCAL style guide
+    When I validate "<metaschema>" content it passes style guide
+    Then all validations should pass without errors
+
+    Examples:
+      | metaschema    |
+      | profile |
+      | catalog |
+      | ssp |
+      | poam |
+      | assessment-common |
+      | implementation-common |
+      | assessment-plan |
+      | assessment-results |

build/features/step_defenitions/steps.ts

@@ -0,0 +1,200 @@
+import { Given, Then, When } from "@cucumber/cucumber";
+import chalk from 'chalk';
+import { Log } from 'sarif';
+import { join, resolve, dirname } from "path";
+import { writeFileSync, existsSync, mkdirSync, readFileSync } from "fs";
+import { execSync } from "child_process";
+
+const sarifDir = join(process.cwd(), "sarif");
+
+if (!existsSync(sarifDir)) {
+  mkdirSync(sarifDir, { recursive: true });
+}
+
+function createTerminalLink(location: any) {
+  const filePath = location?.physicalLocation?.artifactLocation?.uri || '';
+  const lineNumber = location?.physicalLocation?.region?.startLine;
+  const columnNumber = location?.physicalLocation?.region?.startColumn;
+
+  if (filePath.startsWith('http')) {
+    const fileName = filePath.split('/').pop() || filePath;
+    const linkText = `${fileName}:${lineNumber}:${columnNumber}`;
+
+    if (filePath.includes('githubusercontent.com')) {
+        const [org, repo, ref, ...pathParts] = filePath
+            .replace('https://raw.githubusercontent.com/', '')
+            .replace('refs/heads/', '')
+            .split('/');
+
+        const path = pathParts.join('/');
+        const githubLink = `https://github.com/${org}/${repo}/blob/${ref}/${path}#L${lineNumber}`;
+        return `\u001b]8;;${githubLink}\u0007${linkText}\u001b]8;;\u0007`;
+    }
+
+    return `\u001b]8;;${filePath}#L${lineNumber}\u0007${linkText}\u001b]8;;\u0007`;
+  }
+
+  const fileName = filePath.split('/').pop() || filePath;
+  const linkText = `${fileName}:${lineNumber}:${columnNumber}`;
+  return linkText;
+}
+
+interface LogOptions {
+  showFileName: boolean;
+}
+
+export function formatSarifOutput(
+  log: Log,
+  logOptions: LogOptions = { showFileName: true }
+) {
+  try {
+    if (!log || !log.runs || !log.runs[0] || !log.runs[0].results) {
+      return chalk.red('Invalid SARIF log format');
+    }
+
+    const results = log.runs[0].results;
+
+    const formattedOutput = results
+      .filter((x) => x.kind != 'informational' && x.kind !== 'pass')
+      .map((result) => {
+        const fileDetails = logOptions.showFileName
+          ? chalk.gray(
+              (result.ruleId || "") +
+                " " +
+                (result.locations ? createTerminalLink(result.locations[0] as any) : "")
+            )
+          : chalk.gray(result.ruleId || "");
+
+        if (result.kind == 'fail') {
+          return (
+            chalk.red.bold("[" + result.level?.toUpperCase() + "] ") +
+            fileDetails +
+            "\n" +
+            chalk.hex("#b89642")(result.message.text)
+          );
+        } else {
+          return chalk.yellow.bold(result.message.text);
+        }
+      })
+      .join('\n\n');
+
+    return formattedOutput;
+  } catch (error: any) {
+    return chalk.red(`Error processing SARIF log: ${error.message}`);
+  }
+}
+
+Given('the following directories by type:', function(dataTable:any) {
+  this.dirsByType = {};
+  dataTable.rows().forEach(([type, paths]:any) => {
+    this.dirsByType[type] = paths.split(',');
+  });
+});
+
+Given('the OSCAL CLI tool is installed', async function() {
+  const success = execSync(`npx oscal-cli --version`);
+  if (!success) throw new Error('OSCAL CLI not installed');
+});
+
+Given('the metaschema directory is {string}', function(dir) {
+  this.metaschemaDir = dir;
+});
+
+When('I validate {string} content in {string}',{timeout:90000}, async function(type, path) {
+  const metaschema = 'oscal_'+type+'_metaschema.xml';
+  const metaschemaPath = `${this.metaschemaDir}/${metaschema}`;
+  const sarifOutputPath = join(sarifDir, `${type}_validation.sarif`);
+
+  try {
+    const output = execSync(
+      `npx oscal-cli metaschema validate-content ${path} -m ${metaschemaPath} -o ${sarifOutputPath}`,
+      {
+        stdio: 'pipe',
+        encoding: 'utf-8'
+      }
+    );
+
+    const sarifContent = JSON.parse(readFileSync(sarifOutputPath, 'utf-8'));
+    this.result = { 
+      isValid: true, 
+      output,
+      sarifLog: sarifContent
+    };
+
+  } catch (error:any) {
+    let sarifLog = null;
+    try {
+      sarifLog = JSON.parse(readFileSync(sarifOutputPath, 'utf-8'));
+    } catch (e) {
+      // SARIF file might not exist or be invalid in case of errors
+    }
+
+    this.result = { 
+      isValid: false,
+      error: error.message,
+      stderr: error.stderr?.toString(),
+      stdout: error.stdout?.toString(),
+      sarifLog
+    };
+  }
+});
+
+When('I validate {string} content it passes style guide',{timeout:90000}, async function(type) {
+  const metaschema = 'oscal_'+type+'_metaschema.xml';
+  const metaschemaPath = `${this.metaschemaDir}/${metaschema}`;
+  const styleGuide = `${this.metaschemaDir}/oscal_style_guide.xml`;
+  const sarifOutputPath = join(sarifDir, `${type}_style_guide.sarif`);
+
+  try {
+    const output = execSync(
+      `npx oscal-cli metaschema validate ${metaschemaPath} -c ${styleGuide} --disable-schema-validation -o ${sarifOutputPath}`, 
+      {
+        stdio: 'pipe',
+        encoding: 'utf-8'
+      }
+    );
+
+    const sarifContent = JSON.parse(readFileSync(sarifOutputPath, 'utf-8'));
+    this.result = { 
+      isValid: true, 
+      output,
+      sarifLog: sarifContent
+    };
+
+  } catch (error:any) {
+    let sarifLog = null;
+    try {
+      sarifLog = JSON.parse(readFileSync(sarifOutputPath, 'utf-8'));
+    } catch (e) {
+      // SARIF file might not exist or be invalid in case of errors
+    }
+
+    this.result = { 
+      isValid: false,
+      error: error.message,
+      stderr: error.stderr?.toString(),
+      stdout: error.stdout?.toString(),
+      sarifLog
+    };
+  }
+});
+
+Then('all validations should pass without errors', function() {
+  if (!this.result.isValid) {
+    const errors:string[] = [];
+
+    if (this.result.sarifLog) {
+      errors.push(formatSarifOutput(this.result.sarifLog));
+    }
+
+    if (this.result.stderr) {
+      errors.push(chalk.red(this.result.stderr));
+    }
+
+    if (this.result.error) {
+      errors.push(chalk.red(this.result.error));
+    }
+
+    throw new Error(`Validation failed:\n${errors.join('\n\n')}`);
+  }
+});