improve test harness

usnistgov · Jan 22, 2025 · f1a859b · f1a859b
1 parent 39f1ee2
commit f1a859b
Show file tree

Hide file tree

Showing 5 changed files with 198 additions and 20 deletions.
diff --git a/build/Makefile b/build/Makefile
@@ -1,5 +1,7 @@
 SHELL:=/usr/bin/env bash
 
+# Default OSCAL CLI URL
+OSCAL_CLI_URL ?= https://repo1.maven.org/maven2/dev/metaschema/oscal/oscal-cli-enhanced/2.4.0/oscal-cli-enhanced-2.4.0-oscal-cli.zip
 .PHONY: help
 # Run "make" or "make help" to get a list of user targets
 # Adapted from https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
@@ -32,9 +34,9 @@ test:
 .PHONY: configure
 configure: 
 	npm install
-	wget https://repo1.maven.org/maven2/dev/metaschema/oscal/oscal-cli-enhanced/2.4.0/oscal-cli-enhanced-2.4.0-oscal-cli.zip
-	unzip -o oscal-cli-enhanced-2.4.0-oscal-cli.zip -d node_modules/.bin/oscal-cli-enhanced
-	rm oscal-cli-enhanced-2.4.0-oscal-cli.zip
+	wget $(OSCAL_CLI_URL)
+	unzip -o *-oscal-cli.zip -d node_modules/.bin/oscal-cli-enhanced
+	rm *-oscal-cli.zip
 	ln -sf oscal-cli-enhanced/bin/oscal-cli node_modules/.bin/oscal-cli
 	chmod +x node_modules/.bin/oscal-cli-enhanced/bin/oscal-cli
 

diff --git a/build/features/oscal.feature b/build/features/oscal.feature
@@ -18,3 +18,19 @@ Feature: Validate OSCAL Content
       | poam | ../src/specifications/valid-content/poam.xml|
       | assessment-plan | ../src/specifications/valid-content/ap.xml|
       | assessment-results | ../src/specifications/valid-content/ar.xml|
+
+@style
+  Scenario Outline: Validate OSCAL style guide
+    When I validate "<metaschema>" content it passes style guide
+    Then all validations should pass without errors
+
+    Examples:
+      | metaschema    |
+      | profile |
+      | catalog |
+      | ssp |
+      | poam |
+      | assessment-common |
+      | implementation-common |
+      | assessment-plan |
+      | assessment-results |
diff --git a/build/features/step_defenitions/steps.ts b/build/features/step_defenitions/steps.ts
@@ -1,7 +1,89 @@
 import { Given, Then, When } from "@cucumber/cucumber";
-import { statSync } from 'fs';
-import { resolve } from 'path';
-import {execSync} from "child_process"
+import chalk from 'chalk';
+import { Log } from 'sarif';
+import { join, resolve, dirname } from "path";
+import { writeFileSync, existsSync, mkdirSync, readFileSync } from "fs";
+import { execSync } from "child_process";
+
+const sarifDir = join(process.cwd(), "sarif");
+
+if (!existsSync(sarifDir)) {
+  mkdirSync(sarifDir, { recursive: true });
+}
+
+function createTerminalLink(location: any) {
+  const filePath = location?.physicalLocation?.artifactLocation?.uri || '';
+  const lineNumber = location?.physicalLocation?.region?.startLine;
+  const columnNumber = location?.physicalLocation?.region?.startColumn;
+
+  if (filePath.startsWith('http')) {
+    const fileName = filePath.split('/').pop() || filePath;
+    const linkText = `${fileName}:${lineNumber}:${columnNumber}`;
+
+    if (filePath.includes('githubusercontent.com')) {
+        const [org, repo, ref, ...pathParts] = filePath
+            .replace('https://raw.githubusercontent.com/', '')
+            .replace('refs/heads/', '')
+            .split('/');
+
+        const path = pathParts.join('/');
+        const githubLink = `https://github.com/${org}/${repo}/blob/${ref}/${path}#L${lineNumber}`;
+        return `\u001b]8;;${githubLink}\u0007${linkText}\u001b]8;;\u0007`;
+    }
+
+    return `\u001b]8;;${filePath}#L${lineNumber}\u0007${linkText}\u001b]8;;\u0007`;
+  }
+
+  const fileName = filePath.split('/').pop() || filePath;
+  const linkText = `${fileName}:${lineNumber}:${columnNumber}`;
+  return linkText;
+}
+
+interface LogOptions {
+  showFileName: boolean;
+}
+
+export function formatSarifOutput(
+  log: Log,
+  logOptions: LogOptions = { showFileName: true }
+) {
+  try {
+    if (!log || !log.runs || !log.runs[0] || !log.runs[0].results) {
+      return chalk.red('Invalid SARIF log format');
+    }
+
+    const results = log.runs[0].results;
+
+    const formattedOutput = results
+      .filter((x) => x.kind != 'informational' && x.kind !== 'pass')
+      .map((result) => {
+        const fileDetails = logOptions.showFileName
+          ? chalk.gray(
+              (result.ruleId || "") +
+                " " +
+                (result.locations ? createTerminalLink(result.locations[0] as any) : "")
+            )
+          : chalk.gray(result.ruleId || "");
+
+        if (result.kind == 'fail') {
+          return (
+            chalk.red.bold("[" + result.level?.toUpperCase() + "] ") +
+            fileDetails +
+            "\n" +
+            chalk.hex("#b89642")(result.message.text)
+          );
+        } else {
+          return chalk.yellow.bold(result.message.text);
+        }
+      })
+      .join('\n\n');
+
+    return formattedOutput;
+  } catch (error: any) {
+    return chalk.red(`Error processing SARIF log: ${error.message}`);
+  }
+}
+
 Given('the following directories by type:', function(dataTable:any) {
   this.dirsByType = {};
   dataTable.rows().forEach(([type, paths]:any) => {
@@ -10,8 +92,7 @@ Given('the following directories by type:', function(dataTable:any) {
 });
 
 Given('the OSCAL CLI tool is installed', async function() {
-  const success=execSync(`npx oscal-cli --version`)
-
+  const success = execSync(`npx oscal-cli --version`);
   if (!success) throw new Error('OSCAL CLI not installed');
 });
 
@@ -22,26 +103,98 @@ Given('the metaschema directory is {string}', function(dir) {
 When('I validate {string} content in {string}',{timeout:90000}, async function(type, path) {
   const metaschema = 'oscal_'+type+'_metaschema.xml';
   const metaschemaPath = `${this.metaschemaDir}/${metaschema}`;
+  const sarifOutputPath = join(sarifDir, `${type}_validation.sarif`);
 
   try {
-    const output = execSync(`npx oscal-cli metaschema validate-content ${path} -m ${metaschemaPath}`, {
-      stdio: 'pipe',
-      encoding: 'utf-8'
-    });
-    this.result = { isValid: true, output };
+    const output = execSync(
+      `npx oscal-cli metaschema validate-content ${path} -m ${metaschemaPath} -o ${sarifOutputPath}`,
+      {
+        stdio: 'pipe',
+        encoding: 'utf-8'
+      }
+    );
+
+    const sarifContent = JSON.parse(readFileSync(sarifOutputPath, 'utf-8'));
+    this.result = { 
+      isValid: true, 
+      output,
+      sarifLog: sarifContent
+    };
+
   } catch (error:any) {
+    let sarifLog = null;
+    try {
+      sarifLog = JSON.parse(readFileSync(sarifOutputPath, 'utf-8'));
+    } catch (e) {
+      // SARIF file might not exist or be invalid in case of errors
+    }
+
     this.result = { 
       isValid: false,
       error: error.message,
       stderr: error.stderr?.toString(),
-      stdout: error.stdout?.toString()
+      stdout: error.stdout?.toString(),
+      sarifLog
     };
   }
- });
+});
 
-  Then('all validations should pass without errors', function() {
-    if (!this.result.isValid) {
-      throw new Error(`Validation failed:\n`+this.result.stderr);
+When('I validate {string} content it passes style guide',{timeout:90000}, async function(type) {
+  const metaschema = 'oscal_'+type+'_metaschema.xml';
+  const metaschemaPath = `${this.metaschemaDir}/${metaschema}`;
+  const styleGuide = `${this.metaschemaDir}/oscal_style_guide.xml`;
+  const sarifOutputPath = join(sarifDir, `${type}_style_guide.sarif`);
+
+  try {
+    const output = execSync(
+      `npx oscal-cli metaschema validate ${metaschemaPath} -c ${styleGuide} --disable-schema-validation -o ${sarifOutputPath}`, 
+      {
+        stdio: 'pipe',
+        encoding: 'utf-8'
+      }
+    );
+
+    const sarifContent = JSON.parse(readFileSync(sarifOutputPath, 'utf-8'));
+    this.result = { 
+      isValid: true, 
+      output,
+      sarifLog: sarifContent
+    };
+
+  } catch (error:any) {
+    let sarifLog = null;
+    try {
+      sarifLog = JSON.parse(readFileSync(sarifOutputPath, 'utf-8'));
+    } catch (e) {
+      // SARIF file might not exist or be invalid in case of errors
     }
-  });
 
+    this.result = { 
+      isValid: false,
+      error: error.message,
+      stderr: error.stderr?.toString(),
+      stdout: error.stdout?.toString(),
+      sarifLog
+    };
+  }
+});
+
+Then('all validations should pass without errors', function() {
+  if (!this.result.isValid) {
+    const errors:string[] = [];
+
+    if (this.result.sarifLog) {
+      errors.push(formatSarifOutput(this.result.sarifLog));
+    }
+
+    if (this.result.stderr) {
+      errors.push(chalk.red(this.result.stderr));
+    }
+
+    if (this.result.error) {
+      errors.push(chalk.red(this.result.error));
+    }
+
+    throw new Error(`Validation failed:\n${errors.join('\n\n')}`);
+  }
+});
diff --git a/build/package-lock.json b/build/package-lock.json
diff --git a/build/package.json b/build/package.json
@@ -4,14 +4,19 @@
     "version": "1.0.0",
     "scripts": {
         "test": "cross-env-shell NODE_OPTIONS=\"--loader ts-node/esm --no-warnings --experimental-specifier-resolution=node\" cucumber-js",
-        "test:failed": "cross-env NODE_OPTIONS=\"--loader ts-node/esm --no-warnings --experimental-specifier-resolution=node\" cucumber-js -p rerun"
+        "test:failed": "cross-env NODE_OPTIONS=\"--loader ts-node/esm --no-warnings --experimental-specifier-resolution=node\" cucumber-js -p rerun",
+        "test:style": "cross-env NODE_OPTIONS=\"--loader ts-node/esm --no-warnings --experimental-specifier-resolution=node\" cucumber-js --tags @style"
+
     },
+
     "type": "module",
     "devDependencies": {
         "@cucumber/cucumber": "^10.8.0",
         "ajv-cli": "^5.0.0",
         "ajv-formats": "^3.0.1",
+        "@types/sarif": "^2.1.7",
         "cross-env-shell": "7.0.3",
+        "chalk": "^5.3.0",
         "markdown-link-check": "3.12.2",
         "oscal": "2.0.7",
         "ts-node": "10.9.2",