feat: ability to test rule errors and invalid schemas per RFC-103

docs/src/integrate/nodejs-api.md

@@ -796,7 +796,30 @@ ruleTester.run("my-rule", rule, {
             code: "var invalidVariable = true",
             errors: [{ message: /^Unexpected.+variable/ }]
         }
-    ]
+    ],
+
+    // Optional array for testing invalid rule options or custom exceptions thrown by a rule.
+    fatal: [
+        // Test case for invalid rule options. Useful for complex schemas.
+        {
+            // `code` can be omitted as it's irrelevant when testing the schema.
+            options: [{ foo: true }],
+            error: {
+                // Only one property in this error object is required.
+                name: 'SchemaValidationError', // Error class name.
+                message: 'Value {"foo":true} should NOT have additional properties.', // Error message. Can be provided as string or RegExp.
+            },
+        },
+
+        // Test case for a custom exception thrown by the rule.
+        {
+            code: 'for(const x of [1, 2, 3]) {}',
+            error: {
+                name: 'NotYetImplementedError',
+                message: 'Not implemented',
+            },
+        },
+    ],
 });
 ```
 
@@ -810,9 +833,9 @@ The `RuleTester#run()` method is used to run the tests. It should be passed the
 
 * The name of the rule (string)
 * The rule object itself (see ["working with rules"](../extend/custom-rules))
-* An object containing `valid` and `invalid` properties, each of which is an array containing test cases.
+* An object containing the following test case array properties: `valid`, `invalid`, `fatal` (optional)
 
-A test case is an object with the following properties:
+Valid and invalid test cases are objects with the following properties:
 
 * `name` (string, optional): The name to use for the test case, to make it easier to find
 * `code` (string, required): The source code that the rule should be run on
@@ -836,6 +859,8 @@ In addition to the properties above, invalid test cases can also have the follow
     If a string is provided as an error instead of an object, the string is used to assert the `message` of the error.
 * `output` (string, required if the rule fixes code): Asserts the output that will be produced when using this rule for a single pass of autofixing (e.g. with the `--fix` command line flag). If this is `null`, asserts that none of the reported problems suggest autofixes.
 
+Fatal test cases (which are optional) are the same as invalid test cases, except that `code` is optional (it may be irrelevant when testing rule options), and there's an `error` object instead of an `errors` array. The `error` object should include one or both of the `message` of the error and the error (exception) class `name`. Fatal test cases can be used to test custom errors thrown by the rule, or invalid rule options (in which case the error `name` will be `SchemaValidationError`, and the `message` will be come from JSON Schema -- note that strings from JSON Schema are subject to change with future upgrades).
+
 Any additional properties of a test case will be passed directly to the linter as config options. For example, a test case can have a `parserOptions` property to configure parser behavior:
 
 ```js

lib/config/rule-validator.js

@@ -16,6 +16,7 @@ const {
     getRuleOptionsSchema
 } = require("./flat-config-helpers");
 const ruleReplacements = require("../../conf/replacements.json");
+const SchemaValidationError = require("../shared/schema-validation-error");
 
 //-----------------------------------------------------------------------------
 // Helpers
@@ -143,11 +144,16 @@ class RuleValidator {
                 validateRule(ruleOptions.slice(1));
 
                 if (validateRule.errors) {
-                    throw new Error(`Key "rules": Key "${ruleId}": ${
-                        validateRule.errors.map(
-                            error => `\tValue ${JSON.stringify(error.data)} ${error.message}.\n`
-                        ).join("")
+                    const message = validateRule.errors.map(
+                        error => `\tValue ${JSON.stringify(error.data)} ${error.message}.\n`
+                    ).join("");
+                    const error = new SchemaValidationError(`Key "rules": Key "${ruleId}": ${
+                        message
                     }`);
+
+                    error.messageForTest = message.trim(); // Original exception message without extra helper text for asserting in fatal test cases.
+
+                    throw error;
                 }
             }
         }

lib/linter/linter.js

@@ -1340,6 +1340,7 @@ class Linter {
                 providedOptions.physicalFilename
             );
         } catch (err) {
+            err.messageForTest = err.message; // Original exception message without extra helper text for asserting in fatal test cases.
             err.message += `\nOccurred while linting ${options.filename}`;
             debug("An error occurred while traversing");
             debug("Filename:", options.filename);
@@ -1640,6 +1641,7 @@ class Linter {
                 providedOptions.physicalFilename
             );
         } catch (err) {
+            err.messageForTest = err.message; // Original exception message without extra helper text for asserting in fatal test cases.
             err.message += `\nOccurred while linting ${options.filename}`;
             debug("An error occurred while traversing");
             debug("Filename:", options.filename);

lib/rule-tester/flat-rule-tester.js

@@ -60,6 +60,19 @@ const { ConfigArraySymbol } = require("@humanwhocodes/config-array");
  * @property {boolean} [only] Run only this test case or the subset of test cases with this property.
  */
 
+/**
+ * A test case that is expected to trigger a fatal error / exception.
+ * @typedef {Object} FatalTestCase
+ * @property {string} [name] Name for the test case.
+ * @property {string} [code] Code for the test case.
+ * @property {TestCaseFatalError} error Expected error/exception.
+ * @property {any[]} [options] Options for the test case.
+ * @property {{ [name: string]: any }} [settings] Settings for the test case.
+ * @property {string} [filename] The fake filename for the test case. Useful for rules that make assertion about filenames.
+ * @property {LanguageOptions} [languageOptions] The language options to use in the test case.
+ * @property {boolean} [only] Run only this test case or the subset of test cases with this property.
+ */
+
 /**
  * A description of a reported error used in a rule tester test.
  * @typedef {Object} TestCaseError
@@ -72,6 +85,13 @@ const { ConfigArraySymbol } = require("@humanwhocodes/config-array");
  * @property {number} [endLine] The 1-based line number of the reported end location.
  * @property {number} [endColumn] The 1-based column number of the reported end location.
  */
+
+/**
+ * A description of an error/exception from a fatal rule tester test case.
+ * @typedef {Object} TestCaseFatalError
+ * @property {string | RegExp} [message] Error message.
+ * @property {string} [name] Error class name.
+ */
 /* eslint-enable jsdoc/valid-types -- https://github.com/jsdoc-type-pratt-parser/jsdoc-type-pratt-parser/issues/4#issuecomment-778805577 */
 
 //------------------------------------------------------------------------------
@@ -100,6 +120,7 @@ const RuleTesterParameters = [
     "filename",
     "options",
     "errors",
+    "error",
     "output",
     "only"
 ];
@@ -120,6 +141,15 @@ const errorObjectParameters = new Set([
 ]);
 const friendlyErrorObjectParameterList = `[${[...errorObjectParameters].map(key => `'${key}'`).join(", ")}]`;
 
+/*
+ * All allowed property names in fatal error objects.
+ */
+const fatalErrorObjectParameters = new Set([
+    "message",
+    "name"
+]);
+const friendlyFatalErrorObjectParameterList = `[${[...fatalErrorObjectParameters].map(key => `'${key}'`).join(", ")}]`;
+
 /*
  * All allowed property names in suggestion objects.
  */
@@ -405,8 +435,8 @@ class FlatRuleTester {
 
     /**
      * Adds the `only` property to a test to run it in isolation.
-     * @param {string | ValidTestCase | InvalidTestCase} item A single test to run by itself.
-     * @returns {ValidTestCase | InvalidTestCase} The test with `only` set.
+     * @param {string | ValidTestCase | InvalidTestCase | FatalTestCase} item A single test to run by itself.
+     * @returns {ValidTestCase | InvalidTestCase | FatalTestCase} The test with `only` set.
      */
     static only(item) {
         if (typeof item === "string") {
@@ -450,7 +480,8 @@ class FlatRuleTester {
      * @param {Function} rule The rule to test.
      * @param {{
      *   valid: (ValidTestCase | string)[],
-     *   invalid: InvalidTestCase[]
+     *   invalid: InvalidTestCase[],
+     *   fatal: FatalTestCase[]
      * }} test The collection of tests to run.
      * @throws {TypeError|Error} If non-object `test`, or if a required
      * scenario of the given type is missing.
@@ -671,13 +702,42 @@ class FlatRuleTester {
                 configs.normalizeSync();
                 configs.getConfig("test.js");
             } catch (error) {
+                if (item.error && error.messageForTest) {
+
+                    // If this was a testable exception, return it so it can be compared against in the test case.
+                    return {
+                        messages: [{
+                            ruleId: ruleName,
+                            fatal: true,
+                            message: error.messageForTest,
+                            name: error.name === "Error" ? error.constructor.name : error.name
+                        }]
+                    };
+                }
+
                 error.message = `ESLint configuration in rule-tester is invalid: ${error.message}`;
                 throw error;
             }
 
             try {
                 SourceCode.prototype.getComments = getCommentsDeprecation;
                 messages = linter.verify(code, configs, filename);
+            } catch (err) {
+                if (item.error && err.messageForTest) {
+
+                    // If this was a testable exception and we're executing a fatal test case, return it so the error can be compared against in the test case.
+                    return {
+                        messages: [{
+                            ruleId: ruleName,
+                            fatal: true,
+                            message: err.messageForTest,
+                            name: err.name === "Error" ? err.constructor.name : err.name
+                        }]
+                    };
+                }
+
+                // Not testing an exception.
+                throw err;
             } finally {
                 SourceCode.prototype.getComments = getComments;
             }
@@ -1009,6 +1069,55 @@ class FlatRuleTester {
             assertASTDidntChange(result.beforeAST, result.afterAST);
         }
 
+        /**
+         * Check if the template is fatal or not.
+         * All fatal cases go through this.
+         * @param {string|Object} item Item to run the rule against
+         * @returns {void}
+         * @private
+         */
+        function testFatalTemplate(item) {
+            if (item.code) {
+                assert.ok(typeof item.code === "string", "Optional test case property 'code' must be a string");
+            }
+            if (item.name) {
+                assert.ok(typeof item.name === "string", "Optional test case property 'name' must be a string");
+            }
+            assert.ok(typeof item.error === "object",
+                "Test case property 'error' must be an object");
+
+            assert.ok(typeof item.error.name === "string" || typeof item.error.message === "string" || item.error.message instanceof RegExp,
+                "Test case property 'error' object must provide a string 'name' property or string/regexp 'message' property");
+
+            assert.ok(typeof item.code === "string" || typeof item.options !== "undefined", "Test case must contain at least one of `code` and `options`");
+
+            const result = runRuleForItem(item);
+
+            assert.ok(result.messages.length === 1 && result.messages[0].fatal === true, "A fatal test case must throw an exception");
+
+            const errorActual = result.messages[0];
+            const errorExpected = item.error;
+
+            Object.keys(errorExpected).forEach(propertyName => {
+                assert.ok(
+                    fatalErrorObjectParameters.has(propertyName),
+                    `Invalid error property name '${propertyName}'. Expected one of ${friendlyFatalErrorObjectParameterList}.`
+                );
+            });
+
+            if (hasOwnProperty(errorExpected, "name")) {
+                assert.strictEqual(
+                    errorActual.name,
+                    errorExpected.name,
+                    `Fatal error name should be "${errorExpected.name}" but got "${errorActual.name}" instead.`
+                );
+            }
+
+            if (hasOwnProperty(errorExpected, "message")) {
+                assertMessageMatches(errorActual.message, errorExpected.message);
+            }
+        }
+
         /*
          * This creates a mocha test suite and pipes all supplied info through
          * one of the templates above.
@@ -1035,6 +1144,17 @@ class FlatRuleTester {
                     );
                 });
             });
+
+            this.constructor.describe("fatal", () => {
+                (test.fatal || []).forEach((fatal, index) => {
+                    this.constructor[fatal.only ? "itOnly" : "it"](
+                        sanitize(fatal.name || fatal.code || `(Test Case #${index + 1})`),
+                        () => {
+                            testFatalTemplate(fatal);
+                        }
+                    );
+                });
+            });
         });
     }
 }