Set up peggy and connect it with jest/webpack, set up a basic parser and test it

.gitignore

@@ -4,6 +4,7 @@
 /node_modules
 /.pnp
 .pnp.js
+/src/scripting/NomScript.ts
 
 # testing
 /coverage

.idea/codeStyles/codeStyleConfig.xml

@@ -0,0 +1,5 @@
+<component name="ProjectCodeStyleConfiguration">
+  <state>
+    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
+  </state>
+</component>

.idea/runConfigurations/All_Tests.xml

@@ -2,7 +2,7 @@
   <configuration default="false" name="All Tests" type="JavaScriptTestRunnerJest" nameIsGenerated="true">
     <node-interpreter value="project" />
     <node-options value="" />
-    <jest-package value="$PROJECT_DIR$/node_modules/react-scripts" />
+    <jest-package value="$PROJECT_DIR$/node_modules/jest" />
     <working-dir value="$PROJECT_DIR$" />
     <envs />
     <scope-kind value="ALL" />

config/jest/peggyTransform.js

@@ -0,0 +1,53 @@
+'use strict';
+
+const babelJest = require('babel-jest');
+const makeCacheKey = require('@jest/create-cache-key-function').default();
+const peggy = require("peggy");
+
+const hasJsxRuntime = (() => {
+    if (process.env.DISABLE_NEW_JSX_TRANSFORM === 'true') {
+        return false;
+    }
+
+    try {
+        require.resolve('react/jsx-runtime');
+        return true;
+    } catch (e) {
+        return false;
+    }
+})();
+
+const babelJestInstance = babelJest.createTransformer({
+    presets: [
+        [
+            require.resolve('babel-preset-react-app'),
+            {
+                runtime: hasJsxRuntime ? 'automatic' : 'classic',
+            },
+        ],
+    ],
+    babelrc: false,
+    configFile: false,
+});
+
+const startRuleFinder = /\s*\/\/\s*peggy-loader:startRule\s*(\S+)\s*\n/g
+
+module.exports = {
+    canInstrument: babelJestInstance.canInstrument,
+    createTransformer() {
+        return this;
+    },
+    getCacheKey(sourceText, sourcePath, configString, options) {
+        return makeCacheKey(sourceText, sourcePath, configString, options) + babelJestInstance.getCacheKey(sourceText, sourcePath, configString, options)
+    },
+    process(sourceText, sourcePath, configString, options) {
+        const start = Array.from({ [Symbol.iterator]() { return sourceText.matchAll(startRuleFinder) } }, ([_, rule]) => rule)
+        const resultSource = peggy.generate(sourceText, {
+            grammarSource: this.resourcePath,
+            allowedStartRules: start,
+            output: "source",
+            format: "es",
+        })
+        return babelJest.process(resultSource, sourcePath, configString, options)
+    },
+}

config/peggy-loader.js

@@ -0,0 +1,37 @@
+const peggy = require("peggy")
+const loaderUtils = require("loader-utils")
+
+const startRuleFinder = /\s*\/\/\s*peggy-loader:startRule\s*(\S+)\s*\n/g
+
+async function loadPeggy(rawSource) {
+    const callback = this.async();
+    const tracks = await import("peggy-tracks")
+    const rawOpts = loaderUtils.getOptions(this)
+    const opts = typeof rawOpts === "object" && rawOpts !== null ? rawOpts : {}
+    const peggyOpts = opts.hasOwnProperty("peggy") && typeof opts.peggy === "object" && opts.peggy !== null ? opts.peggy : {}
+    const plugins = peggyOpts.hasOwnProperty("plugins") && Array.isArray(peggyOpts.plugins) ? peggyOpts.plugins : []
+    const tracksOpts = opts.hasOwnProperty("tracks") && typeof opts.tracks === "object" && opts.tracks !== null ? opts.tracks : {start: []}
+    const start = Array.from({ [Symbol.iterator]() { return rawSource.matchAll(startRuleFinder) } }, ([_, rule]) => rule)
+    const trackStarts = !opts.hasOwnProperty("tracks") ? [] : typeof tracksOpts.start === "string" ? [tracksOpts.start] : Array.isArray(tracksOpts.start) ? tracksOpts.start : start.length > 0 ? start : [undefined]
+    const source = rawSource.replaceAll(startRuleFinder, "\n")
+    const resultSource = peggy.generate(source, {
+        ...peggyOpts,
+        grammarSource: this.resourcePath,
+        allowedStartRules: peggyOpts.allowedStartRules || start,
+        output: "source",
+        format: "es",
+        plugins,
+    })
+    for (const start of trackStarts) {
+        const diagram = tracks.tracks({
+            ...tracksOpts,
+            text: source,
+            start,
+        })
+        this.emitFile(`/tracks/${this.resourcePath}${typeof start === "string" ? "." + start : ""}.svg`, diagram.toStandalone(tracks.defaultCSS))
+    }
+    // const fixedSource = resultSource.replace("export class SyntaxError", "class SyntaxError")
+    callback(undefined, resultSource)
+}
+
+module.exports = loadPeggy

config/webpack.config.js

@@ -389,6 +389,51 @@ module.exports = function (webpackEnv) {
                 name: 'static/media/[name].[hash:8].[ext]',
               },
             },
+            // Process parser definitions with Peggy, using the TS plugin.
+            {
+              test: /\.peggy$/,
+              include: paths.appSrc,
+              use: [
+                {
+                  loader: require.resolve('babel-loader'),
+                  options: {
+                    customize: require.resolve(
+                        'babel-preset-react-app/webpack-overrides'
+                    ),
+                    presets: [
+                      [ require.resolve('babel-preset-react-app') ],
+                    ],
+
+                    plugins: [
+                      [
+                        require.resolve('babel-plugin-named-asset-import'),
+                        {
+                          loaderMap: {
+                            svg: {
+                              ReactComponent:
+                                  '@svgr/webpack?-svgo,+titleProp,+ref![path]',
+                            },
+                          },
+                        },
+                      ],
+                      isEnvDevelopment &&
+                      shouldUseReactRefresh &&
+                      require.resolve('react-refresh/babel'),
+                    ].filter(Boolean),
+                    // This is a feature of `babel-loader` for webpack (not Babel itself).
+                    // It enables caching results in ./node_modules/.cache/babel-loader/
+                    // directory for faster rebuilds.
+                    cacheDirectory: true,
+                    // See #6846 for context on why cacheCompression is disabled
+                    cacheCompression: false,
+                    compact: isEnvProduction,
+                  },
+                },
+                {
+                  loader: require.resolve('./peggy-loader'),
+                },
+              ],
+            },
             // Process application JS with Babel.
             // The preset includes JSX, Flow, TypeScript, and some ESnext features.
             {

package.json

@@ -4,6 +4,8 @@
   "private": true,
   "dependencies": {
     "@babel/core": "7.12.3",
+    "@babel/preset-typescript": "^7.15.0",
+    "@jest/create-cache-key-function": "^27.3.1",
     "@pmmmwh/react-refresh-webpack-plugin": "0.4.3",
     "@svgr/webpack": "5.5.0",
     "@testing-library/jest-dom": "^5.14.1",
@@ -44,14 +46,18 @@
     "jest-circus": "26.6.0",
     "jest-resolve": "26.6.0",
     "jest-watch-typeahead": "0.6.1",
+    "loader-utils": "^2.0.0",
     "mini-css-extract-plugin": "0.11.3",
     "optimize-css-assets-webpack-plugin": "5.0.4",
+    "peggy": "^1.2.0",
+    "peggy-tracks": "^1.1.0",
     "pnp-webpack-plugin": "1.6.4",
     "postcss-flexbugs-fixes": "4.2.1",
     "postcss-loader": "3.0.0",
     "postcss-normalize": "8.0.1",
     "postcss-preset-env": "6.7.0",
     "postcss-safe-parser": "5.0.2",
+    "prettier": "^2.4.1",
     "prompts": "2.4.0",
     "react": "^17.0.2",
     "react-app-polyfill": "^2.0.0",
@@ -62,6 +68,7 @@
     "resolve-url-loader": "^3.1.2",
     "sass-loader": "^10.0.5",
     "semver": "7.3.2",
+    "serve": "^12.0.1",
     "style-loader": "1.3.0",
     "terser-webpack-plugin": "4.2.3",
     "ts-pnp": "1.2.0",
@@ -76,7 +83,8 @@
   "scripts": {
     "start": "node scripts/start.js",
     "build": "node scripts/build.js",
-    "test": "node scripts/test.js"
+    "test": "node scripts/test.js",
+    "serve": "serve -s build"
   },
   "eslintConfig": {
     "extends": [
@@ -101,7 +109,7 @@
       "<rootDir>/src"
     ],
     "collectCoverageFrom": [
-      "src/**/*.{js,jsx,ts,tsx}",
+      "src/**/*.{js,jsx,ts,tsx,peggy}",
       "!src/**/*.d.ts"
     ],
     "setupFiles": [
@@ -117,9 +125,10 @@
     "testEnvironment": "jsdom",
     "testRunner": "<rootDir>/node_modules/jest-circus/runner.js",
     "transform": {
+      "^.+\\.peggy$": "<rootDir>/config/jest/peggyTransform.js",
       "^.+\\.(js|jsx|mjs|cjs|ts|tsx)$": "<rootDir>/config/jest/babelTransform.js",
       "^.+\\.css$": "<rootDir>/config/jest/cssTransform.js",
-      "^(?!.*\\.(js|jsx|mjs|cjs|ts|tsx|css|json)$)": "<rootDir>/config/jest/fileTransform.js"
+      "^(?!.*\\.(js|jsx|mjs|cjs|ts|tsx|css|json|peggy)$)": "<rootDir>/config/jest/fileTransform.js"
     },
     "transformIgnorePatterns": [
       "[/\\\\]node_modules[/\\\\].+\\.(js|jsx|mjs|cjs|ts|tsx)$",
@@ -131,6 +140,7 @@
       "^.+\\.module\\.(css|sass|scss)$": "identity-obj-proxy"
     },
     "moduleFileExtensions": [
+      "peggy",
       "web.js",
       "js",
       "web.ts",

src/index.tsx

@@ -3,7 +3,9 @@ import ReactDOM from 'react-dom';
 import './index.css';
 import App from './App';
 import reportWebVitals from './reportWebVitals';
+import {parse} from './scripting/NomScript.peggy';
 
+console.log(parse)
 ReactDOM.render(
   <React.StrictMode>
     <App />

src/scripting/NomScript.peggy

@@ -0,0 +1,30 @@
+{{
+    // global (one-time) initializer
+    import { scriptFile } from "./ScriptFile";
+    import { numberValue } from "./ScriptValue";
+    import { versionStatement } from "./TopLevelStatement";
+}}
+
+{
+    // per-parse initializer
+}
+
+// peggy-loader:startRule ScriptFile
+ScriptFile = statements:TopLevelStatementList { return scriptFile(statements); }
+
+NewLine "newline" = "\n" { return null; }
+WhiteSpace "whitespace" = [ \r\v\t]+ { return null; }
+LineComment "line comment" = "//" [^\n]* { return null; }
+BlockCommentTail = "*/" / ( "*" !"/" [^*]* BlockCommentTail ) { return null; }
+BlockComment "block comment" = "/*" [^*]* BlockCommentTail { return null; }
+
+IntegerLiteral "integer literal" = digits:$[0-9]+ { return numberValue(digits); }
+
+NonToken = (LineComment / BlockComment / WhiteSpace) { return null; }
+_ = (NonToken+) { return null; }
+StatementEnd = _? NewLine { return null; }
+
+VersionStatement "version statement" = "script" _ "version" _ version:IntegerLiteral { return versionStatement(version.value); }
+
+TopLevelStatement "top-level statement" = VersionStatement
+TopLevelStatementList = (StatementEnd* _? head:TopLevelStatement tail:(StatementEnd+ _? @TopLevelStatement)* StatementEnd* _? { return [head, ...tail]; }) / (StatementEnd* _? { return []; })

src/scripting/NomScript.peggy.d.ts

@@ -0,0 +1,34 @@
+import {ScriptFile} from "./ScriptValue.js"
+
+export interface Options {
+    /** Whether this is a script or an expression. Required. */
+    readonly start: "ScriptFile"|"Expression"
+    /** The file (or other source) the text is being parsed from. Required.*/
+    readonly grammarSource: string
+}
+
+export interface Position {
+    readonly offset: number
+    readonly line: number
+    readonly column: number
+}
+
+export interface Location {
+    readonly source: string
+    readonly start: Position
+    readonly end: Position
+}
+
+// For later:
+//export function parse(text: string, options: Options & {readonly start: "Expression"}): Expression
+export function parse(text: string, options: Options & {readonly start: "ScriptFile"}): ScriptFile
+
+export class SyntaxError extends Error {
+    readonly name: "SyntaxError"
+    readonly message: string
+    readonly expected: (readonly string[])|null
+    readonly found: string|null
+    readonly location: Location
+    constructor(message: string, expected: (readonly string[])|null, found: string|null, location: Location)
+    format(mappings: readonly {readonly source: string, readonly text: string}[]): string
+}

src/scripting/NomScript.test.ts

@@ -0,0 +1,83 @@
+import { parse, SyntaxError } from "./NomScript.peggy";
+
+describe("parse", () => {
+  describe("ScriptFile", () => {
+    test("works with an empty input", () => {
+      expect(() => parse("", {grammarSource: "testData", start: "ScriptFile"})).not.toThrow()
+    });
+    test("works with empty lines", () => {
+      expect(() => parse("\n\n\n\n\n", {grammarSource: "testData", start: "ScriptFile"})).not.toThrow()
+    });
+    test("works with lines filled only with whitespace", () => {
+      expect(() => parse("\n\n       \n\t\t    \t\n\n", {grammarSource: "testData", start: "ScriptFile"})).not.toThrow()
+    });
+    test("works with lines filled only with comments", () => {
+      expect(() => parse("\n\n    /*      */ \n// test \n /*     */\n", {grammarSource: "testData", start: "ScriptFile"})).not.toThrow()
+    });
+    test("throws SyntaxError for invalid text", () => {
+      expect(() => parse("!! invalid !!", {grammarSource: "testData", start: "ScriptFile"})).toThrow(SyntaxError)
+    });
+  })
+});
+
+describe("SyntaxError", () => {
+  describe("constructor", () => {
+    test("populates the members accordingly", () => {
+      const location = {
+        source: "source",
+        start: {
+          offset: 0,
+          line: 1,
+          column: 1,
+        },
+        end: {
+          offset: 7,
+          line: 1,
+          column: 8,
+        },
+      };
+      const err = new SyntaxError("message", ["expected"], "found", location);
+
+      expect(err.name).toEqual("SyntaxError");
+      expect(err.message).toEqual("message");
+      expect(err.expected).toEqual(["expected"]);
+      expect(err.found).toEqual("found");
+      expect(err.location).toEqual(location);
+    });
+  });
+  describe("format", () => {
+    test("displays a pretty error message for the failure", () => {
+      const location = {
+        source: "source",
+        start: {
+          offset: 0,
+          line: 1,
+          column: 1,
+        },
+        end: {
+          offset: 7,
+          line: 1,
+          column: 8,
+        },
+      };
+      const err = new SyntaxError(
+        "message",
+        ["valid", "cool"],
+        "errored",
+        location
+      );
+      const result = err.format([
+        { source: "source", text: "errored text" },
+        { source: "source2", text: "irrelevant" },
+      ]);
+
+      expect(result).toMatchInlineSnapshot(`
+        "Error: message
+         --> source:1:1
+          |
+        1 | errored text
+          | ^^^^^^^"
+      `);
+    });
+  });
+});

src/scripting/ScriptFile.ts

@@ -0,0 +1,10 @@
+import {TopLevelStatement} from "./TopLevelStatement.js";
+
+export interface ScriptFile {
+    readonly statements: readonly TopLevelStatement[]
+}
+export function scriptFile(statements: readonly TopLevelStatement[]): ScriptFile {
+    return {
+        statements
+    }
+}

src/scripting/ScriptValue.test.ts

@@ -0,0 +1,29 @@
+import {isNumberValue, numberValue, ScriptValueTypes} from "./ScriptValue";
+
+describe("NumberValue", () => {
+    describe("constructor", () => {
+        test("forwards number parameter to the value property", () => {
+            expect(numberValue(5)).toEqual({
+                type: ScriptValueTypes.NUMBER,
+                value: 5
+            })
+        })
+        test("parses string parameter to the value property", () => {
+            expect(numberValue("99")).toEqual({
+                type: ScriptValueTypes.NUMBER,
+                value: 99
+            })
+        })
+    })
+    describe("typecheck", () => {
+        test("passes on the output of the constructor", () => {
+            expect(isNumberValue(numberValue(2))).toBeTruthy()
+        })
+        test("passes on a hand-constructed instance", () => {
+            expect(isNumberValue({
+                type: ScriptValueTypes.NUMBER,
+                value: 21
+            })).toBeTruthy()
+        })
+    })
+})

src/scripting/ScriptValue.ts

@@ -0,0 +1,26 @@
+export enum ScriptValueTypes {
+    NUMBER = "number"
+}
+
+export interface NumberValue {
+    readonly type: ScriptValueTypes.NUMBER,
+    readonly value: number,
+}
+export function numberValue(value: number|string) {
+    if (typeof value === "string") {
+        return {
+            type: ScriptValueTypes.NUMBER,
+            value: parseInt(value, 10),
+        }
+    } else {
+        return {
+            type: ScriptValueTypes.NUMBER,
+            value,
+        }
+    }
+}
+export function isNumberValue(value: ScriptValue): value is NumberValue {
+    return value.type === ScriptValueTypes.NUMBER
+}
+
+export type ScriptValue = NumberValue

src/scripting/TopLevelStatement.test.ts

@@ -0,0 +1,59 @@
+import {parse as parseInternal, SyntaxError} from "./NomScript.peggy"
+import {isVersionStatement, TopLevelStatementTypes, VersionStatement, versionStatement} from "./TopLevelStatement";
+import {scriptFile, ScriptFile} from "./ScriptFile";
+
+function parse(text: string): ScriptFile {
+    return parseInternal(text, {start: "ScriptFile", grammarSource: "testData"})
+}
+
+describe("VersionStatement", () => {
+    describe("constructor", () => {
+        test("forwards parameter to the version property", () => {
+            expect(versionStatement(5)).toEqual({
+                type: TopLevelStatementTypes.VERSION,
+                version: 5
+            })
+        })
+    })
+    describe("typecheck", () => {
+        test("passes on the output of the constructor", () => {
+            expect(isVersionStatement(versionStatement(2))).toBeTruthy()
+        })
+        test("passes on a hand-constructed instance", () => {
+            expect(isVersionStatement({
+                type: TopLevelStatementTypes.VERSION,
+                version: 5
+            })).toBeTruthy()
+        })
+    })
+    describe("parsing", () => {
+        function success(name: string, text: string, ...result: VersionStatement[]) {
+            test(`succeeds for ${name}`, () => {
+                expect(parse(text)).toEqual(scriptFile(result))
+            })
+        }
+
+        success("basic example", "script version 1", versionStatement(1))
+        success("followed by line comment", "script version 3 // and nine tenths", versionStatement(3))
+        success("preceded by block comment", "/* just because it's a */ script version 2", versionStatement(2))
+        success("preceded by line comment", "// line comment\nscript version 9999", versionStatement(9999))
+        success("preceded by spaces", "\n\n\n\t\t\tscript version 2", versionStatement(2))
+        success("separated by tabs instead of spaces", "script\tversion\t4", versionStatement(4))
+        success("separated by comments instead of spaces", "script/* or candy */version/* if it can be called that */4", versionStatement(4))
+        success("separated by multiline comments instead of spaces", "script/* ripped\n */version/* mergin\n */4", versionStatement(4))
+        success("doubled on separate lines", "script version 1\nscript version 2", versionStatement(1), versionStatement(2))
+
+        function failure(name: string, text: string) {
+            test(`fails for ${name}`, () => {
+                expect(() => parse(text)).toThrow(SyntaxError)
+            })
+        }
+
+        failure("separated by newlines", "script\nversion\n1")
+        failure("not separated", "scriptversion1")
+        failure("wrong order", "version script 1")
+        failure("decimal version", "version script 1.5")
+        failure("followed by garbage", "script version 1 and a half")
+        failure("doubled on same line", "script version 1 script version 1")
+    })
+})

src/scripting/TopLevelStatement.ts

@@ -0,0 +1,19 @@
+export enum TopLevelStatementTypes {
+    VERSION = "version",
+}
+
+export interface VersionStatement {
+    readonly type: TopLevelStatementTypes.VERSION,
+    readonly version: number,
+}
+export function versionStatement(version: number): VersionStatement {
+    return {
+        type: TopLevelStatementTypes.VERSION,
+        version,
+    }
+}
+export function isVersionStatement(statement: TopLevelStatement): statement is VersionStatement {
+    return statement.type === TopLevelStatementTypes.VERSION
+}
+
+export type TopLevelStatement = VersionStatement

src/testing/Assertions.ts

@@ -13,7 +13,7 @@ export namespace AssertionConfig {
     /** An assertion configuration suitable for testing, that throws an exception when an assertion fails. */
     export const Throw = {mode: AssertionMode.THROW} as const
     /** An assertion configuration suitable for debugging, that logs an exception when an assertion fails but allows the program to continue. */
-    export function Log(logger: (message?: any, ...params: any[]) => void) {
+    export function Log(logger: (message?: unknown, ...params: unknown[]) => void) {
         return {mode: AssertionMode.LOG, logger} as const
     }
 }

tsconfig.json

@@ -11,6 +11,7 @@
     "esModuleInterop": true,
     "allowSyntheticDefaultImports": true,
     "strict": true,
+    "strictNullChecks": true,
     "forceConsistentCasingInFileNames": true,
     "noFallthroughCasesInSwitch": true,
     "module": "esnext",