Set up peggy and connect it with jest/webpack, set up a basic parser and test it

main
Mari 2 years ago
parent 85b36aa1b1
commit 721364abd4
  1. 1
      .gitignore
  2. 5
      .idea/codeStyles/codeStyleConfig.xml
  3. 2
      .idea/runConfigurations/All_Tests.xml
  4. 53
      config/jest/peggyTransform.js
  5. 37
      config/peggy-loader.js
  6. 45
      config/webpack.config.js
  7. 24343
      package-lock.json
  8. 16
      package.json
  9. 2
      src/index.tsx
  10. 30
      src/scripting/NomScript.peggy
  11. 34
      src/scripting/NomScript.peggy.d.ts
  12. 83
      src/scripting/NomScript.test.ts
  13. 10
      src/scripting/ScriptFile.ts
  14. 29
      src/scripting/ScriptValue.test.ts
  15. 26
      src/scripting/ScriptValue.ts
  16. 59
      src/scripting/TopLevelStatement.test.ts
  17. 19
      src/scripting/TopLevelStatement.ts
  18. 2
      src/testing/Assertions.ts
  19. 1
      tsconfig.json

1
.gitignore vendored

@ -4,6 +4,7 @@
/node_modules
/.pnp
.pnp.js
/src/scripting/NomScript.ts
# testing
/coverage

@ -0,0 +1,5 @@
<component name="ProjectCodeStyleConfiguration">
<state>
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
</state>
</component>

@ -2,7 +2,7 @@
<configuration default="false" name="All Tests" type="JavaScriptTestRunnerJest" nameIsGenerated="true">
<node-interpreter value="project" />
<node-options value="" />
<jest-package value="$PROJECT_DIR$/node_modules/react-scripts" />
<jest-package value="$PROJECT_DIR$/node_modules/jest" />
<working-dir value="$PROJECT_DIR$" />
<envs />
<scope-kind value="ALL" />

@ -0,0 +1,53 @@
'use strict';
const babelJest = require('babel-jest');
const makeCacheKey = require('@jest/create-cache-key-function').default();
const peggy = require("peggy");
const hasJsxRuntime = (() => {
if (process.env.DISABLE_NEW_JSX_TRANSFORM === 'true') {
return false;
}
try {
require.resolve('react/jsx-runtime');
return true;
} catch (e) {
return false;
}
})();
const babelJestInstance = babelJest.createTransformer({
presets: [
[
require.resolve('babel-preset-react-app'),
{
runtime: hasJsxRuntime ? 'automatic' : 'classic',
},
],
],
babelrc: false,
configFile: false,
});
const startRuleFinder = /\s*\/\/\s*peggy-loader:startRule\s*(\S+)\s*\n/g
module.exports = {
canInstrument: babelJestInstance.canInstrument,
createTransformer() {
return this;
},
getCacheKey(sourceText, sourcePath, configString, options) {
return makeCacheKey(sourceText, sourcePath, configString, options) + babelJestInstance.getCacheKey(sourceText, sourcePath, configString, options)
},
process(sourceText, sourcePath, configString, options) {
const start = Array.from({ [Symbol.iterator]() { return sourceText.matchAll(startRuleFinder) } }, ([_, rule]) => rule)
const resultSource = peggy.generate(sourceText, {
grammarSource: this.resourcePath,
allowedStartRules: start,
output: "source",
format: "es",
})
return babelJest.process(resultSource, sourcePath, configString, options)
},
}

@ -0,0 +1,37 @@
const peggy = require("peggy")
const loaderUtils = require("loader-utils")
const startRuleFinder = /\s*\/\/\s*peggy-loader:startRule\s*(\S+)\s*\n/g
async function loadPeggy(rawSource) {
const callback = this.async();
const tracks = await import("peggy-tracks")
const rawOpts = loaderUtils.getOptions(this)
const opts = typeof rawOpts === "object" && rawOpts !== null ? rawOpts : {}
const peggyOpts = opts.hasOwnProperty("peggy") && typeof opts.peggy === "object" && opts.peggy !== null ? opts.peggy : {}
const plugins = peggyOpts.hasOwnProperty("plugins") && Array.isArray(peggyOpts.plugins) ? peggyOpts.plugins : []
const tracksOpts = opts.hasOwnProperty("tracks") && typeof opts.tracks === "object" && opts.tracks !== null ? opts.tracks : {start: []}
const start = Array.from({ [Symbol.iterator]() { return rawSource.matchAll(startRuleFinder) } }, ([_, rule]) => rule)
const trackStarts = !opts.hasOwnProperty("tracks") ? [] : typeof tracksOpts.start === "string" ? [tracksOpts.start] : Array.isArray(tracksOpts.start) ? tracksOpts.start : start.length > 0 ? start : [undefined]
const source = rawSource.replaceAll(startRuleFinder, "\n")
const resultSource = peggy.generate(source, {
...peggyOpts,
grammarSource: this.resourcePath,
allowedStartRules: peggyOpts.allowedStartRules || start,
output: "source",
format: "es",
plugins,
})
for (const start of trackStarts) {
const diagram = tracks.tracks({
...tracksOpts,
text: source,
start,
})
this.emitFile(`/tracks/${this.resourcePath}${typeof start === "string" ? "." + start : ""}.svg`, diagram.toStandalone(tracks.defaultCSS))
}
// const fixedSource = resultSource.replace("export class SyntaxError", "class SyntaxError")
callback(undefined, resultSource)
}
module.exports = loadPeggy

@ -389,6 +389,51 @@ module.exports = function (webpackEnv) {
name: 'static/media/[name].[hash:8].[ext]',
},
},
// Process parser definitions with Peggy, using the TS plugin.
{
test: /\.peggy$/,
include: paths.appSrc,
use: [
{
loader: require.resolve('babel-loader'),
options: {
customize: require.resolve(
'babel-preset-react-app/webpack-overrides'
),
presets: [
[ require.resolve('babel-preset-react-app') ],
],
plugins: [
[
require.resolve('babel-plugin-named-asset-import'),
{
loaderMap: {
svg: {
ReactComponent:
'@svgr/webpack?-svgo,+titleProp,+ref![path]',
},
},
},
],
isEnvDevelopment &&
shouldUseReactRefresh &&
require.resolve('react-refresh/babel'),
].filter(Boolean),
// This is a feature of `babel-loader` for webpack (not Babel itself).
// It enables caching results in ./node_modules/.cache/babel-loader/
// directory for faster rebuilds.
cacheDirectory: true,
// See #6846 for context on why cacheCompression is disabled
cacheCompression: false,
compact: isEnvProduction,
},
},
{
loader: require.resolve('./peggy-loader'),
},
],
},
// Process application JS with Babel.
// The preset includes JSX, Flow, TypeScript, and some ESnext features.
{

24343
package-lock.json generated

File diff suppressed because it is too large Load Diff

@ -4,6 +4,8 @@
"private": true,
"dependencies": {
"@babel/core": "7.12.3",
"@babel/preset-typescript": "^7.15.0",
"@jest/create-cache-key-function": "^27.3.1",
"@pmmmwh/react-refresh-webpack-plugin": "0.4.3",
"@svgr/webpack": "5.5.0",
"@testing-library/jest-dom": "^5.14.1",
@ -44,14 +46,18 @@
"jest-circus": "26.6.0",
"jest-resolve": "26.6.0",
"jest-watch-typeahead": "0.6.1",
"loader-utils": "^2.0.0",
"mini-css-extract-plugin": "0.11.3",
"optimize-css-assets-webpack-plugin": "5.0.4",
"peggy": "^1.2.0",
"peggy-tracks": "^1.1.0",
"pnp-webpack-plugin": "1.6.4",
"postcss-flexbugs-fixes": "4.2.1",
"postcss-loader": "3.0.0",
"postcss-normalize": "8.0.1",
"postcss-preset-env": "6.7.0",
"postcss-safe-parser": "5.0.2",
"prettier": "^2.4.1",
"prompts": "2.4.0",
"react": "^17.0.2",
"react-app-polyfill": "^2.0.0",
@ -62,6 +68,7 @@
"resolve-url-loader": "^3.1.2",
"sass-loader": "^10.0.5",
"semver": "7.3.2",
"serve": "^12.0.1",
"style-loader": "1.3.0",
"terser-webpack-plugin": "4.2.3",
"ts-pnp": "1.2.0",
@ -76,7 +83,8 @@
"scripts": {
"start": "node scripts/start.js",
"build": "node scripts/build.js",
"test": "node scripts/test.js"
"test": "node scripts/test.js",
"serve": "serve -s build"
},
"eslintConfig": {
"extends": [
@ -101,7 +109,7 @@
"<rootDir>/src"
],
"collectCoverageFrom": [
"src/**/*.{js,jsx,ts,tsx}",
"src/**/*.{js,jsx,ts,tsx,peggy}",
"!src/**/*.d.ts"
],
"setupFiles": [
@ -117,9 +125,10 @@
"testEnvironment": "jsdom",
"testRunner": "<rootDir>/node_modules/jest-circus/runner.js",
"transform": {
"^.+\\.peggy$": "<rootDir>/config/jest/peggyTransform.js",
"^.+\\.(js|jsx|mjs|cjs|ts|tsx)$": "<rootDir>/config/jest/babelTransform.js",
"^.+\\.css$": "<rootDir>/config/jest/cssTransform.js",
"^(?!.*\\.(js|jsx|mjs|cjs|ts|tsx|css|json)$)": "<rootDir>/config/jest/fileTransform.js"
"^(?!.*\\.(js|jsx|mjs|cjs|ts|tsx|css|json|peggy)$)": "<rootDir>/config/jest/fileTransform.js"
},
"transformIgnorePatterns": [
"[/\\\\]node_modules[/\\\\].+\\.(js|jsx|mjs|cjs|ts|tsx)$",
@ -131,6 +140,7 @@
"^.+\\.module\\.(css|sass|scss)$": "identity-obj-proxy"
},
"moduleFileExtensions": [
"peggy",
"web.js",
"js",
"web.ts",

@ -3,7 +3,9 @@ import ReactDOM from 'react-dom';
import './index.css';
import App from './App';
import reportWebVitals from './reportWebVitals';
import {parse} from './scripting/NomScript.peggy';
console.log(parse)
ReactDOM.render(
<React.StrictMode>
<App />

@ -0,0 +1,30 @@
{{
// global (one-time) initializer
import { scriptFile } from "./ScriptFile";
import { numberValue } from "./ScriptValue";
import { versionStatement } from "./TopLevelStatement";
}}
{
// per-parse initializer
}
// peggy-loader:startRule ScriptFile
ScriptFile = statements:TopLevelStatementList { return scriptFile(statements); }
NewLine "newline" = "\n" { return null; }
WhiteSpace "whitespace" = [ \r\v\t]+ { return null; }
LineComment "line comment" = "//" [^\n]* { return null; }
BlockCommentTail = "*/" / ( "*" !"/" [^*]* BlockCommentTail ) { return null; }
BlockComment "block comment" = "/*" [^*]* BlockCommentTail { return null; }
IntegerLiteral "integer literal" = digits:$[0-9]+ { return numberValue(digits); }
NonToken = (LineComment / BlockComment / WhiteSpace) { return null; }
_ = (NonToken+) { return null; }
StatementEnd = _? NewLine { return null; }
VersionStatement "version statement" = "script" _ "version" _ version:IntegerLiteral { return versionStatement(version.value); }
TopLevelStatement "top-level statement" = VersionStatement
TopLevelStatementList = (StatementEnd* _? head:TopLevelStatement tail:(StatementEnd+ _? @TopLevelStatement)* StatementEnd* _? { return [head, ...tail]; }) / (StatementEnd* _? { return []; })

@ -0,0 +1,34 @@
import {ScriptFile} from "./ScriptValue.js"
export interface Options {
/** Whether this is a script or an expression. Required. */
readonly start: "ScriptFile"|"Expression"
/** The file (or other source) the text is being parsed from. Required.*/
readonly grammarSource: string
}
export interface Position {
readonly offset: number
readonly line: number
readonly column: number
}
export interface Location {
readonly source: string
readonly start: Position
readonly end: Position
}
// For later:
//export function parse(text: string, options: Options & {readonly start: "Expression"}): Expression
export function parse(text: string, options: Options & {readonly start: "ScriptFile"}): ScriptFile
export class SyntaxError extends Error {
readonly name: "SyntaxError"
readonly message: string
readonly expected: (readonly string[])|null
readonly found: string|null
readonly location: Location
constructor(message: string, expected: (readonly string[])|null, found: string|null, location: Location)
format(mappings: readonly {readonly source: string, readonly text: string}[]): string
}

@ -0,0 +1,83 @@
import { parse, SyntaxError } from "./NomScript.peggy";
describe("parse", () => {
describe("ScriptFile", () => {
test("works with an empty input", () => {
expect(() => parse("", {grammarSource: "testData", start: "ScriptFile"})).not.toThrow()
});
test("works with empty lines", () => {
expect(() => parse("\n\n\n\n\n", {grammarSource: "testData", start: "ScriptFile"})).not.toThrow()
});
test("works with lines filled only with whitespace", () => {
expect(() => parse("\n\n \n\t\t \t\n\n", {grammarSource: "testData", start: "ScriptFile"})).not.toThrow()
});
test("works with lines filled only with comments", () => {
expect(() => parse("\n\n /* */ \n// test \n /* */\n", {grammarSource: "testData", start: "ScriptFile"})).not.toThrow()
});
test("throws SyntaxError for invalid text", () => {
expect(() => parse("!! invalid !!", {grammarSource: "testData", start: "ScriptFile"})).toThrow(SyntaxError)
});
})
});
describe("SyntaxError", () => {
describe("constructor", () => {
test("populates the members accordingly", () => {
const location = {
source: "source",
start: {
offset: 0,
line: 1,
column: 1,
},
end: {
offset: 7,
line: 1,
column: 8,
},
};
const err = new SyntaxError("message", ["expected"], "found", location);
expect(err.name).toEqual("SyntaxError");
expect(err.message).toEqual("message");
expect(err.expected).toEqual(["expected"]);
expect(err.found).toEqual("found");
expect(err.location).toEqual(location);
});
});
describe("format", () => {
test("displays a pretty error message for the failure", () => {
const location = {
source: "source",
start: {
offset: 0,
line: 1,
column: 1,
},
end: {
offset: 7,
line: 1,
column: 8,
},
};
const err = new SyntaxError(
"message",
["valid", "cool"],
"errored",
location
);
const result = err.format([
{ source: "source", text: "errored text" },
{ source: "source2", text: "irrelevant" },
]);
expect(result).toMatchInlineSnapshot(`
"Error: message
--> source:1:1
|
1 | errored text
| ^^^^^^^"
`);
});
});
});

@ -0,0 +1,10 @@
import {TopLevelStatement} from "./TopLevelStatement.js";
export interface ScriptFile {
readonly statements: readonly TopLevelStatement[]
}
export function scriptFile(statements: readonly TopLevelStatement[]): ScriptFile {
return {
statements
}
}

@ -0,0 +1,29 @@
import {isNumberValue, numberValue, ScriptValueTypes} from "./ScriptValue";
describe("NumberValue", () => {
describe("constructor", () => {
test("forwards number parameter to the value property", () => {
expect(numberValue(5)).toEqual({
type: ScriptValueTypes.NUMBER,
value: 5
})
})
test("parses string parameter to the value property", () => {
expect(numberValue("99")).toEqual({
type: ScriptValueTypes.NUMBER,
value: 99
})
})
})
describe("typecheck", () => {
test("passes on the output of the constructor", () => {
expect(isNumberValue(numberValue(2))).toBeTruthy()
})
test("passes on a hand-constructed instance", () => {
expect(isNumberValue({
type: ScriptValueTypes.NUMBER,
value: 21
})).toBeTruthy()
})
})
})

@ -0,0 +1,26 @@
export enum ScriptValueTypes {
NUMBER = "number"
}
export interface NumberValue {
readonly type: ScriptValueTypes.NUMBER,
readonly value: number,
}
export function numberValue(value: number|string) {
if (typeof value === "string") {
return {
type: ScriptValueTypes.NUMBER,
value: parseInt(value, 10),
}
} else {
return {
type: ScriptValueTypes.NUMBER,
value,
}
}
}
export function isNumberValue(value: ScriptValue): value is NumberValue {
return value.type === ScriptValueTypes.NUMBER
}
export type ScriptValue = NumberValue

@ -0,0 +1,59 @@
import {parse as parseInternal, SyntaxError} from "./NomScript.peggy"
import {isVersionStatement, TopLevelStatementTypes, VersionStatement, versionStatement} from "./TopLevelStatement";
import {scriptFile, ScriptFile} from "./ScriptFile";
function parse(text: string): ScriptFile {
return parseInternal(text, {start: "ScriptFile", grammarSource: "testData"})
}
describe("VersionStatement", () => {
describe("constructor", () => {
test("forwards parameter to the version property", () => {
expect(versionStatement(5)).toEqual({
type: TopLevelStatementTypes.VERSION,
version: 5
})
})
})
describe("typecheck", () => {
test("passes on the output of the constructor", () => {
expect(isVersionStatement(versionStatement(2))).toBeTruthy()
})
test("passes on a hand-constructed instance", () => {
expect(isVersionStatement({
type: TopLevelStatementTypes.VERSION,
version: 5
})).toBeTruthy()
})
})
describe("parsing", () => {
function success(name: string, text: string, ...result: VersionStatement[]) {
test(`succeeds for ${name}`, () => {
expect(parse(text)).toEqual(scriptFile(result))
})
}
success("basic example", "script version 1", versionStatement(1))
success("followed by line comment", "script version 3 // and nine tenths", versionStatement(3))
success("preceded by block comment", "/* just because it's a */ script version 2", versionStatement(2))
success("preceded by line comment", "// line comment\nscript version 9999", versionStatement(9999))
success("preceded by spaces", "\n\n\n\t\t\tscript version 2", versionStatement(2))
success("separated by tabs instead of spaces", "script\tversion\t4", versionStatement(4))
success("separated by comments instead of spaces", "script/* or candy */version/* if it can be called that */4", versionStatement(4))
success("separated by multiline comments instead of spaces", "script/* ripped\n */version/* mergin\n */4", versionStatement(4))
success("doubled on separate lines", "script version 1\nscript version 2", versionStatement(1), versionStatement(2))
function failure(name: string, text: string) {
test(`fails for ${name}`, () => {
expect(() => parse(text)).toThrow(SyntaxError)
})
}
failure("separated by newlines", "script\nversion\n1")
failure("not separated", "scriptversion1")
failure("wrong order", "version script 1")
failure("decimal version", "version script 1.5")
failure("followed by garbage", "script version 1 and a half")
failure("doubled on same line", "script version 1 script version 1")
})
})

@ -0,0 +1,19 @@
export enum TopLevelStatementTypes {
VERSION = "version",
}
export interface VersionStatement {
readonly type: TopLevelStatementTypes.VERSION,
readonly version: number,
}
export function versionStatement(version: number): VersionStatement {
return {
type: TopLevelStatementTypes.VERSION,
version,
}
}
export function isVersionStatement(statement: TopLevelStatement): statement is VersionStatement {
return statement.type === TopLevelStatementTypes.VERSION
}
export type TopLevelStatement = VersionStatement

@ -13,7 +13,7 @@ export namespace AssertionConfig {
/** An assertion configuration suitable for testing, that throws an exception when an assertion fails. */
export const Throw = {mode: AssertionMode.THROW} as const
/** An assertion configuration suitable for debugging, that logs an exception when an assertion fails but allows the program to continue. */
export function Log(logger: (message?: any, ...params: any[]) => void) {
export function Log(logger: (message?: unknown, ...params: unknown[]) => void) {
return {mode: AssertionMode.LOG, logger} as const
}
}

@ -11,6 +11,7 @@
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"strict": true,
"strictNullChecks": true,
"forceConsistentCasingInFileNames": true,
"noFallthroughCasesInSwitch": true,
"module": "esnext",

Loading…
Cancel
Save