diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a6e94d8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# The `$fy` fixtures are compared byte for byte by both test suites, so they +# must check out with LF endings on every platform, Windows included. +fixtures/fy/* text eol=lf diff --git a/fixtures/fy/README.md b/fixtures/fy/README.md new file mode 100644 index 0000000..24bc97a --- /dev/null +++ b/fixtures/fy/README.md @@ -0,0 +1,21 @@ +# `$fy` translation fixtures + +`sample.sh` is a shell script exercising every construct the `$fy` translator +supports; `sample.mjs` is the module both implementations must produce from it. + +Both test suites assert against these files, which is what keeps the JavaScript +and Rust translators byte-for-byte equivalent: + +- `js/tests/fy-tool.test.mjs` +- `rust/tests/fy.rs` + +Regenerate `sample.mjs` after an intentional change to the rules — either +implementation produces the same bytes, which is the point of the fixture: + +```sh +cd rust && cargo run --quiet --example fy_translate -- ../fixtures/fy/sample.sh > ../fixtures/fy/sample.mjs +``` + +```sh +cd js && bun -e "import {translateShellToMjs} from './src/fy/index.mjs'; import {readFileSync} from 'node:fs'; process.stdout.write(translateShellToMjs(readFileSync('../fixtures/fy/sample.sh','utf8')).code)" > ../fixtures/fy/sample.mjs +``` diff --git a/fixtures/fy/sample.mjs b/fixtures/fy/sample.mjs new file mode 100644 index 0000000..fb14e2e --- /dev/null +++ b/fixtures/fy/sample.mjs @@ -0,0 +1,41 @@ +#!/usr/bin/env node +import { $, shell } from 'command-stream'; + +const args = process.argv.slice(2); +let TARGET; + +// deploy helper +shell.set("e"); + +TARGET = `${args[0] ?? `staging`}`; +process.env.REGION = `eu`; + +async function deploy(...args) { + let tag = ``; + tag = `${(await $`git rev-parse --short HEAD`).stdout.trim()}`; + await $`echo "deploying ${tag} to ${TARGET}" | tee deploy.log`; +} + +if ((await $`[ -d dist ]`).code === 0) { + await $`deploy`; +} else { + await $`echo "nothing to deploy" >&2`; +} + +for (const env of [`staging`, `prod`]) { + await $`echo "${env}"`; +} + +switch (`${TARGET}`) { + case `staging`: + await $`deploy`; + break; + default: + await $`echo unknown`; + break; +} + +if ((await $`ls && echo ok`).code !== 0) { + await $`echo failed`; +} +process.exit(0); diff --git a/fixtures/fy/sample.sh b/fixtures/fy/sample.sh new file mode 100644 index 0000000..ca53253 --- /dev/null +++ b/fixtures/fy/sample.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# deploy helper +set -e + +TARGET=${1:-staging} +export REGION=eu + +deploy() { + local tag + tag="$(git rev-parse --short HEAD)" + echo "deploying $tag to $TARGET" | tee deploy.log +} + +if [ -d dist ]; then + deploy +else + echo "nothing to deploy" >&2 +fi + +for env in staging prod; do + echo "$env" +done + +case "$TARGET" in + staging) deploy ;; + *) echo unknown ;; +esac + +ls && echo ok || echo failed +exit 0 diff --git a/js/.changeset/fy-rule-based-translation.md b/js/.changeset/fy-rule-based-translation.md new file mode 100644 index 0000000..5e569b5 --- /dev/null +++ b/js/.changeset/fy-rule-based-translation.md @@ -0,0 +1,10 @@ +--- +'command-stream': minor +--- + +Add the `$fy` virtual command, a shell-to-JavaScript translator built as a +rule-based translation on top of link-foundation/meta-language: the script is +first formalized as a links network, then rewritten into a command-stream module +by a `TranslationRuleSet`. Pipelines, `&&`/`||`, `if`/`while`/`until`/`for`/`case`, +functions, redirects, assignments and `${...}` expansions translate structurally +rather than textually, and untranslated constructs are reported as diagnostics. diff --git a/js/bun.lock b/js/bun.lock index 5d93349..37af936 100644 --- a/js/bun.lock +++ b/js/bun.lock @@ -8,6 +8,7 @@ "@resvg/resvg-js": "^2.6.2", "@xterm/headless": "^6.0.0", "gifenc": "^1.0.3", + "meta-language": "^0.46.0", "node-pty": "^1.2.0-beta.14", }, "devDependencies": { @@ -117,6 +118,8 @@ "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], + "@peggyjs/from-mem": ["@peggyjs/from-mem@3.1.3", "", { "dependencies": { "semver": "7.7.4" } }, "sha512-LLlgtfXIaeYXoOYovOI0spLM8ZXaqkAlmcRRrLzHJzLMqkU6Sw0R4KMoCoHx1PjaP815pSCBlS+BN6aD8t1Jgg=="], + "@pkgr/core": ["@pkgr/core@0.2.9", "", {}, "sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA=="], "@resvg/resvg-js": ["@resvg/resvg-js@2.6.2", "", { "optionalDependencies": { "@resvg/resvg-js-android-arm-eabi": "2.6.2", "@resvg/resvg-js-android-arm64": "2.6.2", "@resvg/resvg-js-darwin-arm64": "2.6.2", "@resvg/resvg-js-darwin-x64": "2.6.2", "@resvg/resvg-js-linux-arm-gnueabihf": "2.6.2", "@resvg/resvg-js-linux-arm64-gnu": "2.6.2", "@resvg/resvg-js-linux-arm64-musl": "2.6.2", "@resvg/resvg-js-linux-x64-gnu": "2.6.2", "@resvg/resvg-js-linux-x64-musl": "2.6.2", "@resvg/resvg-js-win32-arm64-msvc": "2.6.2", "@resvg/resvg-js-win32-ia32-msvc": "2.6.2", "@resvg/resvg-js-win32-x64-msvc": "2.6.2" } }, "sha512-xBaJish5OeGmniDj9cW5PRa/PtmuVU3ziqrbr5xJj901ZDN4TosrVaNZpEiLZAxdfnhAe7uQ7QFWfjPe9d9K2Q=="], @@ -405,6 +408,8 @@ "levn": ["levn@0.4.1", "", { "dependencies": { "prelude-ls": "^1.2.1", "type-check": "~0.4.0" } }, "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ=="], + "links-notation": ["links-notation@0.13.0", "", {}, "sha512-52FoUAVOhjfC23bdvrxv6b4Eosp02WeM3qTN9Rsb3ouPof5YVQ5fOMlRPMJmKMmdseVzEnewNLFq3lADRpWTFg=="], + "lint-staged": ["lint-staged@16.2.7", "", { "dependencies": { "commander": "^14.0.2", "listr2": "^9.0.5", "micromatch": "^4.0.8", "nano-spawn": "^2.0.0", "pidtree": "^0.6.0", "string-argv": "^0.3.2", "yaml": "^2.8.1" }, "bin": { "lint-staged": "bin/lint-staged.js" } }, "sha512-lDIj4RnYmK7/kXMya+qJsmkRFkGolciXjrsZ6PC25GdTfWOAWetR0ZbsNXRAj1EHHImRSalc+whZFg56F5DVow=="], "listr2": ["listr2@9.0.5", "", { "dependencies": { "cli-truncate": "^5.0.0", "colorette": "^2.0.20", "eventemitter3": "^5.0.1", "log-update": "^6.1.0", "rfdc": "^1.4.1", "wrap-ansi": "^9.0.0" } }, "sha512-ME4Fb83LgEgwNw96RKNvKV4VTLuXfoKudAmm2lP8Kk87KaMK0/Xrx/aAkMWmT8mDb+3MlFDspfbCs7adjRxA2g=="], @@ -427,6 +432,8 @@ "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], + "meta-language": ["meta-language@0.46.0", "", { "dependencies": { "links-notation": "0.13.0", "peggy": "5.1.0" } }, "sha512-S4HUGAtS5qEkzCh45yOd0+wTZMqsSC4pGWjE+RJtENNPi4u4ggFW5ChHtQ0x5XMHIRcu10fPl0YV2Dk66Dajgw=="], + "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], "mimic-fn": ["mimic-fn@2.1.0", "", {}, "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg=="], @@ -485,6 +492,8 @@ "path-type": ["path-type@4.0.0", "", {}, "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw=="], + "peggy": ["peggy@5.1.0", "", { "dependencies": { "@peggyjs/from-mem": "3.1.3", "commander": "^14.0.3", "source-map-generator": "2.0.6" }, "bin": { "peggy": "bin/peggy.js" } }, "sha512-IEo5aYRZ2kXH4Qby06cjtL114PZnwLoTiA41vUmg2vPZgANn+c87m5BUurhuDr5/cu758ZlpgsAfBVx+hhO5+w=="], + "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], "picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], @@ -565,6 +574,8 @@ "slice-ansi": ["slice-ansi@7.1.2", "", { "dependencies": { "ansi-styles": "^6.2.1", "is-fullwidth-code-point": "^5.0.0" } }, "sha512-iOBWFgUX7caIZiuutICxVgX1SdxwAVFFKwt1EvMYYec/NWO5meOJ6K5uQxhrYBdQJne4KxiqZc+KptFOWFSI9w=="], + "source-map-generator": ["source-map-generator@2.0.6", "", {}, "sha512-IlassDs1Ve8nV6uyQZXF9kdkJpVKnMte2JZQXu13M0A5zwc+vu6+LNHfmxsHBMDtoZE21RHiKI0/xvpecZRCNg=="], + "spark-md5": ["spark-md5@3.0.2", "", {}, "sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw=="], "spawndamnit": ["spawndamnit@3.0.1", "", { "dependencies": { "cross-spawn": "^7.0.5", "signal-exit": "^4.0.1" } }, "sha512-MmnduQUuHCoFckZoWnXsTg7JaiLBJrKFj9UI2MbRPGaJeVpsLcVBu6P/IGZovziM/YBsellCmsprgNA+w0CzVg=="], @@ -641,6 +652,8 @@ "@manypkg/get-packages/fs-extra": ["fs-extra@8.1.0", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^4.0.0", "universalify": "^0.1.0" } }, "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g=="], + "@peggyjs/from-mem/semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], + "cli-truncate/string-width": ["string-width@8.1.0", "", { "dependencies": { "get-east-asian-width": "^1.3.0", "strip-ansi": "^7.1.0" } }, "sha512-Kxl3KJGb/gxkaUMOjRsQ8IrXiGW75O4E3RPjFIINOVH8AMl2SQ/yWdTzWwF3FevIX9LcMAjJW+GRwAlAbTSXdg=="], "execa/signal-exit": ["signal-exit@3.0.7", "", {}, "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ=="], @@ -663,6 +676,8 @@ "p-locate/p-limit": ["p-limit@3.1.0", "", { "dependencies": { "yocto-queue": "^0.1.0" } }, "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ=="], + "peggy/commander": ["commander@14.0.3", "", {}, "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw=="], + "read-yaml-file/js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="], "restore-cursor/onetime": ["onetime@7.0.0", "", { "dependencies": { "mimic-function": "^5.0.0" } }, "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ=="], diff --git a/js/docs/meta-language-gaps.md b/js/docs/meta-language-gaps.md new file mode 100644 index 0000000..85d52d3 --- /dev/null +++ b/js/docs/meta-language-gaps.md @@ -0,0 +1,162 @@ +# Missing meta-language features found while building `$fy` + +`$fy` translates shell scripts into command-stream modules using the model that +[link-foundation/meta-language](https://github.com/link-foundation/meta-language) +defines: a source text is _formalized_ into a links network, and the network is +then _substituted_ into a target language through a `TranslationRuleSet`. + +The data model carried the whole design — `LinkNetwork`, `LinkType`, +`LinkQuery`, `TranslationRule`, `TranslationTemplate` are exactly the right +primitives, and `js/src/fy/shell-formalizer.mjs` and +`js/src/fy/translation-rules.mjs` use them directly. What is missing is the +_evaluation strategy_ that turns a rule set into output. That gap is filled +locally by `js/src/fy/rule-engine.mjs` (~215 lines), which is written so that it +collapses into a single upstream call once these features land. + +Version under test: **meta-language 0.46.0** (the current npm `latest`). Note +that the repository's `main` is already at 0.54.0, so npm publishing lags the +repository — that alone is worth fixing, since consumers cannot try newer work. + +Every gap below is reproduced by `js/experiments/meta-language-gaps.mjs`; run +`node js/experiments/meta-language-gaps.mjs` to see the observed output. + +## 1. `TranslationRuleSet.render` does not substitute placeholders + +The most important gap. `render()` returns the matched template's text +verbatim: + +```js +ruleSet.render(network, linkId, 'JavaScript'); +// template: 'await $`{body}`;' +// returns: 'await $`{body}`;' <- `{body}` is never replaced +``` + +Without substitution a rule set can only emit constants, so it cannot express +any translation whose output depends on the source. + +**Needed:** a placeholder syntax bound to the matched link's references, e.g. +`{name}` resolving through a rule-declared capture map. + +## 2. No recursion into nested nodes + +Related to 1: even given substitution, a template's placeholders must be filled +by _recursively rendering_ the captured child links. Translating `cd /tmp && ls` +requires rendering the `and` node's two operand nodes with their own rules. +Today only one rule fires, at one level. + +**Needed:** bottom-up rendering, where a placeholder recursively applies the +rule set to the captured link. + +## 3. Only the first matching rule is ever applied + +`render()` picks the first rule whose `LinkQuery` matches and stops. A real +rule set has one rule per construct and needs all of them applied across the +network — rule _selection per link_, not per network. + +**Needed:** per-link rule resolution (each link claimed by its best-matching +rule), which is what `RuleEngine`'s `rulesByLink` map does. + +## 4. No reference-capture API on `TranslationRule` + +A rule needs to name its children so templates can refer to them +(`{condition}`, `{body}`). `TranslationRule` exposes no such API, so `$fy` +attaches a plain `referenceCaptures` property and its own engine reads it: + +```js +const created = new TranslationRule(term, syntax(term)); +created.referenceCaptures = { condition: 0, body: 1 }; // no upstream equivalent +``` + +**Needed:** `rule.withCapture(name, index)` (or capture names on the query +itself, which would be nicer — it lets a query bind names structurally rather +than positionally). + +This one is **version-specific**: the Rust crate at 0.54.0 already has +`TranslationRule::with_reference_capture`, so it only needs to be ported to +JavaScript and published to npm. + +## 5. No variadic placeholder for a node's children + +A shell script, a `block`, a pipeline and a word list all have an unbounded +number of children joined by a separator (`"\n"`, `" | "`, `" "`). There is no +way to express "render every reference and join them". + +**Needed:** something like `{*children|separator}`. `$fy` implements +`{*name:mode|sep}` with `\n`/`\t`/`\s` escapes in the separator. + +## 6. No conditional/optional template segments + +A command may or may not have an assignment prefix; an absent capture currently +has no defined behaviour and the placeholder text is emitted literally. + +**Needed:** `{?name}…{/name}` segments, plus the rule that an unresolved +placeholder renders as empty rather than as its own source text. + +## 7. No notion of a target _sub_-language / rendering context + +The same shell node must render differently by context. `ls -la` is a statement +(``await $`ls -la`;``) at the top level but a fragment (`ls -la`) inside a +pipeline, and a variable is `NAME` as an expression but `${NAME}` inside a +template literal. `$fy` models this with four target languages — +`JavaScript`, `JavaScript:command`, `JavaScript:value`, `JavaScript:expression` +— plus a fallback chain, because meta-language has no first-class concept of a +rendering context. + +**Needed:** either sub-languages with declared fallbacks, or per-placeholder +context selection, so context-dependence stays declarative instead of leaking +into imperative code. + +## 8. No automatic indentation of multi-line substitutions + +When a rendered child spans several lines and is substituted after leading +whitespace, its continuation lines must be indented to match, or every nested +block comes out ragged. `$fy` does this in `indentContinuation`. + +**Needed:** indentation-aware substitution (the placeholder's column becomes +the continuation indent). + +## 9. No builtin `Shell` language profile, and `parse()` produces no syntax nodes + +`LanguageProfile.builtin()` resolves only `javascript`/`js`, and `parse()` on +Shell source tokenises per character without producing shell syntax nodes. This +is expected for a young project rather than a defect, but it means `$fy` has to +supply its own parser (`js/src/fy/shell-script-parser.mjs`) and formalizer. + +**Needed (lower priority):** a `Shell` profile, or documentation stating that +front-ends are expected to be supplied by the consumer. + +## 10. No public dynamic-arity link insertion (Rust only) + +Found while porting `$fy` to Rust against the crates.io crate at **0.54.0**. +`LinkNetwork::insert_link` and `insert_syntax_node` are const-generic over the +reference count: + +```rust +pub fn insert_syntax_node(&mut self, language: &str, kind: &str, refs: [LinkId; N]) -> LinkId +``` + +so the arity must be a compile-time constant, while a parser produces nodes of +arity known only at run time (a script has as many statements as it has). The +one dynamic entry point, `insert_dynamic_link`, is `pub(crate)`. + +`rust/src/fy/formalizer.rs` works around this by matching on the child count up +to a fixed maximum and grouping any wider node into nested `chunk` nodes that +`rust/src/fy/engine.rs` flattens again — output-neutral, but it means the +network shape no longer matches the source tree. + +**Needed:** a public `insert_link(&mut self, language, kind, refs: &[LinkId])` +(or simply making `insert_dynamic_link` public). The JavaScript package is +unaffected: `insertSyntaxNode` already takes an array. + +## Summary + +Gaps 1–3 block rule-based translation entirely; 4–8 are what a usable rule +language needs in practice; 9 is scope; 10 is Rust-only ergonomics that a +run-time parser cannot avoid. `js/src/fy/rule-engine.mjs` is a working +reference implementation of 1–8 over meta-language's own types and is offered +upstream. + +Gaps 1–3 were re-confirmed against the Rust crate at 0.54.0 by reading +`translation_rules.rs`: `TranslationRuleSet::render` is `pub(crate)`, applies +only the first matching rule, does not recurse, and does not substitute — so +`rust/src/fy/engine.rs` is the same reference implementation in Rust. diff --git a/js/examples/fy-translate.mjs b/js/examples/fy-translate.mjs new file mode 100644 index 0000000..622c9e9 --- /dev/null +++ b/js/examples/fy-translate.mjs @@ -0,0 +1,60 @@ +#!/usr/bin/env bun + +/** + * Demonstrates the `$fy` shell-to-JavaScript translator. + * + * The translation is rule-based: `formalizeShell` turns the script into a + * link-foundation/meta-language links network, and a `TranslationRuleSet` + * rewrites that network into a command-stream module. This example shows both + * the resulting code and the intermediate network. + */ + +import { LinkType } from 'meta-language'; + +import { $ } from '../src/$.mjs'; +import { formalizeShell, translateShellToMjs } from '../src/fy/index.mjs'; + +const script = `#!/bin/sh +set -e + +GREETING="hello" +count=$(ls | wc -l) + +for name in world there; do + echo "$GREETING $name" +done + +if [ -f package.json ]; then + echo "found $count entries" +else + echo "no package.json" && exit 1 +fi + +cat package.json | grep name +`; + +console.log('=== 1. Formalized as a meta-language links network ===\n'); +const { network, terms } = formalizeShell(script); +const syntaxLinks = network + .links() + .filter((link) => link.metadata().linkType === LinkType.Syntax); +console.log(`${syntaxLinks.length} syntax links using ${terms.size} terms:`); +console.log([...terms].sort().join(', ')); + +console.log('\n=== 2. Substituted into JavaScript ===\n'); +const { code, diagnostics } = translateShellToMjs(script); +console.log(code); + +if (diagnostics.length > 0) { + console.log('=== Diagnostics ==='); + for (const message of new Set(diagnostics)) { + console.log(` ${message}`); + } +} + +console.log('=== 3. The same thing through the `$fy` virtual command ===\n'); +const translated = await $({ + stdin: 'cd /tmp && pwd', + mirror: false, +})`$fy --no-shebang`; +console.log(translated.stdout); diff --git a/js/examples/sample-script.sh b/js/examples/sample-script.sh new file mode 100644 index 0000000..d040767 --- /dev/null +++ b/js/examples/sample-script.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Sample shell script for testing $fy tool +# This demonstrates various shell features + +set -e + +# Variables +PROJECT_DIR="/tmp/test-project" +LOG_FILE="build.log" + +# Create project directory +mkdir -p "$PROJECT_DIR" +cd "$PROJECT_DIR" + +# Initialize project +echo "Initializing project..." +touch package.json +echo '{"name": "test-project", "version": "1.0.0"}' > package.json + +# Build process +echo "Starting build process" | tee "$LOG_FILE" +ls -la >> "$LOG_FILE" + +# Conditional execution +if [ -f "package.json" ]; then + echo "Package.json found" +else + echo "Package.json missing" && exit 1 +fi + +# Pipeline operations +cat package.json | grep "name" | cut -d '"' -f 4 + +# Cleanup +cd .. +rm -rf "$PROJECT_DIR" + +echo "Build complete!" \ No newline at end of file diff --git a/js/experiments/fy-pipeline-repro.sh b/js/experiments/fy-pipeline-repro.sh new file mode 100644 index 0000000..0d8519e --- /dev/null +++ b/js/experiments/fy-pipeline-repro.sh @@ -0,0 +1,9 @@ +#!/bin/sh +GREETING=hello +for name in world there; do + echo "$GREETING $name" +done +if true; then + echo yes +fi +echo done | tr a-z A-Z diff --git a/js/experiments/meta-language-gaps.mjs b/js/experiments/meta-language-gaps.mjs new file mode 100644 index 0000000..8c8ce8d --- /dev/null +++ b/js/experiments/meta-language-gaps.mjs @@ -0,0 +1,182 @@ +// Probes meta-language for the capabilities a rule-based sh -> mjs translator +// needs. Prints PASS/GAP per capability with the observed behaviour. +// +// Usage: +// node experiments/meta-language-gaps.mjs # probe the installed npm package +// node experiments/meta-language-gaps.mjs /path/to/meta-language/js/src/index.js +// +// The second form lets us compare the published npm package against the +// meta-language repository's `main`, which is several unpublished releases +// ahead of npm. + +const entry = process.argv[2] ?? 'meta-language'; +const { + LanguageProfile, + LinkNetwork, + LinkQuery, + LinkType, + TranslationRule, + TranslationRuleSet, +} = await import(entry); + +const report = []; +const probe = (name, fn) => { + try { + report.push([name, ...fn()]); + } catch (error) { + report.push([name, 'GAP', `threw: ${error.message}`]); + } +}; + +const syntaxQuery = (term) => + new LinkQuery({ linkType: LinkType.Syntax }).withTerm(term); + +// Builds a two-level shell network: `cd /tmp && ls -la`. +function buildAndNetwork() { + const network = new LinkNetwork(); + const command = (text) => + network.insertSyntaxNode('Shell', 'command', [ + network.insertSourceToken('Shell', text), + ]); + network.insertSyntaxNode('Shell', 'and', [ + command('cd /tmp'), + command('ls -la'), + ]); + return network; +} + +function commandRule() { + const rule = new TranslationRule('command', syntaxQuery('command')); + rule.withReferenceCapture?.('body', 0); + return rule.withTemplate('JavaScript', 'await $`{body}`;'); +} + +function andRule() { + const rule = new TranslationRule('and', syntaxQuery('and')); + rule.withReferenceCapture?.('left', 0); + rule.withReferenceCapture?.('right', 1); + return rule.withTemplate('JavaScript', '{left}\n{right}'); +} + +probe('builtin shell/bash language profile', () => { + const found = ['bash', 'sh', 'shell'].filter((name) => + LanguageProfile.builtin(name) + ); + return found.length > 0 + ? ['PASS', `builtin profiles: ${found.join(', ')}`] + : ['GAP', 'LanguageProfile.builtin() resolves only javascript/js']; +}); + +probe('shell-aware parsing (syntax nodes from source text)', () => { + const network = LinkNetwork.parse('cd /tmp && ls -la\n', 'Shell'); + const syntax = network + .links() + .filter((link) => link.metadata().linkType === LinkType.Syntax); + return syntax.length > 0 + ? ['PASS', `${syntax.length} syntax nodes`] + : [ + 'GAP', + 'parse() tokenises Shell per character only; no shell syntax nodes are produced', + ]; +}); + +probe('template placeholder substitution', () => { + const network = buildAndNetwork(); + const output = new TranslationRuleSet('shell-to-js', [commandRule()]).render( + 'JavaScript', + network + ); + return output.includes('cd /tmp') + ? ['PASS', output] + : ['GAP', `template text is emitted verbatim: ${JSON.stringify(output)}`]; +}); + +probe('recursive rendering of nested nodes', () => { + const network = buildAndNetwork(); + const output = new TranslationRuleSet('shell-to-js', [ + andRule(), + commandRule(), + ]).render('JavaScript', network); + const expected = 'await $`cd /tmp`;\nawait $`ls -la`;'; + return output === expected + ? ['PASS', output] + : [ + 'GAP', + `rendered ${JSON.stringify(output)}, expected ${JSON.stringify(expected)}`, + ]; +}); + +probe('composing several rules over one network', () => { + // Two sibling top-level nodes that need two different rules. + const network = new LinkNetwork(); + network.insertSyntaxNode('Shell', 'comment', [ + network.insertSourceToken('Shell', 'build step'), + ]); + network.insertSyntaxNode('Shell', 'command', [ + network.insertSourceToken('Shell', 'make all'), + ]); + + const comment = new TranslationRule('comment', syntaxQuery('comment')); + comment.withReferenceCapture?.('body', 0); + comment.withTemplate('JavaScript', '// {body}'); + + const output = new TranslationRuleSet('shell-to-js', [ + comment, + commandRule(), + ]).render('JavaScript', network); + const expected = '// build step\nawait $`make all`;'; + return output === expected + ? ['PASS', output] + : [ + 'GAP', + `only the first matching rule is applied; got ${JSON.stringify(output)}`, + ]; +}); + +probe('variadic / repeated reference capture', () => { + const network = new LinkNetwork(); + const stages = ['ls', 'grep test', 'wc -l'].map((text) => + network.insertSyntaxNode('Shell', 'command', [ + network.insertSourceToken('Shell', text), + ]) + ); + network.insertSyntaxNode('Shell', 'pipeline', stages); + const rule = new TranslationRule( + 'pipeline', + syntaxQuery('pipeline') + ).withTemplate('JavaScript', 'await $`{stages:join= | }`;'); + const output = new TranslationRuleSet('p', [rule]).render( + 'JavaScript', + network + ); + return output.includes('ls | grep test | wc -l') + ? ['PASS', output] + : ['GAP', `no repeated-capture placeholder; got ${JSON.stringify(output)}`]; +}); + +probe('optional / conditional template segment', () => { + const network = new LinkNetwork(); + network.insertSyntaxNode('Shell', 'command', [ + network.insertSourceToken('Shell', 'ls'), + ]); + const rule = new TranslationRule('command', syntaxQuery('command')); + rule.withReferenceCapture?.('body', 0); + rule.withTemplate('JavaScript', 'await $`{body}`;{?args: {args}}'); + const output = new TranslationRuleSet('c', [rule]).render( + 'JavaScript', + network + ); + return output === 'await $`ls`;' + ? ['PASS', output] + : [ + 'GAP', + `an absent capture is emitted literally: ${JSON.stringify(output)}`, + ]; +}); + +console.log(`meta-language entry: ${entry}\n`); +for (const [name, status, detail] of report) { + console.log( + `${status.padEnd(4)} | ${name}\n ${String(detail).replace(/\n/g, '\n ')}` + ); +} diff --git a/js/package-lock.json b/js/package-lock.json index 0a9697f..faf36e8 100644 --- a/js/package-lock.json +++ b/js/package-lock.json @@ -1,12 +1,12 @@ { "name": "command-stream", - "version": "0.17.0", + "version": "0.18.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "command-stream", - "version": "0.17.0", + "version": "0.18.0", "license": "Unlicense", "dependencies": { "@resvg/resvg-js": "^2.6.2", diff --git a/js/package.json b/js/package.json index eda3fe6..9f23bfd 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "command-stream", - "version": "0.17.0", + "version": "0.18.0", "description": "Modern $ shell utility library with streaming, async iteration, and EventEmitter support, optimized for Bun runtime", "type": "module", "main": "src/$.mjs", @@ -89,6 +89,7 @@ "@resvg/resvg-js": "^2.6.2", "@xterm/headless": "^6.0.0", "gifenc": "^1.0.3", + "meta-language": "^0.46.0", "node-pty": "^1.2.0-beta.14" } } diff --git a/js/src/$.mjs b/js/src/$.mjs index c028519..467a1d5 100755 --- a/js/src/$.mjs +++ b/js/src/$.mjs @@ -399,6 +399,7 @@ import dirnameCommand from './commands/$.dirname.mjs'; import yesCommand from './commands/$.yes.mjs'; import seqCommand from './commands/$.seq.mjs'; import testCommand from './commands/$.test.mjs'; +import fyCommand from './commands/$.$fy.mjs'; // Built-in commands that match Bun.$ functionality function registerBuiltins() { @@ -428,6 +429,7 @@ function registerBuiltins() { register('yes', yesCommand); register('seq', seqCommand); register('test', testCommand); + register('$fy', fyCommand); } // Initialize built-in commands diff --git a/js/src/$.process-runner-virtual.mjs b/js/src/$.process-runner-virtual.mjs index efe07db..4143998 100644 --- a/js/src/$.process-runner-virtual.mjs +++ b/js/src/$.process-runner-virtual.mjs @@ -9,7 +9,13 @@ import { safeWrite } from './$.stream-utils.mjs'; * @param {object} options - Runner options * @returns {string} Stdin data */ +/** Stdio *modes*, which are configuration rather than data to feed a command. */ +const STDIN_MODES = new Set(['inherit', 'pipe', 'ignore']); + function getStdinData(options) { + if (typeof options.stdin === 'string' && STDIN_MODES.has(options.stdin)) { + return ''; + } if (options.stdin && typeof options.stdin === 'string') { return options.stdin; } diff --git a/js/src/$.virtual-commands.mjs b/js/src/$.virtual-commands.mjs index f3c5b0d..eca1358 100644 --- a/js/src/$.virtual-commands.mjs +++ b/js/src/$.virtual-commands.mjs @@ -26,6 +26,7 @@ import dirnameCommand from './commands/$.dirname.mjs'; import yesCommand from './commands/$.yes.mjs'; import seqCommand from './commands/$.seq.mjs'; import testCommand from './commands/$.test.mjs'; +import fyCommand from './commands/$.$fy.mjs'; /** * Register a virtual command @@ -110,4 +111,5 @@ export function registerBuiltins() { register('yes', yesCommand); register('seq', seqCommand); register('test', testCommand); + register('$fy', fyCommand); } diff --git a/js/src/commands/$.$fy.mjs b/js/src/commands/$.$fy.mjs new file mode 100644 index 0000000..6d091d2 --- /dev/null +++ b/js/src/commands/$.$fy.mjs @@ -0,0 +1,108 @@ +/** + * `$fy` — shell to mjs translator. + * + * This module is only the command-line surface. The translation itself is + * rule-based and lives in `../fy/`: the script is first formalized as a + * link-foundation/meta-language links network, then rewritten into JavaScript + * by a `TranslationRuleSet`. + */ + +import { readFileSync, writeFileSync } from 'node:fs'; + +const USAGE = `$fy - Convert shell scripts to command-stream JavaScript modules + +Usage: + $fy # Translate and print to stdout + $fy # Translate and save to a file + echo "ls -la" | $fy # Translate from stdin + +Options: + --no-shebang Omit the leading #!/usr/bin/env node line + -h, --help Show this help + +Examples: + $fy deploy.sh # Print the translated deploy script + $fy build.sh build.mjs # Translate build.sh into build.mjs + echo "cd /tmp && ls" | $fy # Translate a one-liner from stdin + +The translation is driven by substitution rules over a meta-language links +network, so pipelines, &&/||, if/while/until/for/case, functions, redirects, +variable assignments and \${...} expansions all translate structurally rather +than textually. +`; + +/** + * Renders translation diagnostics as shell-style warnings. + * + * @param {string[]} diagnostics Messages collected by the rule engine. + * @returns {string} Text for stderr (empty when there is nothing to report). + */ +function formatDiagnostics(diagnostics) { + const unique = [...new Set(diagnostics)]; + return unique.map((message) => `$fy: warning: ${message}\n`).join(''); +} + +/** + * The `$fy` virtual command. + * + * @param {{args?: string[], stdin?: string|Buffer}} invocation Command input. + * @returns {Promise<{stdout?: string, stderr?: string, code: number}>} Command + * result. + */ +export async function $fy({ args = [], stdin } = {}) { + const options = { shebang: !args.includes('--no-shebang') }; + const positional = args.filter((argument) => !argument.startsWith('-')); + // A file argument always wins: `$fy in.sh` must read `in.sh` even when the + // command happens to inherit a non-empty stdin from its caller. + const hasStdin = + positional.length === 0 && stdin !== undefined && String(stdin).length > 0; + + if (args.includes('-h') || args.includes('--help')) { + return { stdout: USAGE, code: 0 }; + } + if (!hasStdin && positional.length === 0) { + return { stderr: USAGE, code: 1 }; + } + + let source; + if (hasStdin) { + source = String(stdin); + } else { + try { + source = readFileSync(positional[0], 'utf8'); + } catch (error) { + return { + stderr: `$fy: cannot read '${positional[0]}': ${error.message}\n`, + code: 1, + }; + } + } + + let translated; + try { + // Loaded on demand: the translator pulls in `meta-language`, which must not + // become an import-time dependency of the library itself. + const { translateShellToMjs } = await import('../fy/index.mjs'); + translated = translateShellToMjs(source, options); + } catch (error) { + return { stderr: `$fy: translation failed: ${error.message}\n`, code: 1 }; + } + + const stderr = formatDiagnostics(translated.diagnostics); + const outputFile = hasStdin ? undefined : positional[1]; + if (!outputFile) { + return { stdout: translated.code, stderr, code: 0 }; + } + + try { + writeFileSync(outputFile, translated.code); + } catch (error) { + return { + stderr: `${stderr}$fy: cannot write '${outputFile}': ${error.message}\n`, + code: 1, + }; + } + return { stdout: `Translated to '${outputFile}'\n`, stderr, code: 0 }; +} + +export default $fy; diff --git a/js/src/commands/index.mjs b/js/src/commands/index.mjs index 42d62f0..90c64b7 100644 --- a/js/src/commands/index.mjs +++ b/js/src/commands/index.mjs @@ -22,3 +22,4 @@ export { default as dirname } from './$.dirname.mjs'; export { default as yes } from './$.yes.mjs'; export { default as seq } from './$.seq.mjs'; export { default as test } from './$.test.mjs'; +export { default as fy } from './$.$fy.mjs'; diff --git a/js/src/fy/index.mjs b/js/src/fy/index.mjs new file mode 100644 index 0000000..5725f77 --- /dev/null +++ b/js/src/fy/index.mjs @@ -0,0 +1,105 @@ +// `$fy`: rule-based translation of shell scripts into command-stream modules. +// +// The pipeline has two halves, exactly as link-foundation/meta-language models +// translation: +// +// 1. Formalize shell source -> links network (`shell-formalizer.mjs`) +// 2. Substitute links network -> JavaScript (`translation-rules.mjs`) +// +// Step 2 contains no shell knowledge beyond the rule table, and step 1 contains +// no JavaScript knowledge at all. + +import { formalizeShell } from './shell-formalizer.mjs'; +import { RuleEngine } from './rule-engine.mjs'; +import { + LANGUAGE_FALLBACKS, + MODES, + TARGET, + buildRuleSet, +} from './translation-rules.mjs'; + +/** Terms whose presence requires `const args = process.argv.slice(2)`. */ +const ARGUMENT_TERMS = ['positional', 'all-arguments', 'argument-count']; + +/** Builds the module header the translated statements need. */ +function buildPreamble(terms, names, options) { + const imports = ['$']; + if (terms.has('set-option')) { + imports.push('shell'); + } + + const lines = []; + if (options.shebang !== false) { + lines.push('#!/usr/bin/env node'); + } + lines.push( + `import { ${imports.join(', ')} } from '${options.moduleName ?? 'command-stream'}';` + ); + lines.push(''); + + if (ARGUMENT_TERMS.some((term) => terms.has(term))) { + lines.push('const args = process.argv.slice(2);'); + } + if (terms.has('exit-status')) { + lines.push('let exitCode = 0;'); + } + // Shell variables have no block scope, so every non-`local` binding is + // hoisted to the top of the module rather than declared at its first + // assignment (which would break on reassignment inside a loop or branch). + const hoisted = [...names.bound].filter((name) => !names.locals.has(name)); + if (hoisted.length > 0) { + lines.push(`let ${hoisted.join(', ')};`); + } + if (lines[lines.length - 1] !== '') { + lines.push(''); + } + // The trailing empty entry plus this newline separate the preamble from the + // first translated statement by exactly one blank line. + return `${lines.join('\n')}\n`; +} + +/** + * Translates a shell script into a command-stream ES module. + * + * @param {string} source Shell source text. + * @param {object} [options] + * @param {boolean} [options.shebang] Emit a `#!/usr/bin/env node` line (default true). + * @param {string} [options.moduleName] Module to import `$` from. + * @returns {{code: string, preamble: string, body: string, diagnostics: string[], network: object}} + * `code` is `preamble + body`; the halves are returned separately so callers + * (and tests) can inspect the translated statements on their own. + */ +export function translateShellToMjs(source, options = {}) { + // A shell script checked out with CRLF endings is still a shell script: `\r` + // is not a token, so it is dropped before anything else looks at the text. + const normalized = source.replace(/\r\n?/g, '\n'); + // A `#!/bin/sh` line selects the interpreter for the *shell* script; the + // translated module gets its own shebang from `buildPreamble`. + const script = normalized.replace(/^#![^\n]*\n?/, ''); + const { network, root, terms, names } = formalizeShell(script); + const ruleSet = buildRuleSet({ trackExitCode: terms.has('exit-status') }); + const engine = new RuleEngine(network, ruleSet, { + language: TARGET, + fallbacks: LANGUAGE_FALLBACKS, + modes: MODES, + }); + + const body = `${engine.render(root)}` + .replace(/\n{3,}/g, '\n\n') + .replace(/\s*$/, '\n'); + const preamble = buildPreamble(terms, names, options); + + return { + code: `${preamble}${body}`, + preamble, + body, + diagnostics: engine.report(), + network, + ruleSet, + }; +} + +export { formalizeShell } from './shell-formalizer.mjs'; +export { parseShellScript } from './shell-script-parser.mjs'; +export { buildRuleSet } from './translation-rules.mjs'; +export { RuleEngine } from './rule-engine.mjs'; diff --git a/js/src/fy/rule-engine.mjs b/js/src/fy/rule-engine.mjs new file mode 100644 index 0000000..141c2e0 --- /dev/null +++ b/js/src/fy/rule-engine.mjs @@ -0,0 +1,239 @@ +// Recursive rule engine for translating a meta-language links network. +// +// meta-language models translation as a `TranslationRuleSet`: a query per +// source construct plus a template per target language. Its own +// `TranslationRuleSet.render` applies the *first* rule that matches and does +// not recurse into a template's placeholders, so it cannot translate a nested +// tree (see `experiments/meta-language-gaps.mjs` for reproductions, and +// `docs/meta-language-gaps.md` for the report filed upstream). +// +// This engine keeps meta-language's rule model — the rules really are +// `TranslationRule`s carrying `LinkQuery`s and per-language templates — and +// supplies only the missing evaluation strategy: +// +// * bottom-up recursion through placeholders, +// * variadic placeholders over a node's children, +// * conditional segments for optional captures, +// * automatic indentation of multi-line substitutions. +// +// Once meta-language grows these, this file collapses into a call to +// `network.reconstructTextAsWithRules(...)`. + +import { LinkType } from 'meta-language'; + +/** Placeholder: `{name}`, `{name:mode}`, `{*name:mode|separator}`. */ +const PLACEHOLDER = + /^\{(\*?)([A-Za-z_][A-Za-z0-9_-]*|\.)(?::([a-z]+))?(?:\|([^}]*))?\}/; +/** Conditional segment: `{?name}` ... `{/name}`. */ +const CONDITIONAL_OPEN = /^\{\?([A-Za-z_][A-Za-z0-9_-]*)\}/; + +const SEPARATOR_ESCAPES = { n: '\n', t: '\t', s: ' ' }; + +function decodeSeparator(separator) { + if (separator === undefined) { + return ''; + } + return separator.replace( + /\\(.)/g, + (match, character) => SEPARATOR_ESCAPES[character] ?? character + ); +} + +/** Indents every line after the first by `indent`. */ +function indentContinuation(value, indent) { + if (indent === '' || !value.includes('\n')) { + return value; + } + return value + .split('\n') + .map((line, position) => + position === 0 || line === '' ? line : indent + line + ) + .join('\n'); +} + +/** The whitespace run at the end of `output`, if `output` ends on a blank line prefix. */ +function currentIndent(output) { + const lastNewline = output.lastIndexOf('\n'); + const line = output.slice(lastNewline + 1); + return /^[ \t]*$/.test(line) ? line : ''; +} + +export class RuleEngine { + /** + * @param {object} network A meta-language `LinkNetwork`. + * @param {object} ruleSet A meta-language `TranslationRuleSet`. + * @param {object} [options] + * @param {string} [options.language] Default target language. + * @param {Record} [options.fallbacks] Per-language template + * fallback chains, e.g. `{'JavaScript:value': ['JavaScript:command']}`. + * @param {Record string}>} [options.modes] + * Placeholder modes. A mode either renders the captured link in another + * target language, post-processes the captured source text, or both. + */ + constructor(network, ruleSet, options = {}) { + this.network = network; + this.ruleSet = ruleSet; + this.language = options.language ?? 'JavaScript'; + this.fallbacks = options.fallbacks ?? {}; + this.modes = options.modes ?? {}; + this.rulesByLink = new Map(); + this.diagnostics = []; + + // Rule selection is query-driven: each rule's `LinkQuery` is evaluated + // against the whole network once, and the first rule to claim a link owns it. + for (const rule of ruleSet.rules) { + for (const match of network.find(rule.query)) { + const key = String(match.linkId?.asU64?.() ?? match.linkId); + if (!this.rulesByLink.has(key)) { + this.rulesByLink.set(key, rule); + } + } + } + } + + /** Diagnostics collected while rendering (constructs with no rule). */ + report() { + return [...this.diagnostics]; + } + + /** + * Renders one link. + * + * @param {object} linkId Link to render. + * @param {string} [language] Target language, defaulting to the engine's. + * @returns {string} Translated text. + */ + render(linkId, language = this.language) { + const key = String(linkId?.asU64?.() ?? linkId); + const rule = this.rulesByLink.get(key); + const link = this.network.link(linkId); + if (!rule) { + // Source tokens are leaves and are expected to render as their own text; + // an unclaimed *syntax* node means a construct nobody translates. + if (link?.metadata().linkType === LinkType.Syntax) { + this.diagnostics.push( + `no translation rule for \`${link.metadata().term}\`` + ); + } + return this.network.capturedText(linkId); + } + + const template = this.templateFor(rule, language); + if (!template) { + this.diagnostics.push( + `rule \`${rule.name}\` has no ${language} template` + ); + return this.network.capturedText(linkId); + } + return this.expand(template.text, rule, link, language); + } + + /** Looks a template up through the configured fallback chain. */ + templateFor(rule, language) { + for (const candidate of [language, ...(this.fallbacks[language] ?? [])]) { + const template = rule.templateFor(candidate); + if (template) { + return template; + } + } + return undefined; + } + + /** Resolves a capture name to a child link id. */ + resolve(rule, link, name) { + if (name === '.') { + return link.id(); + } + const index = rule.referenceCaptures?.[name]; + if (index === undefined) { + return undefined; + } + return link.references()[index]; + } + + /** Renders `linkId` in the mode a placeholder asked for. */ + renderMode(linkId, mode, language) { + if (mode === undefined) { + return this.render(linkId, language); + } + const definition = this.modes[mode]; + if (!definition) { + this.diagnostics.push(`unknown placeholder mode \`${mode}\``); + return this.render(linkId, language); + } + const rendered = definition.language + ? this.render(linkId, definition.language) + : this.network.capturedText(linkId); + return definition.transform ? definition.transform(rendered) : rendered; + } + + expand(source, rule, link, language) { + let output = ''; + let index = 0; + + while (index < source.length) { + const rest = source.slice(index); + + if (rest.startsWith('{{')) { + output += '{'; + index += 2; + continue; + } + if (rest.startsWith('}}')) { + output += '}'; + index += 2; + continue; + } + + const conditional = CONDITIONAL_OPEN.exec(rest); + if (conditional) { + const name = conditional[1]; + const close = source.indexOf(`{/${name}}`, index); + const end = close === -1 ? source.length : close; + const body = source.slice(index + conditional[0].length, end); + if (this.resolve(rule, link, name) !== undefined) { + output += this.expand(body, rule, link, language); + } + index = close === -1 ? source.length : close + name.length + 3; + continue; + } + + const placeholder = PLACEHOLDER.exec(rest); + if (placeholder) { + const [matched, variadic, name, mode, separator] = placeholder; + const target = this.resolve(rule, link, name); + if (target === undefined) { + // An unresolved capture renders as nothing rather than as its own + // source text, so optional children simply disappear. + index += matched.length; + continue; + } + + const indent = currentIndent(output); + let value; + if (variadic) { + const children = this.network.link(target)?.references() ?? []; + value = children + .map((child) => + indentContinuation(this.renderMode(child, mode, language), indent) + ) + .join(decodeSeparator(separator)); + } else { + value = indentContinuation( + this.renderMode(target, mode, language), + indent + ); + } + output += value; + index += matched.length; + continue; + } + + output += source[index]; + index += 1; + } + + return output; + } +} diff --git a/js/src/fy/shell-formalizer.mjs b/js/src/fy/shell-formalizer.mjs new file mode 100644 index 0000000..93ddcf1 --- /dev/null +++ b/js/src/fy/shell-formalizer.mjs @@ -0,0 +1,137 @@ +// Formalizes a shell script as a meta-language links network. +// +// This is the first half of the translation pipeline the way +// link-foundation/meta-language models it: source text becomes a network of +// typed links (`LinkType.Syntax` nodes over `LinkType.SourceToken` leaves). +// The second half — `translation-rules.mjs` plus `rule-engine.mjs` — rewrites +// that network into JavaScript purely by substitution rules. + +import { LanguageProfile, LinkNetwork, LinkType } from 'meta-language'; + +import { parseShellScript } from './shell-script-parser.mjs'; +import { expandWord } from './word-expander.mjs'; + +/** The language name used for every shell link in the network. */ +export const SHELL_LANGUAGE = 'Shell'; + +/** Node terms that carry their own text (materialised as a leading token child). */ +const TEXT_BEARING = new Set([ + 'assignment', + 'export', + 'local', + 'comment', + 'name', + 'literal', + 'variable', + 'env-variable', + 'positional', + 'script-name', + 'all-arguments', + 'argument-count', + 'exit-status', + 'process-id', + 'unsupported-expansion', + 'default-expansion', + 'pattern', + 'redirect', +]); + +/** + * Collects the variable names the script binds, split by how they must be + * declared in JavaScript. + * + * `export X=v` deliberately does not create a binding: it writes to + * `process.env`, so `$X` must translate to `process.env.X`. + * + * @returns {{bound: Set, locals: Set}} `bound` is every name + * that becomes a JavaScript variable; `locals` is the subset already + * declared at its use site (`local`, and the `for` loop variable). + */ +function collectDeclaredNames( + tree, + names = { bound: new Set(), locals: new Set() } +) { + if ((tree.term === 'assignment' || tree.term === 'local') && tree.text) { + names.bound.add(tree.text); + if (tree.term === 'local') { + names.locals.add(tree.text); + } + } + if (tree.term === 'for' && tree.children[0]?.text) { + names.bound.add(tree.children[0].text); + names.locals.add(tree.children[0].text); + } + for (const child of tree.children ?? []) { + collectDeclaredNames(child, names); + } + return names; +} + +/** Rewrites every `word` leaf into a `word` node of typed parts. */ +function expandWords(tree, declared) { + if (tree.term === 'word') { + const parseSubstitution = (source) => + expandWords(parseShellScript(source), declared); + return { + term: 'word', + text: undefined, + children: expandWord(tree.text ?? '', declared, parseSubstitution), + }; + } + return { + ...tree, + children: (tree.children ?? []).map((child) => + expandWords(child, declared) + ), + }; +} + +/** + * The capability profile of the shell dialect this translator formalizes. + * Declaring it in the network makes the supported surface queryable rather + * than implicit, and `validateNetwork` then rejects anything outside it. + */ +export function shellProfile(concepts) { + let profile = LanguageProfile.new('command-stream shell', SHELL_LANGUAGE) + .withLinkType(LinkType.Syntax) + .withLinkType(LinkType.SourceToken) + .withLinkType(LinkType.Semantic); + for (const concept of concepts) { + profile = profile.withConcept(concept); + } + return profile; +} + +/** + * Formalizes shell source as a links network. + * + * @param {string} source Shell source text. + * @returns {{network: LinkNetwork, root: object, terms: Set, names: object}} + * The network, the id of its `script` root link, every node term used, and + * the variable names the script binds. + */ +export function formalizeShell(source) { + const parsed = parseShellScript(source); + const names = collectDeclaredNames(parsed); + const tree = expandWords(parsed, names.bound); + + const network = new LinkNetwork(); + const terms = new Set(); + + const insert = (current) => { + terms.add(current.term); + const children = []; + if (TEXT_BEARING.has(current.term) || current.text !== undefined) { + children.push( + network.insertSourceToken(SHELL_LANGUAGE, current.text ?? '') + ); + } + for (const child of current.children ?? []) { + children.push(insert(child)); + } + return network.insertSyntaxNode(SHELL_LANGUAGE, current.term, children); + }; + + const root = insert(tree); + return { network, root, terms, names }; +} diff --git a/js/src/fy/shell-lexer.mjs b/js/src/fy/shell-lexer.mjs new file mode 100644 index 0000000..49b9090 --- /dev/null +++ b/js/src/fy/shell-lexer.mjs @@ -0,0 +1,207 @@ +// Shell lexer for the $fy translator. +// +// Produces a flat, lossless token stream: every character of the input ends up +// in exactly one token's `raw`, so the formalizer can rebuild the original +// source from the links network. + +const REDIRECT = /^(\d*)(>>|>&|>|<<<|<<|<)/; +const OPERATORS = ['&&', '||', ';;', ';', '|', '(', ')']; +const WORD_TERMINATORS = new Set([ + ' ', + '\t', + '\n', + ';', + '|', + '&', + '(', + ')', + '<', + '>', +]); + +/** Token types emitted by {@link tokenize}. */ +export const TokenType = Object.freeze({ + WORD: 'word', + OPERATOR: 'operator', + REDIRECT: 'redirect', + NEWLINE: 'newline', + COMMENT: 'comment', + EOF: 'eof', +}); + +/** + * Consumes a balanced span that starts at `start` (e.g. `$(`, `${`). + * Returns the index just past the closing delimiter. + */ +function readBalanced(source, start, open, close) { + let depth = 0; + let index = start; + while (index < source.length) { + const character = source[index]; + if (character === '\\') { + index += 2; + continue; + } + if (character === open) { + depth += 1; + } else if (character === close) { + depth -= 1; + if (depth === 0) { + return index + 1; + } + } + index += 1; + } + return source.length; +} + +/** Consumes a quoted span including both quote characters. */ +function readQuoted(source, start, quote) { + let index = start + 1; + while (index < source.length) { + const character = source[index]; + if (quote === '"' && character === '\\') { + index += 2; + continue; + } + if (character === quote) { + return index + 1; + } + index += 1; + } + return source.length; +} + +/** + * Reads one shell word starting at `start`, treating quotes, `$(...)`, + * `${...}` and backticks as atomic spans. + */ +function readWord(source, start) { + let index = start; + while (index < source.length) { + const character = source[index]; + if (character === '\\') { + index += 2; + continue; + } + if (character === "'" || character === '"') { + index = readQuoted(source, index, character); + continue; + } + if (character === '`') { + index = readQuoted(source, index, '`'); + continue; + } + if (character === '$' && source[index + 1] === '(') { + index = readBalanced(source, index + 1, '(', ')'); + continue; + } + if (character === '$' && source[index + 1] === '{') { + index = readBalanced(source, index + 1, '{', '}'); + continue; + } + if (WORD_TERMINATORS.has(character)) { + break; + } + index += 1; + } + return { text: source.slice(start, index), next: index }; +} + +/** + * A redirection operator may be prefixed by a file descriptor (`2>`). Only + * treat a leading run of digits as part of the operator, never as a word. + */ +function matchRedirect(source, index) { + const match = REDIRECT.exec(source.slice(index)); + return match ? match[0] : undefined; +} + +function matchOperator(source, index) { + return OPERATORS.find((operator) => source.startsWith(operator, index)); +} + +/** + * Matches whitespace, line splices, newlines and comments. + * + * @returns {{token: object | null, next: number} | null} The token to emit (or + * `null` to emit nothing) and the index to continue from, or `null` when the + * character starts a word or operator instead. + */ +function matchTrivia(source, index) { + const character = source[index]; + + if (character === '\n') { + return { token: { type: TokenType.NEWLINE, text: '\n' }, next: index + 1 }; + } + if (character === ' ' || character === '\t' || character === '\r') { + return { token: null, next: index + 1 }; + } + // A backslash-newline pair splices two physical lines into one. + if (character === '\\' && source[index + 1] === '\n') { + return { token: null, next: index + 2 }; + } + // `#` only opens a comment at the start of a word. + if ( + character === '#' && + (index === 0 || /[\s;|&()]/.test(source[index - 1])) + ) { + let end = source.indexOf('\n', index); + if (end === -1) { + end = source.length; + } + const text = source.slice(index, end); + return { token: { type: TokenType.COMMENT, text }, next: end }; + } + return null; +} + +/** + * Tokenizes a complete shell script. + * + * @param {string} source Shell source text. + * @returns {Array<{type: string, text: string}>} Tokens, ending with an EOF token. + */ +export function tokenize(source) { + const tokens = []; + let index = 0; + + while (index < source.length) { + const character = source[index]; + + const trivia = matchTrivia(source, index); + if (trivia) { + if (trivia.token) { + tokens.push(trivia.token); + } + index = trivia.next; + continue; + } + + const redirect = matchRedirect(source, index); + if (redirect) { + tokens.push({ type: TokenType.REDIRECT, text: redirect }); + index += redirect.length; + continue; + } + // `&` on its own backgrounds a command; `&&` was matched by the operator list. + const operator = + matchOperator(source, index) ?? (character === '&' ? '&' : undefined); + if (operator) { + tokens.push({ type: TokenType.OPERATOR, text: operator }); + index += operator.length; + continue; + } + + const { text, next } = readWord(source, index); + if (text === '') { + index += 1; + continue; + } + tokens.push({ type: TokenType.WORD, text }); + index = next; + } + + tokens.push({ type: TokenType.EOF, text: '' }); + return tokens; +} diff --git a/js/src/fy/shell-script-parser.mjs b/js/src/fy/shell-script-parser.mjs new file mode 100644 index 0000000..7944673 --- /dev/null +++ b/js/src/fy/shell-script-parser.mjs @@ -0,0 +1,509 @@ +// Recursive-descent shell parser for the $fy translator. +// +// Produces a plain node tree (`{ term, text?, children }`) which +// `shell-formalizer.mjs` then materialises as a meta-language links network. +// Keeping the two steps apart means the grammar stays readable and the +// network construction stays mechanical. +// +// This parser is deliberately separate from `src/shell-parser.mjs`, which +// parses a *single* command line for the runtime. `$fy` needs whole-script +// structure: comments, blank lines, control flow and function definitions. + +import { TokenType, tokenize } from './shell-lexer.mjs'; + +/** Reserved words that terminate a command list. */ +const BLOCK_TERMINATORS = new Set([ + 'then', + 'elif', + 'else', + 'fi', + 'do', + 'done', + 'esac', + '}', +]); + +const ASSIGNMENT = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/s; + +const node = (term, children = [], text = undefined) => ({ + term, + children, + text, +}); +const leaf = (term, text) => node(term, [], text); + +class ShellParser { + constructor(source) { + this.tokens = tokenize(source); + this.position = 0; + } + + current() { + return this.tokens[this.position]; + } + + at(type, text = undefined) { + const token = this.current(); + return token.type === type && (text === undefined || token.text === text); + } + + atWord(text) { + return this.at(TokenType.WORD, text); + } + + consume() { + return this.tokens[this.position++]; + } + + expectWord(text) { + if (this.atWord(text)) { + this.consume(); + return true; + } + return false; + } + + /** Skips newlines and `;` separators, emitting `blank` nodes for blank lines. */ + skipSeparators(collector) { + let newlines = 0; + for (;;) { + if (this.at(TokenType.NEWLINE)) { + this.consume(); + newlines += 1; + // The first newline ends the previous statement; each further one is a + // blank line worth preserving in the output. + if (newlines > 1 && collector) { + collector.push(node('blank')); + } + continue; + } + if (this.at(TokenType.OPERATOR, ';')) { + this.consume(); + continue; + } + return; + } + } + + /** Whether the current token closes the enclosing block. */ + atBlockEnd(extraTerminators) { + const token = this.current(); + if (token.type === TokenType.EOF) { + return true; + } + if ( + token.type === TokenType.OPERATOR && + (token.text === ')' || token.text === ';;') + ) { + return true; + } + if (token.type !== TokenType.WORD) { + return false; + } + return ( + BLOCK_TERMINATORS.has(token.text) || extraTerminators.has(token.text) + ); + } + + parseScript() { + return node('script', this.parseStatements(new Set())); + } + + parseStatements(extraTerminators) { + const statements = []; + // Leading blank lines are not "blank lines between statements". + this.skipSeparators(undefined); + + while (!this.atBlockEnd(extraTerminators)) { + if (this.at(TokenType.COMMENT)) { + statements.push( + leaf('comment', this.consume().text.replace(/^#\s?/, '')) + ); + } else { + const statement = this.parseList(); + if (!statement) { + break; + } + statements.push(statement); + } + this.skipSeparators(statements); + } + + // Trailing blank lines add nothing to the translation. + while ( + statements.length > 0 && + statements[statements.length - 1].term === 'blank' + ) { + statements.pop(); + } + return statements; + } + + /** `pipeline (&& | ||) pipeline ...`, left associative. */ + parseList() { + let left = this.parsePipeline(); + if (!left) { + return undefined; + } + while ( + this.at(TokenType.OPERATOR, '&&') || + this.at(TokenType.OPERATOR, '||') + ) { + const operator = this.consume().text === '&&' ? 'and' : 'or'; + // An operator may be followed by a newline before its right operand. + while (this.at(TokenType.NEWLINE)) { + this.consume(); + } + const right = this.parsePipeline(); + if (!right) { + break; + } + left = node(operator, [left, right]); + } + return left; + } + + parsePipeline() { + const stages = []; + let stage = this.parseCommand(); + if (!stage) { + return undefined; + } + stages.push(stage); + while (this.at(TokenType.OPERATOR, '|')) { + this.consume(); + while (this.at(TokenType.NEWLINE)) { + this.consume(); + } + stage = this.parseCommand(); + if (!stage) { + break; + } + stages.push(stage); + } + return stages.length === 1 ? stages[0] : node('pipeline', stages); + } + + parseCommand() { + if (this.at(TokenType.OPERATOR, '(')) { + this.consume(); + const body = this.parseStatements(new Set()); + if (this.at(TokenType.OPERATOR, ')')) { + this.consume(); + } + return node('subshell', [node('block', body)]); + } + if (this.atWord('{')) { + this.consume(); + const body = this.parseStatements(new Set()); + this.expectWord('}'); + return node('block', body); + } + if (this.atWord('if')) { + return this.parseIf(); + } + if (this.atWord('while') || this.atWord('until')) { + return this.parseLoop(); + } + if (this.atWord('for')) { + return this.parseFor(); + } + if (this.atWord('case')) { + return this.parseCase(); + } + const functionNode = this.tryParseFunction(); + if (functionNode) { + return functionNode; + } + return this.parseSimpleCommand(); + } + + /** `if list; then block [elif ...] [else block] fi` */ + parseIf() { + this.consume(); // `if` / `elif` + const condition = this.parseCondition('then'); + const consequent = node('block', this.parseStatements(new Set())); + + let alternative; + if (this.atWord('elif')) { + alternative = this.parseIf(); + return node('if', [condition, consequent, alternative]); + } + if (this.expectWord('else')) { + alternative = node('block', this.parseStatements(new Set())); + } + this.expectWord('fi'); + return alternative + ? node('if', [condition, consequent, alternative]) + : node('if', [condition, consequent]); + } + + parseLoop() { + const term = this.consume().text === 'while' ? 'while' : 'until'; + const condition = this.parseCondition('do'); + const body = node('block', this.parseStatements(new Set())); + this.expectWord('done'); + return node(term, [condition, body]); + } + + /** `for NAME in word...; do block done` */ + parseFor() { + this.consume(); // `for` + const variable = this.at(TokenType.WORD) ? this.consume().text : '_'; + const items = []; + if (this.expectWord('in')) { + while ( + this.at(TokenType.WORD) && + !BLOCK_TERMINATORS.has(this.current().text) + ) { + items.push(leaf('word', this.consume().text)); + } + } + this.skipSeparators(undefined); + this.expectWord('do'); + const body = node('block', this.parseStatements(new Set())); + this.expectWord('done'); + return node('for', [ + leaf('name', variable), + node('word-list', items), + body, + ]); + } + + /** `case word in pattern) block ;; ... esac` */ + parseCase() { + this.consume(); // `case` + const subject = this.at(TokenType.WORD) + ? leaf('word', this.consume().text) + : leaf('word', ''); + this.expectWord('in'); + this.skipSeparators(undefined); + + const branches = []; + while (!this.atWord('esac') && !this.at(TokenType.EOF)) { + // A pattern list is `a|b|c)`. The lexer already split on `|` and `)`. + const patterns = []; + while (this.at(TokenType.WORD)) { + // `*` is the shell's catch-all pattern, i.e. `default:`. + const pattern = this.consume().text; + patterns.push( + leaf(pattern === '*' ? 'pattern-default' : 'pattern', pattern) + ); + if (this.at(TokenType.OPERATOR, '|')) { + this.consume(); + continue; + } + break; + } + if (this.at(TokenType.OPERATOR, ')')) { + this.consume(); + } + const body = node('block', this.parseStatements(new Set())); + if (this.at(TokenType.OPERATOR, ';;')) { + this.consume(); + } + this.skipSeparators(undefined); + branches.push( + node('case-branch', [node('pattern-list', patterns), body]) + ); + } + this.expectWord('esac'); + return node('case', [subject, node('case-branch-list', branches)]); + } + + /** Consumes `function name [()]`, or `undefined` when that is not next. */ + parseKeywordFunctionName() { + this.consume(); + if (!this.at(TokenType.WORD)) { + return undefined; + } + const name = this.consume().text; + if (this.at(TokenType.OPERATOR, '(')) { + this.consume(); + if (this.at(TokenType.OPERATOR, ')')) { + this.consume(); + } + } + return name; + } + + /** Consumes `name ()`, or `undefined` when that is not next. */ + parseBareFunctionName() { + const isOperator = (offset, text) => + this.tokens[this.position + offset]?.type === TokenType.OPERATOR && + this.tokens[this.position + offset]?.text === text; + + if ( + !this.at(TokenType.WORD) || + !/^[A-Za-z_][A-Za-z0-9_]*$/.test(this.current().text) || + !isOperator(1, '(') || + !isOperator(2, ')') + ) { + return undefined; + } + const name = this.consume().text; + this.consume(); + this.consume(); + return name; + } + + /** `name() { ... }` or `function name { ... }` */ + tryParseFunction() { + const start = this.position; + const name = this.atWord('function') + ? this.parseKeywordFunctionName() + : this.parseBareFunctionName(); + + if (name === undefined) { + this.position = start; + return undefined; + } + + this.skipSeparators(undefined); + if (!this.expectWord('{')) { + this.position = start; + return undefined; + } + const body = node('block', this.parseStatements(new Set())); + this.expectWord('}'); + return node('function', [leaf('name', name), body]); + } + + /** + * Parses the condition of `if`/`while`/`until` up to its introducer keyword + * (`then`/`do`), which the caller has named. + */ + parseCondition(introducer) { + const condition = this.parseStatements(new Set([introducer])); + this.skipSeparators(undefined); + this.expectWord(introducer); + if (condition.length === 1) { + return condition[0]; + } + return node('block', condition); + } + + /** + * True once the current token can no longer belong to a simple command. + * + * @param {object[]} words Words read so far; an empty list means the parser + * is still in command position. + */ + atCommandEnd(words) { + // Reserved words are only reserved in command position: `echo done` runs + // `echo` with the argument `done`, it does not close a loop. Assignment + // prefixes do not consume the command position, so the check stays on + // `words`. + if (words.length === 0 && this.atBlockEnd(new Set())) { + return true; + } + return ( + this.at(TokenType.EOF) || + this.at(TokenType.NEWLINE) || + this.at(TokenType.COMMENT) || + this.at(TokenType.OPERATOR) + ); + } + + /** Reads the assignment prefixes, words and redirects of one command. */ + parseCommandParts() { + const assignments = []; + const words = []; + const redirects = []; + + while (!this.atCommandEnd(words)) { + if (this.at(TokenType.REDIRECT)) { + const operator = this.consume().text; + const target = this.at(TokenType.WORD) ? this.consume().text : ''; + redirects.push(node('redirect', [leaf('word', target)], operator)); + continue; + } + + const token = this.consume(); + const assignment = + words.length === 0 ? ASSIGNMENT.exec(token.text) : undefined; + if (assignment) { + assignments.push( + node('assignment', [leaf('word', assignment[2])], assignment[1]) + ); + continue; + } + words.push(leaf('word', token.text)); + } + + return { assignments, words, redirects }; + } + + parseSimpleCommand() { + const { assignments, words, redirects } = this.parseCommandParts(); + + if (words.length === 0) { + if (assignments.length === 0) { + return undefined; + } + // Bare `NAME=value` statements are variable declarations. + return assignments.length === 1 + ? assignments[0] + : node('block', assignments); + } + + const name = words[0].text; + const rest = words.slice(1); + + if (name === 'export' || name === 'local' || name === 'readonly') { + return this.buildDeclaration(name, rest); + } + if (name === 'set' && rest.length > 0) { + return node('set-option', rest); + } + if (name === 'exit') { + return node('exit', rest); + } + if (name === 'return') { + return node('return', rest); + } + if (name === 'source' || name === '.') { + return node('source', rest); + } + + const children = [ + node('word-list', words), + node('redirect-list', redirects), + ]; + if (assignments.length > 0) { + // Prefixed assignments (`FOO=bar cmd`) scope an environment variable to + // one command; keep them attached so a rule can render them. + return node('command', [ + ...children, + node('assignment-list', assignments), + ]); + } + return node('command', children); + } + + buildDeclaration(keyword, words) { + const declarations = words.map((word) => { + const assignment = ASSIGNMENT.exec(word.text); + return assignment + ? node('assignment', [leaf('word', assignment[2])], assignment[1]) + : node('assignment', [leaf('word', '')], word.text); + }); + const term = keyword === 'export' ? 'export' : 'local'; + return declarations.length === 1 + ? node(term, declarations[0].children, declarations[0].text) + : node( + 'block', + declarations.map((entry) => node(term, entry.children, entry.text)) + ); + } +} + +/** + * Parses a shell script into a plain node tree. + * + * @param {string} source Shell source text. + * @returns {{term: string, children: object[], text?: string}} The `script` root node. + */ +export function parseShellScript(source) { + return new ShellParser(source).parseScript(); +} diff --git a/js/src/fy/translation-rules.mjs b/js/src/fy/translation-rules.mjs new file mode 100644 index 0000000..9466a41 --- /dev/null +++ b/js/src/fy/translation-rules.mjs @@ -0,0 +1,358 @@ +// The Shell -> JavaScript substitution rules. +// +// Every construct the formalizer can produce has exactly one +// `TranslationRule` here: a `LinkQuery` selecting the links it owns, plus one +// template per target language. Nothing about the translation lives outside +// this file — `rule-engine.mjs` only walks the network and substitutes. +// +// Four target languages express the four contexts a node can appear in: +// +// JavaScript a statement `await $`ls`;` +// JavaScript:command text inside $`...` `ls` +// JavaScript:value text inside `...` `ls` without shell quoting +// JavaScript:expression a JavaScript value `process.env.HOME` +// +// Placeholder syntax is documented in `rule-engine.mjs`. + +import { + LinkQuery, + LinkType, + TranslationRule, + TranslationRuleSet, +} from 'meta-language'; + +import { SHELL_LANGUAGE } from './shell-formalizer.mjs'; + +export const TARGET = 'JavaScript'; +export const COMMAND = `${TARGET}:command`; +export const VALUE = `${TARGET}:value`; +export const EXPRESSION = `${TARGET}:expression`; + +/** A `:value` rendering falls back to the `:command` one when unquoting is a no-op. */ +export const LANGUAGE_FALLBACKS = { [VALUE]: [COMMAND] }; + +/** `set -e` and friends map onto `shell.set('e')`. */ +const SET_FLAGS = { e: 'errexit', v: 'verbose', x: 'xtrace' }; + +/** Escapes text so it survives inside a JavaScript template literal. */ +function escapeTemplate(text) { + return text.replace(/([`\\])/g, '\\$1').replace(/\$\{/g, '\\${'); +} + +/** + * Removes the shell quoting from a literal. Quotes are meaningful to the shell + * (word splitting, globbing) so they are kept in `:command` context, but a + * JavaScript value must not contain them. + */ +function stripShellQuotes(text) { + let output = ''; + let quote; + for (let index = 0; index < text.length; index += 1) { + const character = text[index]; + if (character === '\\' && quote !== "'") { + output += text[index + 1] ?? ''; + index += 1; + continue; + } + if (!quote && (character === '"' || character === "'")) { + quote = character; + continue; + } + if (character === quote) { + quote = undefined; + continue; + } + output += character; + } + return output; +} + +/** Placeholder modes available to the templates below. */ +export const MODES = { + /** Raw shell source text of the captured link. */ + text: {}, + /** The captured link rendered as shell command text. */ + command: { language: COMMAND }, + /** The captured link rendered as an unquoted value. */ + value: { language: VALUE }, + /** The captured link rendered as a JavaScript expression. */ + expr: { language: EXPRESSION }, + /** The captured link as a JavaScript string (a template literal). */ + string: { language: VALUE, transform: (text) => `\`${text}\`` }, + /** Literal source text, escaped for a template literal. */ + literal: { transform: escapeTemplate }, + /** Literal source text with shell quoting removed. */ + unquoted: { transform: (text) => escapeTemplate(stripShellQuotes(text)) }, + /** A `set` operand such as `-e`, as a `shell.set` argument. */ + flag: { + transform: (text) => { + const letter = text.replace(/^-+/, ''); + return JSON.stringify(SET_FLAGS[letter] ? letter : text); + }, + }, +}; + +const syntax = (term) => + new LinkQuery({ + linkType: LinkType.Syntax, + language: SHELL_LANGUAGE, + }).withTerm(term); + +/** + * Declares one rule. + * + * @param {string} term Node term the rule owns. + * @param {Record} captures Capture name -> reference index. + * @param {Record} templates Target language -> template text. + */ +function rule(term, captures, templates) { + const created = new TranslationRule(term, syntax(term)); + // meta-language 0.46.0 has no reference-capture API (see + // `docs/meta-language-gaps.md`); the engine reads this property directly. + created.referenceCaptures = captures; + for (const [language, text] of Object.entries(templates)) { + created.withTemplate(language, text); + } + return created; +} + +/** A word expansion: one JavaScript expression, interpolated in command text. */ +function expansion(term, expression) { + return rule( + term, + { name: 0 }, + { + // `{{`/`}}` are literal braces, so this emits `${}`. + [COMMAND]: `\${{${expression}}}`, + [VALUE]: `\${{${expression}}}`, + [EXPRESSION]: expression, + } + ); +} + +/** + * Builds the Shell -> JavaScript rule set. + * + * @param {object} [options] + * @param {boolean} [options.trackExitCode] Emit `exitCode = ...` for each + * command, which is only needed when the script reads `$?`. + * @returns {TranslationRuleSet} + */ +export function buildRuleSet(options = {}) { + const runCommand = options.trackExitCode + ? 'exitCode = (await $`{.:command}`).code;' + : 'await $`{.:command}`;'; + + return new TranslationRuleSet('shell-to-javascript', [ + ...structureRules(runCommand), + ...controlFlowRules(), + ...declarationRules(), + ...expansionRules(), + ]); +} + +/** Script structure, commands, pipelines and the `&&`/`||` operators. */ +function structureRules(runCommand) { + return [ + // ---- Script structure ---------------------------------------------- + rule('script', {}, { [TARGET]: '{*.|\\n}', [COMMAND]: '{*.:command|; }' }), + rule('block', {}, { [TARGET]: '{*.|\\n}', [COMMAND]: '{*.:command|; }' }), + rule('blank', {}, { [TARGET]: '' }), + rule('comment', { body: 0 }, { [TARGET]: '// {body:text}' }), + + // ---- Commands ------------------------------------------------------- + rule( + 'command', + { words: 0, redirects: 1, prefix: 2 }, + { + [TARGET]: runCommand, + [COMMAND]: + '{?prefix}{prefix:command} {/prefix}{words:command}{redirects:command}', + } + ), + rule('word-list', {}, { [COMMAND]: '{*.:command| }' }), + rule('redirect-list', {}, { [COMMAND]: '{*.:command}' }), + rule( + 'redirect', + { operator: 0, target: 1 }, + { [COMMAND]: ' {operator:text}{target:command}' } + ), + rule('assignment-list', {}, { [COMMAND]: '{*.:command| }' }), + rule('word', {}, { [COMMAND]: '{*.:command}', [VALUE]: '{*.:value}' }), + + rule( + 'pipeline', + {}, + { [TARGET]: runCommand, [COMMAND]: '{*.:command| | }' } + ), + rule( + 'subshell', + { body: 0 }, + { [TARGET]: runCommand, [COMMAND]: '({body:command})' } + ), + + // ---- Operators ------------------------------------------------------ + // `a && b` is a conditional, not a boolean expression: `b` runs only when + // `a` exits zero. In command position the shell operator is kept as is. + rule( + 'and', + { left: 0, right: 1 }, + { + [TARGET]: 'if ((await $`{left:command}`).code === 0) {\n {right}\n}', + [COMMAND]: '{left:command} && {right:command}', + } + ), + rule( + 'or', + { left: 0, right: 1 }, + { + [TARGET]: 'if ((await $`{left:command}`).code !== 0) {\n {right}\n}', + [COMMAND]: '{left:command} || {right:command}', + } + ), + ]; +} + +/** `if`, loops, `case` and function definitions. */ +function controlFlowRules() { + return [ + // ---- Control flow --------------------------------------------------- + rule( + 'if', + { condition: 0, consequent: 1, alternative: 2 }, + { + [TARGET]: + 'if ((await $`{condition:command}`).code === 0) {\n {consequent}\n}' + + '{?alternative} else {\n {alternative}\n}{/alternative}', + } + ), + rule( + 'while', + { condition: 0, body: 1 }, + { + [TARGET]: + 'while ((await $`{condition:command}`).code === 0) {\n {body}\n}', + } + ), + rule( + 'until', + { condition: 0, body: 1 }, + { + [TARGET]: + 'while ((await $`{condition:command}`).code !== 0) {\n {body}\n}', + } + ), + rule( + 'for', + { name: 0, items: 1, body: 2 }, + { + [TARGET]: + 'for (const {name:text} of [{*items:string|, }]) {\n {body}\n}', + } + ), + rule( + 'case', + { subject: 0, branches: 1 }, + { [TARGET]: 'switch ({subject:string}) {\n {*branches|\n }\n}' } + ), + rule( + 'case-branch', + { patterns: 0, body: 1 }, + { [TARGET]: '{patterns}\n {body}\n break;' } + ), + rule('pattern-list', {}, { [TARGET]: '{*.|\\n}' }), + rule('pattern', { name: 0 }, { [TARGET]: 'case {name:string}:' }), + // `*` is the shell's catch-all pattern. + rule('pattern-default', {}, { [TARGET]: 'default:' }), + rule( + 'function', + { name: 0, body: 1 }, + { [TARGET]: 'async function {name:text}(...args) {\n {body}\n}' } + ), + rule( + 'name', + { name: 0 }, + { [TARGET]: '{name:text}', [COMMAND]: '{name:text}' } + ), + ]; +} + +/** Assignments, `export`/`local`, and the shell builtins with a JS equivalent. */ +function declarationRules() { + return [ + // ---- Declarations and builtins -------------------------------------- + rule( + 'assignment', + { name: 0, value: 1 }, + { [TARGET]: '{name:text} = {value:string};' } + ), + rule( + 'local', + { name: 0, value: 1 }, + { [TARGET]: 'let {name:text} = {value:string};' } + ), + rule( + 'export', + { name: 0, value: 1 }, + { [TARGET]: 'process.env.{name:text} = {value:string};' } + ), + rule('set-option', {}, { [TARGET]: 'shell.set({*.:flag|, });' }), + rule('exit', {}, { [TARGET]: 'process.exit({*.:value|});' }), + rule( + 'return', + { first: 0 }, + { [TARGET]: 'return{?first} {first:string}{/first};' } + ), + rule('source', { file: 0 }, { [TARGET]: 'await import({file:string});' }), + ]; +} + +/** The leaves produced by the word expander. */ +function expansionRules() { + return [ + // ---- Word expansions ------------------------------------------------- + rule( + 'literal', + { name: 0 }, + { + [COMMAND]: '{name:literal}', + [VALUE]: '{name:unquoted}', + [EXPRESSION]: '{name:string}', + } + ), + expansion('variable', '{name:text}'), + expansion('env-variable', 'process.env.{name:text}'), + expansion('positional', 'args[{name:text}]'), + expansion('script-name', 'process.argv[1]'), + expansion('all-arguments', "args.join(' ')"), + expansion('argument-count', 'args.length'), + expansion('exit-status', 'exitCode'), + expansion('process-id', 'process.pid'), + rule( + 'default-expansion', + { name: 0, value: 1, fallback: 2 }, + { + [COMMAND]: '${{{value:expr} ?? {fallback:string}}}', + [VALUE]: '${{{value:expr} ?? {fallback:string}}}', + [EXPRESSION]: '({value:expr} ?? {fallback:string})', + } + ), + rule( + 'substitution', + { body: 0 }, + { + [COMMAND]: '${{(await $`{body:command}`).stdout.trim()}}', + [VALUE]: '${{(await $`{body:command}`).stdout.trim()}}', + [EXPRESSION]: '(await $`{body:command}`).stdout.trim()', + } + ), + // Anything the expander could not classify stays a shell expansion: the + // escaped `\${...}` reaches command-stream's shell verbatim rather than + // being silently mistranslated. + rule( + 'unsupported-expansion', + { name: 0 }, + { [COMMAND]: '\\${{{name:text}}}', [VALUE]: '\\${{{name:text}}}' } + ), + ]; +} diff --git a/js/src/fy/word-expander.mjs b/js/src/fy/word-expander.mjs new file mode 100644 index 0000000..83b6087 --- /dev/null +++ b/js/src/fy/word-expander.mjs @@ -0,0 +1,215 @@ +// Splits a shell word into typed parts so that every leaf of the network is +// translated by a rule rather than by ad-hoc string surgery. +// +// A word such as `"${DIR}/$(basename "$1").log"` becomes: +// literal(") variable(DIR) literal(/) substitution(...) literal(.log") + +const NAME = /^[A-Za-z_][A-Za-z0-9_]*/; +// The parameter a `${...}` body can start with: a name, a positional number, +// or one of the special parameters. +const PARAMETER = /^(?:[A-Za-z_][A-Za-z0-9_]*|\d+|[@*#?$])/; + +const node = (term, children = [], text = undefined) => ({ + term, + children, + text, +}); +const leaf = (term, text) => node(term, [], text); + +/** Merges adjacent literal parts so the output stays compact. */ +function compact(parts) { + const merged = []; + for (const part of parts) { + const previous = merged[merged.length - 1]; + if (part.term === 'literal' && previous?.term === 'literal') { + previous.text += part.text; + continue; + } + merged.push(part); + } + return merged.filter((part) => part.term !== 'literal' || part.text !== ''); +} + +/** + * Classifies a `$...` expansion. + * + * @param {string} name Variable name or special parameter. + * @param {Set} declared Names the script assigns itself. + */ +function expansionNode(name, declared) { + if (/^\d+$/.test(name)) { + // `$0` is the script itself; `$1` is the first user argument, which is + // index 0 of `process.argv.slice(2)`. Doing the renumbering here keeps the + // translation rules free of arithmetic. + return name === '0' + ? leaf('script-name', name) + : leaf('positional', String(Number(name) - 1)); + } + if (name === '@' || name === '*') { + return leaf('all-arguments', name); + } + if (name === '#') { + return leaf('argument-count', name); + } + if (name === '?') { + return leaf('exit-status', name); + } + if (name === '$') { + return leaf('process-id', name); + } + // A name the script assigns becomes a JavaScript binding; anything else can + // only come from the environment. + return leaf(declared.has(name) ? 'variable' : 'env-variable', name); +} + +/** + * Reads the expansion starting at `index`, if there is one. + * + * @returns {{part: object, end: number} | null} The expansion node and the + * index just past it, or `null` when the character is ordinary text. + */ +function expansionAt(text, index, declared, parseSubstitution) { + const character = text[index]; + + if (character === '`') { + const end = text.indexOf('`', index + 1); + const close = end === -1 ? text.length : end; + const body = parseSubstitution(text.slice(index + 1, close)); + return { part: node('substitution', [body]), end: close + 1 }; + } + + if (character !== '$') { + return null; + } + + if (text[index + 1] === '(') { + const close = matchingParenthesis(text, index + 1); + const body = parseSubstitution(text.slice(index + 2, close)); + return { part: node('substitution', [body]), end: close + 1 }; + } + + if (text[index + 1] === '{') { + const close = matchingBrace(text, index + 1); + const body = text.slice(index + 2, close); + return { part: braceExpansion(body, declared), end: close + 1 }; + } + + const rest = text.slice(index + 1); + const match = NAME.exec(rest); + const name = match ? match[0] : rest.slice(0, 1); + if (name && (match || '@*#?$'.includes(name) || /^\d$/.test(name))) { + return { + part: expansionNode(name, declared), + end: index + 1 + name.length, + }; + } + return null; +} + +/** + * Expands a shell word into part nodes. + * + * @param {string} text Raw word text, quotes included. + * @param {Set} declared Variable names assigned by the script. + * @param {(source: string) => object} parseSubstitution Parses the body of a + * command substitution into a statement node. + * @returns {object[]} Part nodes. + */ +export function expandWord(text, declared, parseSubstitution) { + const parts = []; + let index = 0; + let literal = ''; + let singleQuoted = false; + + const flush = () => { + if (literal !== '') { + parts.push(leaf('literal', literal)); + literal = ''; + } + }; + + while (index < text.length) { + const character = text[index]; + + if (character === "'") { + singleQuoted = !singleQuoted; + literal += character; + index += 1; + continue; + } + // Inside single quotes the shell performs no expansion at all. + if (singleQuoted) { + literal += character; + index += 1; + continue; + } + if (character === '\\') { + literal += text.slice(index, index + 2); + index += 2; + continue; + } + + const expansion = expansionAt(text, index, declared, parseSubstitution); + if (expansion) { + flush(); + parts.push(expansion.part); + index = expansion.end; + continue; + } + + literal += character; + index += 1; + } + + flush(); + return compact(parts); +} + +/** + * `${NAME}`, `${NAME:-default}` and `${NAME:=default}` are supported directly; + * anything more exotic is preserved verbatim as an environment lookup so no + * information is silently dropped. + */ +function braceExpansion(body, declared) { + const plain = PARAMETER.exec(body); + if (plain && plain[0] === body) { + return expansionNode(body, declared); + } + const withDefault = /^(\d+|[@*#?$]|[A-Za-z_][A-Za-z0-9_]*):?[-=](.*)$/s.exec( + body + ); + if (withDefault) { + return node( + 'default-expansion', + [ + expansionNode(withDefault[1], declared), + leaf('literal', withDefault[2]), + ], + withDefault[1] + ); + } + return leaf('unsupported-expansion', body); +} + +function matchingParenthesis(text, start) { + return matchingDelimiter(text, start, '(', ')'); +} + +function matchingBrace(text, start) { + return matchingDelimiter(text, start, '{', '}'); +} + +function matchingDelimiter(text, start, open, close) { + let depth = 0; + for (let index = start; index < text.length; index += 1) { + if (text[index] === open) { + depth += 1; + } else if (text[index] === close) { + depth -= 1; + if (depth === 0) { + return index; + } + } + } + return text.length; +} diff --git a/js/tests/fy-tool.test.mjs b/js/tests/fy-tool.test.mjs new file mode 100644 index 0000000..50270f8 --- /dev/null +++ b/js/tests/fy-tool.test.mjs @@ -0,0 +1,334 @@ +/** + * Tests for the `$fy` tool (shell -> mjs translator). + * + * The translator is rule-based: `formalizeShell` builds a meta-language links + * network and `buildRuleSet` rewrites it. The tests below check both halves + * separately and the CLI on top of them, and finish by executing a translated + * script to prove the output is not merely well-shaped but correct. + */ + +import { beforeEach, describe, expect, test } from 'bun:test'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { LinkType } from 'meta-language'; + +import { $, enableVirtualCommands } from '../src/$.mjs'; +import { formalizeShell, translateShellToMjs } from '../src/fy/index.mjs'; +import { parseShellScript } from '../src/fy/shell-script-parser.mjs'; + +// Other test files call `disableVirtualCommands()` and never re-enable it, so +// `$fy` has to be re-enabled here or it would fall through to `/bin/sh`. +beforeEach(() => { + enableVirtualCommands(); +}); + +/** Translates `source` and returns the body without the generated preamble. */ +function body(source) { + const translated = translateShellToMjs(source, { shebang: false }); + expect(translated.diagnostics).toEqual([]); + return translated.body.trim(); +} + +/** Collects the terms of every syntax node in a formalized script. */ +function terms(source) { + const { network } = formalizeShell(source); + return network + .links() + .filter((link) => link.metadata().linkType === LinkType.Syntax) + .map((link) => link.metadata().term); +} + +describe('shell parser', () => { + test('parses a pipeline into stages', () => { + const script = parseShellScript('ls | grep test | wc -l'); + expect(script.term).toBe('script'); + expect(script.children[0].term).toBe('pipeline'); + expect(script.children[0].children).toHaveLength(3); + }); + + test('parses && and || left-associatively', () => { + const script = parseShellScript('a && b || c'); + expect(script.children[0].term).toBe('or'); + expect(script.children[0].children[0].term).toBe('and'); + }); + + test('parses control flow, functions and case', () => { + const script = parseShellScript(` + if [ -f x ]; then echo a; else echo b; fi + while true; do echo loop; done + for i in 1 2; do echo $i; done + greet() { echo hi; } + case "$1" in start) run ;; *) usage ;; esac + `); + expect(script.children.map((child) => child.term)).toEqual([ + 'if', + 'while', + 'for', + 'function', + 'case', + ]); + }); +}); + +describe('shell formalizer', () => { + test('produces a meta-language network of typed syntax nodes', () => { + const { network, root, terms: used } = formalizeShell('echo hi | wc -l'); + expect(network.link(root).metadata().linkType).toBe(LinkType.Syntax); + expect(network.link(root).metadata().term).toBe('script'); + expect(network.link(root).metadata().language).toBe('Shell'); + expect(used.has('pipeline')).toBe(true); + expect(used.has('command')).toBe(true); + }); + + test('classifies word expansions rather than treating words as text', () => { + // `NAME` is assigned by the script, `HOME` is not, `$1` is positional. + expect(terms('NAME=x\necho "$NAME $HOME $1 $(date)"')).toEqual( + expect.arrayContaining([ + 'variable', + 'env-variable', + 'positional', + 'substitution', + 'literal', + ]) + ); + }); + + test('keeps unsupported expansions instead of dropping them', () => { + expect(terms('echo "${x#prefix}"')).toContain('unsupported-expansion'); + expect(body('echo "${x#prefix}"')).toContain('\\${x#prefix}'); + }); +}); + +describe('translation rules', () => { + test('translates a simple command', () => { + expect(body('ls -la')).toBe('await $`ls -la`;'); + }); + + test('keeps a pipeline as one shell invocation', () => { + expect(body('ls -la | grep test')).toBe('await $`ls -la | grep test`;'); + }); + + test('translates && into a status check, not a boolean', () => { + expect(body('cd /tmp && pwd')).toBe( + 'if ((await $`cd /tmp`).code === 0) {\n await $`pwd`;\n}' + ); + }); + + test('translates || into an inverted status check', () => { + expect(body('test -f x || echo missing')).toBe( + 'if ((await $`test -f x`).code !== 0) {\n await $`echo missing`;\n}' + ); + }); + + test('translates if/else', () => { + expect(body('if [ -f x ]; then echo a; else echo b; fi')).toBe( + 'if ((await $`[ -f x ]`).code === 0) {\n' + + ' await $`echo a`;\n' + + '} else {\n' + + ' await $`echo b`;\n' + + '}' + ); + }); + + test('translates while and until', () => { + expect(body('while true; do echo x; done')).toContain( + 'while ((await $`true`).code === 0) {' + ); + expect(body('until false; do echo x; done')).toContain( + 'while ((await $`false`).code !== 0) {' + ); + }); + + test('translates for..in into for..of', () => { + expect(body('for f in a b; do echo $f; done')).toBe( + 'for (const f of [`a`, `b`]) {\n await $`echo ${f}`;\n}' + ); + }); + + test('translates case into switch with a default branch', () => { + const output = body('case "$1" in start) run ;; *) usage ;; esac'); + expect(output).toContain('switch (`${args[0]}`) {'); + expect(output).toContain('case `start`:'); + expect(output).toContain('default:'); + expect(output).toContain('break;'); + }); + + test('translates functions into async functions', () => { + expect(body('greet() { echo "hi $1"; }')).toBe( + 'async function greet(...args) {\n await $`echo "hi ${args[0]}"`;\n}' + ); + }); + + test('translates assignments, exports and locals', () => { + expect(body('NAME=value')).toBe('NAME = `value`;'); + expect(body('export PATH_EXTRA=/opt/bin')).toBe( + 'process.env.PATH_EXTRA = `/opt/bin`;' + ); + expect(body('f() { local x=1; }')).toContain('let x = `1`;'); + }); + + test('hoists shell variables so reassignment stays valid JavaScript', () => { + const { code } = translateShellToMjs('X=1\nX=2', { shebang: false }); + expect(code).toContain('let X;'); + expect(code).toContain('X = `1`;'); + expect(code).toContain('X = `2`;'); + }); + + test('translates set -e into shell settings', () => { + const { code } = translateShellToMjs('set -e', { shebang: false }); + expect(code).toContain("import { $, shell } from 'command-stream';"); + expect(code).toContain('shell.set("e");'); + }); + + test('translates command substitution into a captured stdout', () => { + expect(body('count=$(ls | wc -l)')).toBe( + 'count = `${(await $`ls | wc -l`).stdout.trim()}`;' + ); + }); + + test('translates parameter expansions', () => { + expect(body('echo "${HOME:-/root}"')).toContain( + '${process.env.HOME ?? `/root`}' + ); + expect(body('echo "$#"')).toContain('${args.length}'); + expect(body('echo "$@"')).toContain("${args.join(' ')}"); + expect(body('echo "$$"')).toContain('${process.pid}'); + }); + + test('tracks $? only when the script reads it', () => { + const withStatus = translateShellToMjs('ls\necho $?', { shebang: false }); + expect(withStatus.code).toContain('let exitCode = 0;'); + expect(withStatus.code).toContain('exitCode = (await $`ls`).code;'); + expect(translateShellToMjs('ls', { shebang: false }).code).not.toContain( + 'exitCode' + ); + }); + + test('preserves comments, blank lines and redirects', () => { + expect(body('# note\n\nls > out.txt 2>&1')).toBe( + '// note\n\nawait $`ls >out.txt 2>&1`;' + ); + }); + + test('replaces the shell shebang with a node one', () => { + const { code } = translateShellToMjs('#!/bin/bash\nls'); + expect(code.startsWith('#!/usr/bin/env node\n')).toBe(true); + expect(code).not.toContain('/bin/bash'); + }); +}); + +describe('$fy command', () => { + test('shows usage when given no input', async () => { + const result = await $`$fy`; + expect(result.code).toBe(1); + expect(result.stderr).toContain('$fy - Convert shell scripts'); + expect(result.stderr).toContain('Usage:'); + }); + + test('shows help on --help', async () => { + const result = await $`$fy --help`; + expect(result.code).toBe(0); + expect(result.stdout).toContain('Usage:'); + }); + + test('translates from stdin', async () => { + const result = await $({ stdin: 'cd /tmp && pwd' })`$fy`; + expect(result.code).toBe(0); + expect(result.stdout).toContain("import { $ } from 'command-stream';"); + expect(result.stdout).toContain('if ((await $`cd /tmp`).code === 0) {'); + }); + + test('translates a file to stdout and to an output file', async () => { + const directory = mkdtempSync(join(tmpdir(), 'fy-')); + try { + const input = join(directory, 'in.sh'); + const output = join(directory, 'out.mjs'); + writeFileSync(input, 'echo hello\n'); + + const printed = await $`$fy ${input}`; + expect(printed.code).toBe(0); + expect(printed.stdout).toContain('await $`echo hello`;'); + + const written = await $`$fy ${input} ${output}`; + expect(written.code).toBe(0); + expect(readFileSync(output, 'utf8')).toContain('await $`echo hello`;'); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + test('reports a missing input file', async () => { + const result = await $`$fy /nonexistent/script.sh`; + expect(result.code).toBe(1); + expect(result.stderr).toContain('cannot read'); + }); +}); + +describe('translated scripts run', () => { + test('a translated script produces the same output as the shell script', async () => { + const directory = mkdtempSync(join(tmpdir(), 'fy-run-')); + try { + const script = [ + '#!/bin/sh', + 'GREETING=hello', + 'for name in world there; do', + ' echo "$GREETING $name"', + 'done', + 'if true; then', + ' echo yes', + 'fi', + 'echo done | tr a-z A-Z', + ].join('\n'); + + const shellFile = join(directory, 'script.sh'); + const moduleFile = join(directory, 'script.mjs'); + writeFileSync(shellFile, `${script}\n`); + + const { code, diagnostics } = translateShellToMjs(script, { + // A file URL, not a path: `URL.pathname` yields `/D:/...` on Windows, + // which is not importable. + moduleName: new URL('../src/$.mjs', import.meta.url).href, + }); + expect(diagnostics).toEqual([]); + writeFileSync(moduleFile, code); + + const fromShell = await $`sh ${shellFile}`; + const fromModule = await $`bun ${moduleFile}`; + expect(fromModule.code).toBe(0); + expect(fromModule.stdout).toBe(fromShell.stdout); + expect(fromShell.stdout.trim().split('\n')).toEqual([ + 'hello world', + 'hello there', + 'yes', + 'DONE', + ]); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + test('translates a script with CRLF line endings', () => { + const translated = translateShellToMjs('ls\r\necho hi\r\n', { + shebang: false, + }); + expect(translated.diagnostics).toEqual([]); + expect(translated.body).toBe('await $`ls`;\nawait $`echo hi`;\n'); + }); + + test('reproduces the golden fixture shared with the Rust suite', () => { + // `rust/tests/fy.rs` asserts the same two files, which is what keeps both + // translators byte-for-byte equivalent. See `fixtures/fy/README.md`. + const fixture = (name) => + readFileSync( + new URL(`../../fixtures/fy/${name}`, import.meta.url), + 'utf8' + ); + + const translated = translateShellToMjs(fixture('sample.sh')); + expect(translated.diagnostics).toEqual([]); + expect(translated.code).toBe(fixture('sample.mjs')); + }); +}); diff --git a/rust/Cargo.lock b/rust/Cargo.lock index f4984c0..37d889d 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2,6 +2,25 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "abnf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "087113bd50d9adce24850eed5d0476c7d199d532fce8fab5173650331e09033a" +dependencies = [ + "abnf-core", + "nom 7.1.3", +] + +[[package]] +name = "abnf-core" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c44e09c43ae1c368fb91a03a566472d0087c26cf7e1b9e8e289c14ede681dd7d" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -11,6 +30,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -20,12 +45,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -104,6 +173,18 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +[[package]] +name = "bnf" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35b77b055f8cb1d566fa4ef55bc699f60eefb17927dc25fa454a05b6fabf7aa4" +dependencies = [ + "getrandom 0.3.4", + "hashbrown 0.16.1", + "nom 8.0.0", + "rand", +] + [[package]] name = "bstr" version = "1.12.1" @@ -168,6 +249,52 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clap" +version = "4.5.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "command-stream" version = "0.14.0" @@ -178,6 +305,7 @@ dependencies = [ "filetime", "glob", "libc", + "meta-language", "nix 0.29.0", "once_cell", "portable-pty", @@ -198,6 +326,75 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "counter" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66e8e052be91f1c8aae2c1d81307d9f6e67f5f37001e3ddee419e971e73f03bc" +dependencies = [ + "num-traits", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "difflib" version = "0.4.0" @@ -210,6 +407,17 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" +[[package]] +name = "ebnf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2ace35cc1894cd8e6f972fa215cb5d9ddd7ffd63302139a4554e67d2ae5f8f" +dependencies = [ + "nom 7.1.3", + "parse-hyperlinks", + "serde", +] + [[package]] name = "either" version = "1.15.0" @@ -222,6 +430,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7f84e12ccf0a7ddc17a6c41c93326024c42920d7ee630d04950e6926645c0fe" +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -237,6 +451,9 @@ name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +dependencies = [ + "getrandom 0.2.17", +] [[package]] name = "filedescriptor" @@ -267,12 +484,43 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "fst" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" + [[package]] name = "futures-core" version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -280,9 +528,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasip2", + "wasm-bindgen", ] [[package]] @@ -291,6 +541,61 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "html-escape" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c1ff2d1cbf39efe5af0900ced8a069b5e61557a17544eb0c4a50239937389e" + [[package]] name = "iana-time-zone" version = "0.1.64" @@ -315,6 +620,50 @@ dependencies = [ "cc", ] +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -331,6 +680,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json5_nodes" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40f9c35bff86d8321148897f878ad862c22b09549eff24c805bf00eebcb4a2e" +dependencies = [ + "hashlink", + "pest", + "pest_derive", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -354,6 +714,137 @@ dependencies = [ "redox_syscall 0.7.0", ] +[[package]] +name = "lingua" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40d9129bb9fe42c95d1bd420d6891607eaff17df16ee15674aed2d05b0ec8f4" +dependencies = [ + "counter", + "dashmap", + "fastrand", + "fst", + "include_dir", + "itertools", + "lingua-arabic-language-model", + "lingua-bengali-language-model", + "lingua-chinese-language-model", + "lingua-english-language-model", + "lingua-french-language-model", + "lingua-hindi-language-model", + "lingua-portuguese-language-model", + "lingua-russian-language-model", + "lingua-spanish-language-model", + "lingua-urdu-language-model", + "maplit", + "rayon", + "regex", + "serde", + "serde-wasm-bindgen", + "strum", + "strum_macros", + "wasm-bindgen", +] + +[[package]] +name = "lingua-arabic-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bcc254ff44209c2a50dea58a644f9c257a0aa5cd7e1739fd9c373fe4c7456cf" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bengali-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f94131563e0e0c571cebf03f867d0e1b7dd15278f7cab3e4ba8b0f8026072cc" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-chinese-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ca7fa9f7671d684c82c168725f380fc873f14d6f4e8c82f0da681bcc0048d1" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-english-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97102de08b134a49f1cce05a1b6f5bf08ef21fe858074ae2b794e7892c43dd4b" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-french-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45508227e42c9cc5eb202e17c4c40e38ea7b9be9421faeb3ab0fb7ac37d9c681" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-hindi-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca88af8dede1a30fcd79318b9c22f3662785fd87ed5b39881d19770651bf720" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-portuguese-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "432eda7984456055033ffa168037be9afe0f5c9ecd891bf5f27435247d496b47" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-russian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc0850578299531b50192be2c1de1af651f1289784253645e737fe456a6a138" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-spanish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56395a8d96c892130a9efb433c5f042977fb75ae5fb6e0058c8814a947459137" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-urdu-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d308fdae7c8caa05ffe455e8a2e568e5f018cb2742f588a810ecf306b890622" +dependencies = [ + "include_dir", +] + +[[package]] +name = "links-notation" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4c952b42a8c6ff6f849d7cafe3b1e13f1063a51bbb144bc6c62026ab327814c" +dependencies = [ + "nom 8.0.0", +] + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -375,12 +866,76 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "memchr" version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "meta-language" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fe5d961129d219bf2b6a5471d2eab55debb44f5bf73f393d46daeefe51e987f" +dependencies = [ + "abnf", + "bnf", + "clap", + "csv", + "ebnf", + "json5_nodes", + "lingua", + "links-notation", + "pest_meta", + "serde_json", + "tree-sitter", + "tree-sitter-c", + "tree-sitter-c-sharp", + "tree-sitter-cpp", + "tree-sitter-css", + "tree-sitter-go", + "tree-sitter-graphql", + "tree-sitter-html", + "tree-sitter-ini", + "tree-sitter-java", + "tree-sitter-javascript", + "tree-sitter-json", + "tree-sitter-kotlin-ng", + "tree-sitter-lua", + "tree-sitter-pascal", + "tree-sitter-php", + "tree-sitter-proto", + "tree-sitter-python", + "tree-sitter-r", + "tree-sitter-ruby", + "tree-sitter-rust", + "tree-sitter-scala", + "tree-sitter-sequel", + "tree-sitter-swift", + "tree-sitter-toml-ng", + "tree-sitter-typescript", + "tree-sitter-vb-dotnet", + "tree-sitter-xml", + "tree-sitter-yaml", + "ts-parser-perl", + "unicode-bidi", + "unicode-normalization", + "unicode-segmentation", + "whatlang", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "mio" version = "1.1.1" @@ -416,6 +971,25 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -427,9 +1001,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "parking_lot" @@ -454,6 +1034,66 @@ dependencies = [ "windows-link", ] +[[package]] +name = "parse-hyperlinks" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0181d37c4d5ae35cc8be7cf823c1a933005661da6a08bcb2855aa392c9a54b8e" +dependencies = [ + "html-escape", + "nom 7.1.3", + "percent-encoding", + "thiserror 1.0.69", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pest" +version = "2.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df728be843c7070fab6ab7c328c4e9e9d78e23bf749c0669c86ee7ebfa050a2" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2dd6fc3b26b3462ee188aac870f5a41d398f1cd5e2408d16531bd71c9591fd" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a7a9205cfb6f596a9e8b689c0a15f9ceb7a1aafae7aaf788150ac65b29975b6" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85abd351c0de1e8384fc791a0737111a350394937e92b956b743dac12429f57c" +dependencies = [ + "pest", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -481,6 +1121,15 @@ dependencies = [ "winreg", ] +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "predicates" version = "3.1.3" @@ -532,6 +1181,55 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -598,6 +1296,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "scopeguard" version = "1.2.0" @@ -614,6 +1318,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_core" version = "1.0.228" @@ -640,6 +1355,7 @@ version = "1.0.148" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3084b546a1dd6289475996f182a22aba973866ea8e8b02c51d9f46b1336a22da" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -706,6 +1422,36 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "syn" version = "2.0.111" @@ -724,7 +1470,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" dependencies = [ "fastrand", - "getrandom", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -776,6 +1522,21 @@ dependencies = [ "syn", ] +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.48.0" @@ -828,12 +1589,349 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "tree-sitter" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78f873475d258561b06f1c595d93308a7ed124d9977cb26b148c2084a4a3cc87" +dependencies = [ + "cc", + "regex", + "regex-syntax", + "serde_json", + "streaming-iterator", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-c" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9b2eb57a55fed6b00812912e730b7a275cf4fe98bfd6a5d76263d4438371728" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-c-sharp" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1aac67f1ad71de1d6d39708d34811081c26dfa495658de6c14c34200849357c" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-cpp" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2196ea9d47b4ab4a31b9297eaa5a5d19a0b121dceb9f118f6790ad0ab94743" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-css" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5cbc5e18f29a2c6d6435891f42569525cf95435a3e01c2f1947abcde178686f" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-go" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8560a4d2f835cc0d4d2c2e03cbd0dde2f6114b43bc491164238d333e28b16ea" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-graphql" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efedc4cac157161cc23a0adc4553a2cedc908e1cd754b6cd033a919bb81ce5d6" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-html" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261b708e5d92061ede329babaaa427b819329a9d427a1d710abb0f67bbef63ee" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-ini" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "387f79682cd53b7c0a5777c96e601a02b9965a787984ef86dbb8952bdab2d62f" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-java" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa6cbcdc8c679b214e616fd3300da67da0e492e066df01bcf5a5921a71e90d6" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-javascript" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-json" +version = "0.24.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d727acca406c0020cffc6cf35516764f36c8e3dc4408e5ebe2cb35a947ec471" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-kotlin-ng" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e800ebbda938acfbf224f4d2c34947a31994b1295ee6e819b65226c7b51b4450" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-language" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" + +[[package]] +name = "tree-sitter-lua" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb9adf0965fec58e7660cbb3a059dbb12ebeec9459e6dcbae3db004739641e" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-pascal" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb51e9a57493fd237e4517566749f7f7453349261a72a427e5f11d3b34b72a8" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-php" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c17c3ab69052c5eeaa7ff5cd972dd1bc25d1b97ee779fec391ad3b5df5592" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-proto" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e410ccb5fa3cbd6bf7b8e512ecf7ad9d5254395b822bfe9f751b50fa978f31c" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-python" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf85fd39652e740bf60f46f4cda9492c3a9ad75880575bf14960f775cb74a1c" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-r" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc9954ec870dcad6cffdd302b405306c68cf031ed79a78cd9746f6740d9fe20" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-ruby" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0484ea4ef6bb9c575b4fdabde7e31340a8d2dbc7d52b321ac83da703249f95" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-rust" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439e577dbe07423ec2582ac62c7531120dbfccfa6e5f92406f93dd271a120e45" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-scala" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3394d6bc99bceae03c75482a93f1bcefff11e69d3a405f1410e864212b52739a" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-sequel" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d198ad3c319c02e43c21efa1ec796b837afcb96ffaef1a40c1978fbdcec7d17" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-swift" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe36052155b9dd69ca82b3b8f1b4ccfb2d867125ac1a4db1dd7331829242668c" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-toml-ng" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9adc2c898ae49730e857d75be403da3f92bb81d8e37a2f918a08dd10de5ebb1" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-typescript" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-vb-dotnet" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e4a199378505f9305c3d01d6ece8a8d34035f310de1c727693f8a2296025e" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-xml" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e670041f591d994f54d597ddcd8f4ebc930e282c4c76a42268743b71f0c8b6b3" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-yaml" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c223db85f05e34794f065454843b0668ebc15d240ada63e2b5939f43ce7c97" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "ts-parser-perl" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d125f7bfdd1fd82a7e87d2e85793f486ad1b5f465144e9e22132dbe5bd80e694" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "unicode-width" version = "0.1.14" @@ -948,6 +2046,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "whatlang" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5e8f38b596e2a359b755342473520a99421e43658548c79489ee221b728c107" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "which" version = "7.0.3" @@ -1145,6 +2252,26 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.2" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index cb04d1f..1a72123 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -34,6 +34,7 @@ chrono = "0.4" filetime = "0.2" portable-pty = "0.9" vt100 = "0.15" +meta-language = { version = "0.54", default-features = false } [dev-dependencies] tokio-test = "0.4" diff --git a/rust/changelog.d/20260725_120000_fy_rule_based_translation.md b/rust/changelog.d/20260725_120000_fy_rule_based_translation.md new file mode 100644 index 0000000..2650fcc --- /dev/null +++ b/rust/changelog.d/20260725_120000_fy_rule_based_translation.md @@ -0,0 +1,11 @@ +--- +bump: minor +--- + +### Added + +- `$fy` virtual command: a shell-to-JavaScript translator implemented as a + rule-based translation over a link-foundation/meta-language links network + (formalize, then substitute). It mirrors the JavaScript implementation + byte-for-byte, which `rust/tests/fy.rs` and `js/tests/fy-tool.test.mjs` both + assert against the shared golden fixture in `fixtures/fy/`. diff --git a/rust/examples/fy_translate.rs b/rust/examples/fy_translate.rs new file mode 100644 index 0000000..4233c17 --- /dev/null +++ b/rust/examples/fy_translate.rs @@ -0,0 +1,33 @@ +//! Translates a shell script into a command-stream module with `$fy`. +//! +//! ```sh +//! cargo run --example fy_translate -- ../fixtures/fy/sample.sh +//! ``` +//! +//! The translation is rule-based: the script is first formalized as a +//! link-foundation/meta-language links network, then rewritten into JavaScript +//! by a `TranslationRuleSet`. Diagnostics for untranslated constructs go to +//! stderr, so the stdout stream is exactly the translated module. + +use command_stream::fy::{translate_shell_to_mjs, TranslateOptions}; + +fn main() { + let Some(path) = std::env::args().nth(1) else { + eprintln!("usage: fy_translate "); + std::process::exit(1); + }; + + let source = match std::fs::read_to_string(&path) { + Ok(source) => source, + Err(error) => { + eprintln!("cannot read '{path}': {error}"); + std::process::exit(1); + } + }; + + let translation = translate_shell_to_mjs(&source, &TranslateOptions::default()); + print!("{}", translation.code); + for diagnostic in &translation.diagnostics { + eprintln!("$fy: warning: {diagnostic}"); + } +} diff --git a/rust/src/commands/fy.rs b/rust/src/commands/fy.rs new file mode 100644 index 0000000..52d5eb7 --- /dev/null +++ b/rust/src/commands/fy.rs @@ -0,0 +1,205 @@ +//! Virtual `$fy` command implementation — shell to mjs translator. +//! +//! This module is only the command-line surface. The translation itself is +//! rule-based and lives in [`crate::fy`]: the script is first formalized as a +//! link-foundation/meta-language links network, then rewritten into JavaScript +//! by a `TranslationRuleSet`. + +use std::collections::BTreeSet; +use std::fs; + +use crate::commands::CommandContext; +use crate::fy::{translate_shell_to_mjs, TranslateOptions}; +use crate::utils::{CommandResult, VirtualUtils}; + +const USAGE: &str = r#"$fy - Convert shell scripts to command-stream JavaScript modules + +Usage: + $fy # Translate and print to stdout + $fy # Translate and save to a file + echo "ls -la" | $fy # Translate from stdin + +Options: + --no-shebang Omit the leading #!/usr/bin/env node line + -h, --help Show this help + +Examples: + $fy deploy.sh # Print the translated deploy script + $fy build.sh build.mjs # Translate build.sh into build.mjs + echo "cd /tmp && ls" | $fy # Translate a one-liner from stdin + +The translation is driven by substitution rules over a meta-language links +network, so pipelines, &&/||, if/while/until/for/case, functions, redirects, +variable assignments and ${...} expansions all translate structurally rather +than textually. +"#; + +/// Renders translation diagnostics as shell-style warnings. +fn format_diagnostics(diagnostics: &[String]) -> String { + let unique: BTreeSet<&String> = diagnostics.iter().collect(); + unique + .into_iter() + .map(|message| format!("$fy: warning: {message}\n")) + .collect() +} + +/// Execute the `$fy` command. +pub async fn fy(ctx: CommandContext) -> CommandResult { + if ctx.args.iter().any(|arg| arg == "-h" || arg == "--help") { + return CommandResult::success(USAGE); + } + + let options = TranslateOptions { + shebang: !ctx.args.iter().any(|arg| arg == "--no-shebang"), + ..TranslateOptions::default() + }; + let positional: Vec<&String> = ctx + .args + .iter() + .filter(|argument| !argument.starts_with('-')) + .collect(); + + // A file argument always wins: `$fy in.sh` must read `in.sh` even when the + // command happens to inherit a non-empty stdin from its caller. + let stdin = ctx + .stdin + .as_deref() + .filter(|stdin| positional.is_empty() && !stdin.is_empty()); + + let source = match (stdin, positional.first()) { + (Some(stdin), _) => stdin.to_string(), + (None, Some(path)) => { + let resolved = VirtualUtils::resolve_path(path, Some(&ctx.get_cwd())); + match fs::read_to_string(&resolved) { + Ok(source) => source, + Err(error) => { + return CommandResult::error(format!("$fy: cannot read '{path}': {error}\n")) + } + } + } + (None, None) => return CommandResult::error(USAGE), + }; + + let translated = translate_shell_to_mjs(&source, &options); + let stderr = format_diagnostics(&translated.diagnostics); + + let output_file = if stdin.is_some() { + None + } else { + positional.get(1) + }; + let Some(output_file) = output_file else { + return CommandResult { + stdout: translated.code, + stderr, + code: 0, + }; + }; + + let resolved = VirtualUtils::resolve_path(output_file, Some(&ctx.get_cwd())); + if let Err(error) = fs::write(&resolved, &translated.code) { + return CommandResult::error(format!( + "{stderr}$fy: cannot write '{output_file}': {error}\n" + )); + } + CommandResult { + stdout: format!("Translated to '{output_file}'\n"), + stderr, + code: 0, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tempfile::NamedTempFile; + + #[tokio::test] + async fn translates_a_file_argument() { + let mut temp = NamedTempFile::new().unwrap(); + writeln!(temp, "ls -la").unwrap(); + + let ctx = CommandContext::new(vec![temp.path().to_string_lossy().to_string()]); + let result = fy(ctx).await; + + assert!(result.is_success()); + assert!(result.stdout.contains("await $`ls -la`;")); + assert!(result.stdout.starts_with("#!/usr/bin/env node\n")); + } + + #[tokio::test] + async fn translates_stdin() { + let mut ctx = CommandContext::new(vec![]); + ctx.stdin = Some("echo hi\n".to_string()); + let result = fy(ctx).await; + + assert!(result.is_success()); + assert!(result.stdout.contains("await $`echo hi`;")); + } + + #[tokio::test] + async fn a_file_argument_wins_over_inherited_stdin() { + let mut temp = NamedTempFile::new().unwrap(); + writeln!(temp, "pwd").unwrap(); + + let mut ctx = CommandContext::new(vec![temp.path().to_string_lossy().to_string()]); + ctx.stdin = Some("echo from-stdin\n".to_string()); + let result = fy(ctx).await; + + assert!(result.stdout.contains("await $`pwd`;")); + assert!(!result.stdout.contains("from-stdin")); + } + + #[tokio::test] + async fn writes_to_an_output_file() { + let mut input = NamedTempFile::new().unwrap(); + writeln!(input, "echo hi").unwrap(); + let output = NamedTempFile::new().unwrap(); + + let ctx = CommandContext::new(vec![ + input.path().to_string_lossy().to_string(), + output.path().to_string_lossy().to_string(), + ]); + let result = fy(ctx).await; + + assert!(result.is_success()); + assert!(result.stdout.starts_with("Translated to '")); + let written = fs::read_to_string(output.path()).unwrap(); + assert!(written.contains("await $`echo hi`;")); + } + + #[tokio::test] + async fn omits_the_shebang_on_request() { + let mut temp = NamedTempFile::new().unwrap(); + writeln!(temp, "ls").unwrap(); + + let ctx = CommandContext::new(vec![ + temp.path().to_string_lossy().to_string(), + "--no-shebang".to_string(), + ]); + let result = fy(ctx).await; + + assert!(!result.stdout.contains("#!/usr/bin/env node")); + } + + #[tokio::test] + async fn reports_usage_without_input() { + let result = fy(CommandContext::new(vec![])).await; + assert_eq!(result.code, 1); + assert!(result.stderr.contains("Usage:")); + + let help = fy(CommandContext::new(vec!["--help".to_string()])).await; + assert!(help.is_success()); + assert!(help.stdout.contains("Usage:")); + } + + #[tokio::test] + async fn reports_an_unreadable_file() { + let ctx = CommandContext::new(vec!["/nonexistent/script.sh".to_string()]); + let result = fy(ctx).await; + + assert_eq!(result.code, 1); + assert!(result.stderr.starts_with("$fy: cannot read ")); + } +} diff --git a/rust/src/commands/mod.rs b/rust/src/commands/mod.rs index eaeaaf0..86881c1 100644 --- a/rust/src/commands/mod.rs +++ b/rust/src/commands/mod.rs @@ -13,6 +13,7 @@ mod echo; mod env; mod exit; mod r#false; +mod fy; mod ls; mod mkdir; mod mv; @@ -34,6 +35,7 @@ pub use dirname::dirname; pub use echo::echo; pub use env::env; pub use exit::exit; +pub use fy::fy; pub use ls::ls; pub use mkdir::mkdir; pub use mv::mv; diff --git a/rust/src/commands/which.rs b/rust/src/commands/which.rs index 671ea7e..a9da426 100644 --- a/rust/src/commands/which.rs +++ b/rust/src/commands/which.rs @@ -6,7 +6,7 @@ use crate::utils::{CommandResult, VirtualUtils}; /// List of virtual (shell builtin) commands const VIRTUAL_COMMANDS: &[&str] = &[ "echo", "pwd", "cd", "true", "false", "sleep", "cat", "ls", "mkdir", "rm", "touch", "cp", "mv", - "basename", "dirname", "env", "exit", "which", "yes", "seq", "test", + "basename", "dirname", "env", "exit", "which", "yes", "seq", "test", "$fy", ]; /// Execute the which command diff --git a/rust/src/fy/engine.rs b/rust/src/fy/engine.rs new file mode 100644 index 0000000..eb5a247 --- /dev/null +++ b/rust/src/fy/engine.rs @@ -0,0 +1,430 @@ +//! Recursive rule engine for translating a meta-language links network. +//! +//! meta-language models translation as a `TranslationRuleSet`: a query per +//! source construct plus a template per target language. Its own +//! `TranslationRuleSet::render` is `pub(crate)`, applies the *first* rule that +//! matches and does not recurse into a template's placeholders, so it cannot +//! translate a nested tree (see `js/docs/meta-language-gaps.md` for the report +//! filed upstream). +//! +//! This engine keeps meta-language's rule model — the rules really are +//! `TranslationRule`s carrying `LinkQuery`s and per-language templates — and +//! supplies only the missing evaluation strategy: +//! +//! * bottom-up recursion through placeholders, +//! * variadic placeholders over a node's children, +//! * conditional segments for optional captures, +//! * automatic indentation of multi-line substitutions. +//! +//! Once meta-language grows these, this file collapses into a call to its own +//! rule-set rendering. + +use std::collections::HashMap; + +use meta_language::link_network::{LinkId, LinkNetwork, LinkType}; +use meta_language::translation_rules::{TranslationRule, TranslationRuleSet}; + +use super::formalizer::CHUNK_TERM; + +/// How a placeholder renders its captured link. +#[derive(Clone, Default)] +pub struct Mode { + /// Render the captured link in this target language instead of the current one. + pub language: Option, + /// Post-process the rendered text. + pub transform: Option String>, +} + +/// A parsed `{...}` placeholder. +struct Placeholder<'a> { + variadic: bool, + name: &'a str, + mode: Option<&'a str>, + separator: Option<&'a str>, + length: usize, +} + +/// Parses `{name}`, `{name:mode}` or `{*name:mode|separator}` at the start of +/// `source`, or the conditional opener `{?name}`. +fn parse_placeholder(source: &str) -> Option> { + let body_end = source.find('}')?; + if !source.starts_with('{') { + return None; + } + let mut body = &source[1..body_end]; + if body.is_empty() || body.starts_with('?') || body.starts_with('/') { + return None; + } + + let variadic = body.starts_with('*'); + if variadic { + body = &body[1..]; + } + + let (head, separator) = match body.find('|') { + Some(bar) => (&body[..bar], Some(&body[bar + 1..])), + None => (body, None), + }; + let (name, mode) = match head.find(':') { + Some(colon) => (&head[..colon], Some(&head[colon + 1..])), + None => (head, None), + }; + + if name != "." && !is_capture_name(name) { + return None; + } + Some(Placeholder { + variadic, + name, + mode, + separator, + length: body_end + 1, + }) +} + +fn is_capture_name(name: &str) -> bool { + let mut characters = name.chars(); + matches!(characters.next(), Some(first) if first.is_ascii_alphabetic() || first == '_') + && characters.all(|character| { + character.is_ascii_alphanumeric() || character == '_' || character == '-' + }) +} + +/// Parses the conditional opener `{?name}`. +fn parse_conditional(source: &str) -> Option<(&str, usize)> { + let body_end = source.find('}')?; + let body = source.strip_prefix("{?")?; + let name = &body[..body_end - 2]; + if !is_capture_name(name) { + return None; + } + Some((name, body_end + 1)) +} + +fn decode_separator(separator: Option<&str>) -> String { + let Some(separator) = separator else { + return String::new(); + }; + let mut decoded = String::new(); + let mut characters = separator.chars(); + while let Some(character) = characters.next() { + if character != '\\' { + decoded.push(character); + continue; + } + match characters.next() { + Some('n') => decoded.push('\n'), + Some('t') => decoded.push('\t'), + Some('s') => decoded.push(' '), + Some(other) => decoded.push(other), + None => decoded.push('\\'), + } + } + decoded +} + +/// Indents every line after the first by `indent`. +fn indent_continuation(value: &str, indent: &str) -> String { + if indent.is_empty() || !value.contains('\n') { + return value.to_string(); + } + value + .split('\n') + .enumerate() + .map(|(position, line)| { + if position == 0 || line.is_empty() { + line.to_string() + } else { + format!("{indent}{line}") + } + }) + .collect::>() + .join("\n") +} + +/// The whitespace run at the end of `output`, when `output` ends on a blank +/// line prefix. +fn current_indent(output: &str) -> &str { + let line = match output.rfind('\n') { + Some(newline) => &output[newline + 1..], + None => output, + }; + if line + .chars() + .all(|character| character == ' ' || character == '\t') + { + line + } else { + "" + } +} + +/// Renders a links network by recursively substituting translation-rule templates. +pub struct RuleEngine<'a> { + network: &'a LinkNetwork, + rule_set: &'a TranslationRuleSet, + language: String, + fallbacks: HashMap>, + modes: HashMap, + rules_by_link: HashMap, + diagnostics: Vec, +} + +impl<'a> RuleEngine<'a> { + /// Builds an engine for `network`, resolving each rule's query once. + pub fn new( + network: &'a LinkNetwork, + rule_set: &'a TranslationRuleSet, + language: &str, + fallbacks: HashMap>, + modes: HashMap, + ) -> Self { + // Rule selection is query-driven: each rule's `LinkQuery` is evaluated + // against the whole network once, and the first rule to claim a link + // owns it. + let mut rules_by_link = HashMap::new(); + for (index, rule) in rule_set.rules().iter().enumerate() { + for query_match in network.query_matches(rule.query()) { + rules_by_link.entry(query_match.link_id()).or_insert(index); + } + } + + RuleEngine { + network, + rule_set, + language: language.to_string(), + fallbacks, + modes, + rules_by_link, + diagnostics: Vec::new(), + } + } + + /// Diagnostics collected while rendering (constructs with no rule). + pub fn report(&self) -> Vec { + self.diagnostics.clone() + } + + /// Renders one link in the engine's default target language. + pub fn render_root(&mut self, link_id: LinkId) -> String { + let language = self.language.clone(); + self.render(link_id, &language) + } + + /// Renders one link in `language`. + pub fn render(&mut self, link_id: LinkId, language: &str) -> String { + let Some(&index) = self.rules_by_link.get(&link_id) else { + // Token links are leaves and render as their own text; an unclaimed + // *syntax* node means a construct nobody translates. + if let Some(link) = self.network.link(link_id) { + if link.metadata().link_type() == Some(LinkType::Syntax) { + let term = link.metadata().term().unwrap_or("?").to_string(); + self.diagnostics + .push(format!("no translation rule for `{term}`")); + } + } + return self.captured_text(link_id); + }; + + let rule = &self.rule_set.rules()[index]; + let Some(template) = self.template_for(rule, language) else { + self.diagnostics + .push(format!("rule `{}` has no {language} template", rule.name())); + return self.captured_text(link_id); + }; + self.expand(&template, index, link_id, language) + } + + /// The source text a link captures, without applying any rule. + fn captured_text(&self, link_id: LinkId) -> String { + self.network.render_source_from(link_id, "Shell") + } + + /// Looks a template up through the configured fallback chain. + fn template_for(&self, rule: &TranslationRule, language: &str) -> Option { + if let Some(template) = rule.templates().get(language) { + return Some(template.source().to_string()); + } + for candidate in self.fallbacks.get(language)? { + if let Some(template) = rule.templates().get(candidate) { + return Some(template.source().to_string()); + } + } + None + } + + /// Resolves a capture name to a child link id. + fn resolve(&self, rule_index: usize, link_id: LinkId, name: &str) -> Option { + if name == "." { + return Some(link_id); + } + let index = *self.rule_set.rules()[rule_index] + .reference_captures() + .get(name)?; + self.network.link(link_id)?.references().get(index).copied() + } + + /// The renderable children of a link, with chunk nodes flattened. + fn children_of(&self, link_id: LinkId) -> Vec { + let Some(link) = self.network.link(link_id) else { + return Vec::new(); + }; + let mut children = Vec::new(); + for &child in link.references() { + let is_chunk = self + .network + .link(child) + .and_then(|child| child.metadata().term().map(|term| term == CHUNK_TERM)) + .unwrap_or(false); + if is_chunk { + children.extend(self.children_of(child)); + } else { + children.push(child); + } + } + children + } + + /// Renders `link_id` in the mode a placeholder asked for. + fn render_mode(&mut self, link_id: LinkId, mode: Option<&str>, language: &str) -> String { + let Some(mode) = mode else { + return self.render(link_id, language); + }; + let Some(definition) = self.modes.get(mode).cloned() else { + self.diagnostics + .push(format!("unknown placeholder mode `{mode}`")); + return self.render(link_id, language); + }; + let rendered = match definition.language.as_deref() { + Some(language) => self.render(link_id, language), + None => self.captured_text(link_id), + }; + match definition.transform { + Some(transform) => transform(&rendered), + None => rendered, + } + } + + fn expand( + &mut self, + source: &str, + rule_index: usize, + link_id: LinkId, + language: &str, + ) -> String { + let mut output = String::new(); + let mut index = 0usize; + + while index < source.len() { + let rest = &source[index..]; + + if let Some(stripped) = rest.strip_prefix("{{") { + let _ = stripped; + output.push('{'); + index += 2; + continue; + } + if rest.starts_with("}}") { + output.push('}'); + index += 2; + continue; + } + + if let Some((name, opener)) = parse_conditional(rest) { + let closer = format!("{{/{name}}}"); + let end = rest.find(&closer).unwrap_or(rest.len()); + let body = rest[opener.min(end)..end].to_string(); + if self.resolve(rule_index, link_id, name).is_some() { + let expanded = self.expand(&body, rule_index, link_id, language); + output.push_str(&expanded); + } + index += if end == rest.len() { + rest.len() + } else { + end + closer.len() + }; + continue; + } + + if let Some(placeholder) = parse_placeholder(rest) { + let target = self.resolve(rule_index, link_id, placeholder.name); + let Some(target) = target else { + // An unresolved capture renders as nothing rather than as + // its own source text, so optional children disappear. + index += placeholder.length; + continue; + }; + + let indent = current_indent(&output).to_string(); + let mode = placeholder.mode.map(str::to_string); + let value = if placeholder.variadic { + let separator = decode_separator(placeholder.separator); + let children = self.children_of(target); + let rendered: Vec = children + .into_iter() + .map(|child| { + let text = self.render_mode(child, mode.as_deref(), language); + indent_continuation(&text, &indent) + }) + .collect(); + rendered.join(&separator) + } else { + let text = self.render_mode(target, mode.as_deref(), language); + indent_continuation(&text, &indent) + }; + output.push_str(&value); + index += placeholder.length; + continue; + } + + let character = rest.chars().next().expect("rest is non-empty"); + output.push(character); + index += character.len_utf8(); + } + + output + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decodes_separator_escapes() { + assert_eq!(decode_separator(Some("\\n")), "\n"); + assert_eq!(decode_separator(Some(", ")), ", "); + assert_eq!(decode_separator(None), ""); + } + + #[test] + fn indents_continuation_lines_only() { + assert_eq!(indent_continuation("a\nb\n\nc", " "), "a\n b\n\n c"); + assert_eq!(indent_continuation("a", " "), "a"); + } + + #[test] + fn reports_the_current_indent() { + assert_eq!(current_indent("x\n "), " "); + assert_eq!(current_indent("x\n y"), ""); + } + + #[test] + fn parses_placeholder_forms() { + let placeholder = parse_placeholder("{*items:string|, }rest").expect("placeholder"); + assert!(placeholder.variadic); + assert_eq!(placeholder.name, "items"); + assert_eq!(placeholder.mode, Some("string")); + assert_eq!(placeholder.separator, Some(", ")); + + let plain = parse_placeholder("{.}").expect("placeholder"); + assert_eq!(plain.name, "."); + assert!(!plain.variadic); + + assert!(parse_placeholder("{?name}").is_none()); + assert_eq!( + parse_conditional("{?name}body").map(|it| it.0), + Some("name") + ); + } +} diff --git a/rust/src/fy/formalizer.rs b/rust/src/fy/formalizer.rs new file mode 100644 index 0000000..8c43e6d --- /dev/null +++ b/rust/src/fy/formalizer.rs @@ -0,0 +1,286 @@ +//! Formalizes a shell script as a meta-language links network. +//! +//! This is the first half of the translation pipeline the way +//! link-foundation/meta-language models it: source text becomes a network of +//! typed links (`LinkType::Syntax` nodes over `LinkType::Token` leaves). The +//! second half — [`super::rules`] plus [`super::engine`] — rewrites that +//! network into JavaScript purely by substitution rules. + +use std::collections::{HashSet, VecDeque}; + +use meta_language::language_profile::LanguageProfile; +use meta_language::link_network::{LinkId, LinkNetwork, LinkType}; + +use super::parser::{parse_shell_script, Node}; +use super::word_expander::expand_word; + +/// The language name used for every shell link in the network. +pub const SHELL_LANGUAGE: &str = "Shell"; + +/// Term of the nodes used to work around meta-language's fixed-arity insertion +/// API (see `js/docs/meta-language-gaps.md`). The engine flattens them, so they +/// are invisible to the rules. +pub const CHUNK_TERM: &str = "chunk"; + +/// The largest arity [`LinkNetwork::insert_syntax_node`] is instantiated for +/// here. `insert_syntax_node` is const-generic and there is no public +/// dynamic-arity insertion, so wider nodes are chunked. +const MAX_ARITY: usize = 8; + +/// Node terms that carry their own text (materialised as a leading token child). +const TEXT_BEARING: [&str; 18] = [ + "assignment", + "export", + "local", + "comment", + "name", + "literal", + "variable", + "env-variable", + "positional", + "script-name", + "all-arguments", + "argument-count", + "exit-status", + "process-id", + "unsupported-expansion", + "default-expansion", + "pattern", + "redirect", +]; + +/// The variable names a script binds, split by how they must be declared in +/// JavaScript. +#[derive(Debug, Default, Clone)] +pub struct DeclaredNames { + /// Every name that becomes a JavaScript variable, in first-seen order. + pub bound: Vec, + /// The subset already declared at its use site (`local`, `for` variables). + pub locals: HashSet, +} + +impl DeclaredNames { + fn bind(&mut self, name: &str) { + if !self.bound.iter().any(|bound| bound == name) { + self.bound.push(name.to_string()); + } + } + + /// The names of [`Self::bound`] that still need a hoisted declaration. + pub fn hoisted(&self) -> Vec<&str> { + self.bound + .iter() + .filter(|name| !self.locals.contains(*name)) + .map(String::as_str) + .collect() + } +} + +/// Collects the variable names the script binds. +/// +/// `export X=v` deliberately does not create a binding: it writes to the +/// environment, so `$X` must translate to `process.env.X`. +fn collect_declared_names(tree: &Node, names: &mut DeclaredNames) { + if let Some(text) = tree.text.as_deref() { + if tree.term == "assignment" || tree.term == "local" { + names.bind(text); + if tree.term == "local" { + names.locals.insert(text.to_string()); + } + } + } + if tree.term == "for" { + if let Some(text) = tree + .children + .first() + .and_then(|child| child.text.as_deref()) + { + names.bind(text); + names.locals.insert(text.to_string()); + } + } + for child in &tree.children { + collect_declared_names(child, names); + } +} + +/// Rewrites every `word` leaf into a `word` node of typed parts. +fn expand_words(tree: Node, declared: &HashSet) -> Node { + if tree.term == "word" { + let parse_substitution = |source: &str| { + let parsed = parse_shell_script(source); + expand_words(parsed, declared) + }; + let parts = expand_word( + tree.text.as_deref().unwrap_or_default(), + declared, + &parse_substitution, + ); + return Node::new("word", parts); + } + Node { + term: tree.term, + text: tree.text, + children: tree + .children + .into_iter() + .map(|child| expand_words(child, declared)) + .collect(), + } +} + +/// The capability profile of the shell dialect this translator formalizes. +/// +/// Declaring it in the network makes the supported surface queryable rather +/// than implicit, and `validate_network` then rejects anything outside it. +pub fn shell_profile(concepts: &[String]) -> LanguageProfile { + let mut profile = LanguageProfile::new("command-stream shell", SHELL_LANGUAGE) + .with_link_type(LinkType::Syntax) + .with_link_type(LinkType::Token) + .with_link_type(LinkType::Semantic); + for concept in concepts { + profile = profile.with_concept(concept.clone()); + } + profile +} + +/// Inserts a syntax node of any arity. +/// +/// `insert_syntax_node` takes a fixed-size array, so children beyond +/// [`MAX_ARITY`] are grouped into nested [`CHUNK_TERM`] nodes that the rule +/// engine flattens again. +fn insert_syntax_node(network: &mut LinkNetwork, term: &str, children: Vec) -> LinkId { + if children.len() > MAX_ARITY { + let mut queue: VecDeque = children.into(); + let mut grouped = Vec::new(); + while !queue.is_empty() { + let chunk: Vec = queue.drain(..queue.len().min(MAX_ARITY)).collect(); + grouped.push(insert_syntax_node(network, CHUNK_TERM, chunk)); + } + return insert_syntax_node(network, term, grouped); + } + + // Slots past `count` are never referenced by the inserted node. + let mut fixed = [LinkId::from_u64(0); MAX_ARITY]; + let count = children.len(); + fixed[..count].copy_from_slice(&children); + + match count { + 0 => network.insert_syntax_node(SHELL_LANGUAGE, term, []), + 1 => network.insert_syntax_node(SHELL_LANGUAGE, term, [fixed[0]]), + 2 => network.insert_syntax_node(SHELL_LANGUAGE, term, [fixed[0], fixed[1]]), + 3 => network.insert_syntax_node(SHELL_LANGUAGE, term, [fixed[0], fixed[1], fixed[2]]), + 4 => network.insert_syntax_node( + SHELL_LANGUAGE, + term, + [fixed[0], fixed[1], fixed[2], fixed[3]], + ), + 5 => network.insert_syntax_node( + SHELL_LANGUAGE, + term, + [fixed[0], fixed[1], fixed[2], fixed[3], fixed[4]], + ), + 6 => network.insert_syntax_node( + SHELL_LANGUAGE, + term, + [fixed[0], fixed[1], fixed[2], fixed[3], fixed[4], fixed[5]], + ), + 7 => network.insert_syntax_node( + SHELL_LANGUAGE, + term, + [ + fixed[0], fixed[1], fixed[2], fixed[3], fixed[4], fixed[5], fixed[6], + ], + ), + _ => network.insert_syntax_node(SHELL_LANGUAGE, term, fixed), + } +} + +/// A formalized shell script. +pub struct Formalization { + pub network: LinkNetwork, + /// The `script` root link. + pub root: LinkId, + /// Every node term used, so callers can build the right preamble. + pub terms: HashSet, + pub names: DeclaredNames, +} + +fn insert_tree(network: &mut LinkNetwork, node: &Node, terms: &mut HashSet) -> LinkId { + terms.insert(node.term.clone()); + let mut children = Vec::new(); + if TEXT_BEARING.contains(&node.term.as_str()) || node.text.is_some() { + children + .push(network.insert_source_token(SHELL_LANGUAGE, node.text.as_deref().unwrap_or(""))); + } + for child in &node.children { + children.push(insert_tree(network, child, terms)); + } + insert_syntax_node(network, &node.term, children) +} + +/// Formalizes shell source as a links network. +pub fn formalize_shell(source: &str) -> Formalization { + let parsed = parse_shell_script(source); + let mut names = DeclaredNames::default(); + collect_declared_names(&parsed, &mut names); + + let declared: HashSet = names.bound.iter().cloned().collect(); + let tree = expand_words(parsed, &declared); + + let mut network = LinkNetwork::new(); + let mut terms = HashSet::new(); + let root = insert_tree(&mut network, &tree, &mut terms); + + Formalization { + network, + root, + terms, + names, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn builds_a_typed_network() { + let formalization = formalize_shell("NAME=world\necho \"hi $NAME\"\n"); + assert!(formalization.terms.contains("assignment")); + assert!(formalization.terms.contains("variable")); + assert_eq!(formalization.names.bound, vec!["NAME".to_string()]); + + let root = formalization + .network + .link(formalization.root) + .expect("root link exists"); + assert_eq!(root.metadata().term(), Some("script")); + assert_eq!(root.metadata().link_type(), Some(LinkType::Syntax)); + } + + #[test] + fn chunks_nodes_wider_than_the_fixed_insertion_arity() { + let commands: String = (0..20) + .map(|index| format!("echo {index}\n")) + .collect::>() + .concat(); + let formalization = formalize_shell(&commands); + let root = formalization + .network + .link(formalization.root) + .expect("root link exists"); + // 20 statements do not fit in one fixed-arity node, so they are grouped. + assert!(root.references().len() <= MAX_ARITY); + assert!(formalization.terms.contains("command")); + } + + #[test] + fn the_profile_accepts_the_formalized_network() { + let formalization = formalize_shell("ls | wc -l\n"); + let concepts: Vec = formalization.terms.iter().cloned().collect(); + assert!(shell_profile(&concepts) + .validate_network(&formalization.network) + .is_ok()); + } +} diff --git a/rust/src/fy/lexer.rs b/rust/src/fy/lexer.rs new file mode 100644 index 0000000..6f9f4ff --- /dev/null +++ b/rust/src/fy/lexer.rs @@ -0,0 +1,288 @@ +//! Shell lexer for the `$fy` translator. +//! +//! Produces a flat, lossless token stream: every character of the input ends +//! up in exactly one token's text, so the formalizer can rebuild the original +//! source from the links network. + +/// Token kinds emitted by [`tokenize`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TokenType { + Word, + Operator, + Redirect, + Newline, + Comment, + Eof, +} + +/// One lexed token. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Token { + pub token_type: TokenType, + pub text: String, +} + +impl Token { + fn new(token_type: TokenType, text: impl Into) -> Self { + Token { + token_type, + text: text.into(), + } + } +} + +const OPERATORS: [&str; 7] = ["&&", "||", ";;", ";", "|", "(", ")"]; +const WORD_TERMINATORS: [char; 10] = [' ', '\t', '\n', ';', '|', '&', '(', ')', '<', '>']; + +/// Consumes a balanced span that starts at `start` (for example `$(`, `${`). +/// Returns the index just past the closing delimiter. +fn read_balanced(source: &[char], start: usize, open: char, close: char) -> usize { + let mut depth = 0usize; + let mut index = start; + while index < source.len() { + let character = source[index]; + if character == '\\' { + index += 2; + continue; + } + if character == open { + depth += 1; + } else if character == close { + depth -= 1; + if depth == 0 { + return index + 1; + } + } + index += 1; + } + source.len() +} + +/// Consumes a quoted span including both quote characters. +fn read_quoted(source: &[char], start: usize, quote: char) -> usize { + let mut index = start + 1; + while index < source.len() { + let character = source[index]; + if quote == '"' && character == '\\' { + index += 2; + continue; + } + if character == quote { + return index + 1; + } + index += 1; + } + source.len() +} + +/// Reads one shell word starting at `start`, treating quotes, `$(...)`, +/// `${...}` and backticks as atomic spans. +fn read_word(source: &[char], start: usize) -> (String, usize) { + let mut index = start; + while index < source.len() { + let character = source[index]; + if character == '\\' { + index += 2; + continue; + } + if character == '\'' || character == '"' { + index = read_quoted(source, index, character); + continue; + } + if character == '`' { + index = read_quoted(source, index, '`'); + continue; + } + if character == '$' && source.get(index + 1) == Some(&'(') { + index = read_balanced(source, index + 1, '(', ')'); + continue; + } + if character == '$' && source.get(index + 1) == Some(&'{') { + index = read_balanced(source, index + 1, '{', '}'); + continue; + } + if WORD_TERMINATORS.contains(&character) { + break; + } + index += 1; + } + let end = index.min(source.len()); + (source[start..end].iter().collect(), end) +} + +/// A redirection operator may be prefixed by a file descriptor (`2>`). Only a +/// leading run of digits counts as part of the operator, never as a word. +fn match_redirect(source: &[char], index: usize) -> Option { + let mut cursor = index; + while source.get(cursor).is_some_and(char::is_ascii_digit) { + cursor += 1; + } + for operator in [">>", ">&", "<<<", "<<", ">", "<"] { + if starts_with(source, cursor, operator) { + return Some( + source[index..cursor + operator.chars().count()] + .iter() + .collect(), + ); + } + } + None +} + +fn starts_with(source: &[char], index: usize, text: &str) -> bool { + text.chars() + .enumerate() + .all(|(offset, expected)| source.get(index + offset) == Some(&expected)) +} + +fn match_operator(source: &[char], index: usize) -> Option<&'static str> { + OPERATORS + .into_iter() + .find(|operator| starts_with(source, index, operator)) +} + +/// Whitespace, line splices, newlines and comments. +/// +/// Returns the token to emit (`None` emits nothing) and the index to continue +/// from, or `None` when the character starts a word or operator instead. +fn match_trivia(source: &[char], index: usize) -> Option<(Option, usize)> { + let character = source[index]; + + if character == '\n' { + return Some((Some(Token::new(TokenType::Newline, "\n")), index + 1)); + } + if character == ' ' || character == '\t' || character == '\r' { + return Some((None, index + 1)); + } + // A backslash-newline pair splices two physical lines into one. + if character == '\\' && source.get(index + 1) == Some(&'\n') { + return Some((None, index + 2)); + } + // `#` only opens a comment at the start of a word. + let after_separator = index == 0 + || source[index - 1].is_whitespace() + || [';', '|', '&', '(', ')'].contains(&source[index - 1]); + if character == '#' && after_separator { + let end = source[index..] + .iter() + .position(|&c| c == '\n') + .map_or(source.len(), |offset| index + offset); + let text: String = source[index..end].iter().collect(); + return Some((Some(Token::new(TokenType::Comment, text)), end)); + } + None +} + +/// Tokenizes a complete shell script. The stream always ends with an EOF token. +pub fn tokenize(source: &str) -> Vec { + let source: Vec = source.chars().collect(); + let mut tokens = Vec::new(); + let mut index = 0usize; + + while index < source.len() { + if let Some((token, next)) = match_trivia(&source, index) { + if let Some(token) = token { + tokens.push(token); + } + index = next; + continue; + } + + if let Some(redirect) = match_redirect(&source, index) { + index += redirect.chars().count(); + tokens.push(Token::new(TokenType::Redirect, redirect)); + continue; + } + // `&` on its own backgrounds a command; `&&` was matched by the operator list. + let operator = match_operator(&source, index).or({ + if source[index] == '&' { + Some("&") + } else { + None + } + }); + if let Some(operator) = operator { + tokens.push(Token::new(TokenType::Operator, operator)); + index += operator.chars().count(); + continue; + } + + let (text, next) = read_word(&source, index); + if text.is_empty() { + index += 1; + continue; + } + tokens.push(Token::new(TokenType::Word, text)); + index = next; + } + + tokens.push(Token::new(TokenType::Eof, "")); + tokens +} + +#[cfg(test)] +mod tests { + use super::*; + + fn kinds(source: &str) -> Vec<(TokenType, String)> { + tokenize(source) + .into_iter() + .map(|token| (token.token_type, token.text)) + .collect() + } + + #[test] + fn tokenizes_words_and_operators() { + assert_eq!( + kinds("ls -la | grep x"), + vec![ + (TokenType::Word, "ls".to_string()), + (TokenType::Word, "-la".to_string()), + (TokenType::Operator, "|".to_string()), + (TokenType::Word, "grep".to_string()), + (TokenType::Word, "x".to_string()), + (TokenType::Eof, String::new()), + ] + ); + } + + #[test] + fn keeps_substitutions_and_quotes_in_one_word() { + assert_eq!( + kinds("echo \"$(ls | wc -l) items\""), + vec![ + (TokenType::Word, "echo".to_string()), + (TokenType::Word, "\"$(ls | wc -l) items\"".to_string()), + (TokenType::Eof, String::new()), + ] + ); + } + + #[test] + fn reads_a_file_descriptor_prefixed_redirect() { + assert_eq!( + kinds("ls >out.txt 2>&1"), + vec![ + (TokenType::Word, "ls".to_string()), + (TokenType::Redirect, ">".to_string()), + (TokenType::Word, "out.txt".to_string()), + (TokenType::Redirect, "2>&".to_string()), + (TokenType::Word, "1".to_string()), + (TokenType::Eof, String::new()), + ] + ); + } + + #[test] + fn emits_comments_and_newlines() { + assert_eq!( + kinds("# note\nls"), + vec![ + (TokenType::Comment, "# note".to_string()), + (TokenType::Newline, "\n".to_string()), + (TokenType::Word, "ls".to_string()), + (TokenType::Eof, String::new()), + ] + ); + } +} diff --git a/rust/src/fy/mod.rs b/rust/src/fy/mod.rs new file mode 100644 index 0000000..1e9ca38 --- /dev/null +++ b/rust/src/fy/mod.rs @@ -0,0 +1,241 @@ +//! `$fy`: rule-based translation of shell scripts into command-stream modules. +//! +//! The pipeline has two halves, exactly as link-foundation/meta-language models +//! translation: +//! +//! 1. Formalize: shell source -> links network ([`formalizer`]) +//! 2. Substitute: links network -> JavaScript ([`rules`] + [`engine`]) +//! +//! Step 2 contains no shell knowledge beyond the rule table, and step 1 +//! contains no JavaScript knowledge at all. + +pub mod engine; +pub mod formalizer; +pub mod lexer; +pub mod parser; +pub mod rules; +pub mod word_expander; + +use std::collections::HashSet; + +use engine::RuleEngine; +use formalizer::{formalize_shell, DeclaredNames}; +use rules::{build_rule_set, language_fallbacks, modes, TARGET}; + +/// Terms whose presence requires `const args = process.argv.slice(2)`. +const ARGUMENT_TERMS: [&str; 3] = ["positional", "all-arguments", "argument-count"]; + +/// Options for [`translate_shell_to_mjs`]. +#[derive(Debug, Clone)] +pub struct TranslateOptions { + /// Emit a `#!/usr/bin/env node` line. + pub shebang: bool, + /// Module to import `$` from. + pub module_name: String, +} + +impl Default for TranslateOptions { + fn default() -> Self { + TranslateOptions { + shebang: true, + module_name: "command-stream".to_string(), + } + } +} + +/// A translated module. +#[derive(Debug, Clone)] +pub struct Translation { + /// `preamble` followed by `body`. + pub code: String, + pub preamble: String, + pub body: String, + /// Constructs the rule set could not translate. + pub diagnostics: Vec, +} + +/// Builds the module header the translated statements need. +fn build_preamble( + terms: &HashSet, + names: &DeclaredNames, + options: &TranslateOptions, +) -> String { + let mut imports = vec!["$"]; + if terms.contains("set-option") { + imports.push("shell"); + } + + let mut lines: Vec = Vec::new(); + if options.shebang { + lines.push("#!/usr/bin/env node".to_string()); + } + lines.push(format!( + "import {{ {} }} from '{}';", + imports.join(", "), + options.module_name + )); + lines.push(String::new()); + + if ARGUMENT_TERMS.iter().any(|term| terms.contains(*term)) { + lines.push("const args = process.argv.slice(2);".to_string()); + } + if terms.contains("exit-status") { + lines.push("let exitCode = 0;".to_string()); + } + // Shell variables have no block scope, so every non-`local` binding is + // hoisted to the top of the module rather than declared at its first + // assignment (which would break on reassignment inside a loop or branch). + let hoisted = names.hoisted(); + if !hoisted.is_empty() { + lines.push(format!("let {};", hoisted.join(", "))); + } + if lines.last().is_some_and(|line| !line.is_empty()) { + lines.push(String::new()); + } + // The trailing empty entry plus this newline separate the preamble from the + // first translated statement by exactly one blank line. + format!("{}\n", lines.join("\n")) +} + +/// Collapses runs of three or more newlines into a blank line. +fn collapse_blank_lines(text: &str) -> String { + let mut output = String::with_capacity(text.len()); + let mut newlines = 0usize; + for character in text.chars() { + if character == '\n' { + newlines += 1; + if newlines > 2 { + continue; + } + } else { + newlines = 0; + } + output.push(character); + } + output +} + +/// Translates a shell script into a command-stream ES module. +pub fn translate_shell_to_mjs(source: &str, options: &TranslateOptions) -> Translation { + // A shell script checked out with CRLF endings is still a shell script: + // `\r` is not a token, so it is dropped before anything else looks at the + // text. + let normalized = source.replace("\r\n", "\n").replace('\r', "\n"); + let source = normalized.as_str(); + // A `#!/bin/sh` line selects the interpreter for the *shell* script; the + // translated module gets its own shebang from `build_preamble`. + let script = match source.strip_prefix("#!") { + Some(rest) => rest.split_once('\n').map_or("", |(_, rest)| rest), + None => source, + }; + + let formalization = formalize_shell(script); + let rule_set = build_rule_set(formalization.terms.contains("exit-status")); + let mut engine = RuleEngine::new( + &formalization.network, + &rule_set, + TARGET, + language_fallbacks(), + modes(), + ); + + let rendered = engine.render_root(formalization.root); + let body = format!("{}\n", collapse_blank_lines(&rendered).trim_end()); + let preamble = build_preamble(&formalization.terms, &formalization.names, options); + + Translation { + code: format!("{preamble}{body}"), + preamble, + body, + diagnostics: engine.report(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn translate(source: &str) -> Translation { + translate_shell_to_mjs(source, &TranslateOptions::default()) + } + + #[test] + fn translates_a_command_into_an_awaited_template_literal() { + let translation = translate("ls -la\n"); + assert_eq!(translation.body, "await $`ls -la`;\n"); + assert!(translation.diagnostics.is_empty()); + } + + #[test] + fn emits_a_preamble_with_the_import_and_hoisted_bindings() { + let translation = translate("NAME=world\necho $NAME\n"); + assert!(translation + .preamble + .starts_with("#!/usr/bin/env node\nimport { $ } from 'command-stream';")); + assert!(translation.preamble.contains("let NAME;")); + assert!(translation.body.contains("NAME = `world`;")); + assert!(translation.body.contains("await $`echo ${NAME}`;")); + } + + #[test] + fn translates_control_flow_into_javascript_control_flow() { + let translation = translate("if [ -f x ]; then\n echo yes\nelse\n echo no\nfi\n"); + assert!(translation + .body + .contains("if ((await $`[ -f x ]`).code === 0) {")); + assert!(translation.body.contains("} else {")); + } + + #[test] + fn translates_a_for_loop_and_a_pipeline() { + let translation = translate("for i in a b; do\n echo $i | tr a-z A-Z\ndone\n"); + assert!(translation.body.contains("for (const i of [`a`, `b`]) {")); + assert!(translation + .body + .contains("await $`echo ${i} | tr a-z A-Z`;")); + } + + #[test] + fn tracks_the_exit_code_only_when_the_script_reads_it() { + let plain = translate("false\n"); + assert!(!plain.body.contains("exitCode")); + + let tracked = translate("false\necho $?\n"); + assert!(tracked.preamble.contains("let exitCode = 0;")); + assert!(tracked.body.contains("exitCode = (await $`false`).code;")); + } + + #[test] + fn maps_expansions_onto_javascript_expressions() { + let translation = translate("echo \"$1 $# ${HOME}\"\n"); + assert!(translation.body.contains("${args[0]}")); + assert!(translation.body.contains("${args.length}")); + assert!(translation.body.contains("${process.env.HOME}")); + } + + #[test] + fn keeps_comments_and_drops_the_source_shebang() { + let translation = translate("#!/bin/sh\n# note\nls\n"); + assert_eq!(translation.body, "// note\nawait $`ls`;\n"); + } + + #[test] + fn translates_a_script_with_crlf_line_endings() { + let translation = translate("ls\r\necho hi\r\n"); + assert_eq!(translation.body, "await $`ls`;\nawait $`echo hi`;\n"); + assert!(translation.diagnostics.is_empty()); + } + + #[test] + fn honours_the_options() { + let translation = translate_shell_to_mjs( + "ls\n", + &TranslateOptions { + shebang: false, + module_name: "../src/index.mjs".to_string(), + }, + ); + assert!(!translation.preamble.contains("#!")); + assert!(translation.preamble.contains("from '../src/index.mjs';")); + } +} diff --git a/rust/src/fy/parser.rs b/rust/src/fy/parser.rs new file mode 100644 index 0000000..435a703 --- /dev/null +++ b/rust/src/fy/parser.rs @@ -0,0 +1,641 @@ +//! Recursive-descent shell parser for the `$fy` translator. +//! +//! Produces a plain node tree which [`crate::fy::formalizer`] then +//! materialises as a meta-language links network. Keeping the two steps apart +//! means the grammar stays readable and the network construction stays +//! mechanical. +//! +//! This parser is deliberately separate from [`crate::shell_parser`], which +//! parses a *single* command line for the runtime. `$fy` needs whole-script +//! structure: comments, blank lines, control flow and function definitions. + +use std::collections::HashSet; + +use super::lexer::{tokenize, Token, TokenType}; + +/// A parsed shell construct. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Node { + /// Construct name, for example `command`, `pipeline` or `if`. + pub term: String, + /// The node's own text, when it carries one (a variable name, a comment). + pub text: Option, + pub children: Vec, +} + +impl Node { + pub fn new(term: &str, children: Vec) -> Self { + Node { + term: term.to_string(), + text: None, + children, + } + } + + pub fn with_text(term: &str, children: Vec, text: impl Into) -> Self { + Node { + term: term.to_string(), + text: Some(text.into()), + children, + } + } + + pub fn leaf(term: &str, text: impl Into) -> Self { + Node::with_text(term, Vec::new(), text) + } +} + +/// Reserved words that terminate a command list. +const BLOCK_TERMINATORS: [&str; 8] = ["then", "elif", "else", "fi", "do", "done", "esac", "}"]; + +/// Splits `NAME=value` into its two halves. +fn match_assignment(text: &str) -> Option<(String, String)> { + let equals = text.find('=')?; + let (name, value) = text.split_at(equals); + if !is_identifier(name) { + return None; + } + Some((name.to_string(), value[1..].to_string())) +} + +fn is_identifier(text: &str) -> bool { + let mut characters = text.chars(); + matches!(characters.next(), Some(first) if first.is_ascii_alphabetic() || first == '_') + && characters.all(|character| character.is_ascii_alphanumeric() || character == '_') +} + +struct ShellParser { + tokens: Vec, + position: usize, +} + +/// The words and redirects making up one simple command. +struct CommandParts { + assignments: Vec, + words: Vec, + redirects: Vec, +} + +impl ShellParser { + fn new(source: &str) -> Self { + ShellParser { + tokens: tokenize(source), + position: 0, + } + } + + fn current(&self) -> &Token { + &self.tokens[self.position] + } + + fn at(&self, token_type: TokenType) -> bool { + self.current().token_type == token_type + } + + fn at_text(&self, token_type: TokenType, text: &str) -> bool { + self.at(token_type) && self.current().text == text + } + + fn at_word(&self, text: &str) -> bool { + self.at_text(TokenType::Word, text) + } + + fn consume(&mut self) -> Token { + let token = self.tokens[self.position].clone(); + self.position += 1; + token + } + + fn expect_word(&mut self, text: &str) -> bool { + if self.at_word(text) { + self.consume(); + true + } else { + false + } + } + + /// Skips newlines and `;` separators, emitting `blank` nodes for blank lines. + fn skip_separators(&mut self, collector: Option<&mut Vec>) { + let mut newlines = 0usize; + let mut collector = collector; + loop { + if self.at(TokenType::Newline) { + self.consume(); + newlines += 1; + // The first newline ends the previous statement; each further + // one is a blank line worth preserving in the output. + if newlines > 1 { + if let Some(statements) = collector.as_deref_mut() { + statements.push(Node::new("blank", Vec::new())); + } + } + continue; + } + if self.at_text(TokenType::Operator, ";") { + self.consume(); + continue; + } + return; + } + } + + /// Whether the current token closes the enclosing block. + fn at_block_end(&self, extra_terminators: &HashSet) -> bool { + let token = self.current(); + if token.token_type == TokenType::Eof { + return true; + } + if token.token_type == TokenType::Operator && (token.text == ")" || token.text == ";;") { + return true; + } + if token.token_type != TokenType::Word { + return false; + } + BLOCK_TERMINATORS.contains(&token.text.as_str()) + || extra_terminators.contains(token.text.as_str()) + } + + fn parse_script(&mut self) -> Node { + let statements = self.parse_statements(&HashSet::new()); + Node::new("script", statements) + } + + fn parse_statements(&mut self, extra_terminators: &HashSet) -> Vec { + let mut statements = Vec::new(); + // Leading blank lines are not "blank lines between statements". + self.skip_separators(None); + + while !self.at_block_end(extra_terminators) { + if self.at(TokenType::Comment) { + let text = self.consume().text; + let body = text.trim_start_matches('#'); + let body = body.strip_prefix(' ').unwrap_or(body); + statements.push(Node::leaf("comment", body)); + } else { + match self.parse_list() { + Some(statement) => statements.push(statement), + None => break, + } + } + self.skip_separators(Some(&mut statements)); + } + + // Trailing blank lines add nothing to the translation. + while statements.last().is_some_and(|last| last.term == "blank") { + statements.pop(); + } + statements + } + + /// `pipeline (&& | ||) pipeline ...`, left associative. + fn parse_list(&mut self) -> Option { + let mut left = self.parse_pipeline()?; + while self.at_text(TokenType::Operator, "&&") || self.at_text(TokenType::Operator, "||") { + let operator = if self.consume().text == "&&" { + "and" + } else { + "or" + }; + // An operator may be followed by a newline before its right operand. + while self.at(TokenType::Newline) { + self.consume(); + } + match self.parse_pipeline() { + Some(right) => left = Node::new(operator, vec![left, right]), + None => break, + } + } + Some(left) + } + + fn parse_pipeline(&mut self) -> Option { + let mut stages = vec![self.parse_command()?]; + while self.at_text(TokenType::Operator, "|") { + self.consume(); + while self.at(TokenType::Newline) { + self.consume(); + } + match self.parse_command() { + Some(stage) => stages.push(stage), + None => break, + } + } + if stages.len() == 1 { + stages.pop() + } else { + Some(Node::new("pipeline", stages)) + } + } + + fn parse_command(&mut self) -> Option { + if self.at_text(TokenType::Operator, "(") { + self.consume(); + let body = self.parse_statements(&HashSet::new()); + if self.at_text(TokenType::Operator, ")") { + self.consume(); + } + return Some(Node::new("subshell", vec![Node::new("block", body)])); + } + if self.at_word("{") { + self.consume(); + let body = self.parse_statements(&HashSet::new()); + self.expect_word("}"); + return Some(Node::new("block", body)); + } + if self.at_word("if") { + return Some(self.parse_if()); + } + if self.at_word("while") || self.at_word("until") { + return Some(self.parse_loop()); + } + if self.at_word("for") { + return Some(self.parse_for()); + } + if self.at_word("case") { + return Some(self.parse_case()); + } + if let Some(function) = self.try_parse_function() { + return Some(function); + } + self.parse_simple_command() + } + + /// `if list; then block [elif ...] [else block] fi` + fn parse_if(&mut self) -> Node { + self.consume(); // `if` / `elif` + let condition = self.parse_condition("then"); + let consequent = Node::new("block", self.parse_statements(&HashSet::new())); + + if self.at_word("elif") { + let alternative = self.parse_if(); + return Node::new("if", vec![condition, consequent, alternative]); + } + let alternative = if self.expect_word("else") { + Some(Node::new("block", self.parse_statements(&HashSet::new()))) + } else { + None + }; + self.expect_word("fi"); + match alternative { + Some(alternative) => Node::new("if", vec![condition, consequent, alternative]), + None => Node::new("if", vec![condition, consequent]), + } + } + + fn parse_loop(&mut self) -> Node { + let term = if self.consume().text == "while" { + "while" + } else { + "until" + }; + let condition = self.parse_condition("do"); + let body = Node::new("block", self.parse_statements(&HashSet::new())); + self.expect_word("done"); + Node::new(term, vec![condition, body]) + } + + /// `for NAME in word...; do block done` + fn parse_for(&mut self) -> Node { + self.consume(); // `for` + let variable = if self.at(TokenType::Word) { + self.consume().text + } else { + "_".to_string() + }; + let mut items = Vec::new(); + if self.expect_word("in") { + while self.at(TokenType::Word) + && !BLOCK_TERMINATORS.contains(&self.current().text.as_str()) + { + let text = self.consume().text; + items.push(Node::leaf("word", text)); + } + } + self.skip_separators(None); + self.expect_word("do"); + let body = Node::new("block", self.parse_statements(&HashSet::new())); + self.expect_word("done"); + Node::new( + "for", + vec![ + Node::leaf("name", variable), + Node::new("word-list", items), + body, + ], + ) + } + + /// `case word in pattern) block ;; ... esac` + fn parse_case(&mut self) -> Node { + self.consume(); // `case` + let subject = if self.at(TokenType::Word) { + let text = self.consume().text; + Node::leaf("word", text) + } else { + Node::leaf("word", "") + }; + self.expect_word("in"); + self.skip_separators(None); + + let mut branches = Vec::new(); + while !self.at_word("esac") && !self.at(TokenType::Eof) { + branches.push(self.parse_case_branch()); + } + self.expect_word("esac"); + Node::new( + "case", + vec![subject, Node::new("case-branch-list", branches)], + ) + } + + fn parse_case_branch(&mut self) -> Node { + // A pattern list is `a|b|c)`. The lexer already split on `|` and `)`. + let mut patterns = Vec::new(); + while self.at(TokenType::Word) { + // `*` is the shell's catch-all pattern, i.e. `default:`. + let pattern = self.consume().text; + let term = if pattern == "*" { + "pattern-default" + } else { + "pattern" + }; + patterns.push(Node::leaf(term, pattern)); + if self.at_text(TokenType::Operator, "|") { + self.consume(); + continue; + } + break; + } + if self.at_text(TokenType::Operator, ")") { + self.consume(); + } + let body = Node::new("block", self.parse_statements(&HashSet::new())); + if self.at_text(TokenType::Operator, ";;") { + self.consume(); + } + self.skip_separators(None); + Node::new( + "case-branch", + vec![Node::new("pattern-list", patterns), body], + ) + } + + /// Consumes `function name [()]`, or `None` when that is not next. + fn parse_keyword_function_name(&mut self) -> Option { + self.consume(); + if !self.at(TokenType::Word) { + return None; + } + let name = self.consume().text; + if self.at_text(TokenType::Operator, "(") { + self.consume(); + if self.at_text(TokenType::Operator, ")") { + self.consume(); + } + } + Some(name) + } + + /// Consumes `name ()`, or `None` when that is not next. + fn parse_bare_function_name(&mut self) -> Option { + let is_operator = |token: Option<&Token>, text: &str| { + token.is_some_and(|token| token.token_type == TokenType::Operator && token.text == text) + }; + + if !self.at(TokenType::Word) + || !is_identifier(&self.current().text) + || !is_operator(self.tokens.get(self.position + 1), "(") + || !is_operator(self.tokens.get(self.position + 2), ")") + { + return None; + } + let name = self.consume().text; + self.consume(); + self.consume(); + Some(name) + } + + /// `name() { ... }` or `function name { ... }` + fn try_parse_function(&mut self) -> Option { + let start = self.position; + let name = if self.at_word("function") { + self.parse_keyword_function_name() + } else { + self.parse_bare_function_name() + }; + + let Some(name) = name else { + self.position = start; + return None; + }; + + self.skip_separators(None); + if !self.expect_word("{") { + self.position = start; + return None; + } + let body = Node::new("block", self.parse_statements(&HashSet::new())); + self.expect_word("}"); + Some(Node::new("function", vec![Node::leaf("name", name), body])) + } + + /// Parses the condition of `if`/`while`/`until` up to its introducer + /// keyword (`then`/`do`), which the caller has named. + fn parse_condition(&mut self, introducer: &str) -> Node { + let terminators: HashSet = HashSet::from([introducer.to_string()]); + let mut condition = self.parse_statements(&terminators); + self.skip_separators(None); + self.expect_word(introducer); + if condition.len() == 1 { + return condition.remove(0); + } + Node::new("block", condition) + } + + /// True once the current token can no longer belong to a simple command. + /// + /// `words` are the words read so far; an empty list means the parser is + /// still in command position. + fn at_command_end(&self, words: &[Node]) -> bool { + // Reserved words are only reserved in command position: `echo done` + // runs `echo` with the argument `done`, it does not close a loop. + // Assignment prefixes do not consume the command position, so the + // check stays on `words`. + if words.is_empty() && self.at_block_end(&HashSet::new()) { + return true; + } + self.at(TokenType::Eof) + || self.at(TokenType::Newline) + || self.at(TokenType::Comment) + || self.at(TokenType::Operator) + } + + /// Reads the assignment prefixes, words and redirects of one command. + fn parse_command_parts(&mut self) -> CommandParts { + let mut parts = CommandParts { + assignments: Vec::new(), + words: Vec::new(), + redirects: Vec::new(), + }; + + while !self.at_command_end(&parts.words) { + if self.at(TokenType::Redirect) { + let operator = self.consume().text; + let target = if self.at(TokenType::Word) { + self.consume().text + } else { + String::new() + }; + parts.redirects.push(Node::with_text( + "redirect", + vec![Node::leaf("word", target)], + operator, + )); + continue; + } + + let token = self.consume(); + let assignment = if parts.words.is_empty() { + match_assignment(&token.text) + } else { + None + }; + if let Some((name, value)) = assignment { + parts.assignments.push(Node::with_text( + "assignment", + vec![Node::leaf("word", value)], + name, + )); + continue; + } + parts.words.push(Node::leaf("word", token.text)); + } + + parts + } + + fn parse_simple_command(&mut self) -> Option { + let CommandParts { + mut assignments, + words, + redirects, + } = self.parse_command_parts(); + + if words.is_empty() { + if assignments.is_empty() { + return None; + } + // Bare `NAME=value` statements are variable declarations. + return Some(if assignments.len() == 1 { + assignments.remove(0) + } else { + Node::new("block", assignments) + }); + } + + let name = words[0].text.clone().unwrap_or_default(); + let rest: Vec = words[1..].to_vec(); + + match name.as_str() { + "export" | "local" | "readonly" => return Some(build_declaration(&name, rest)), + "set" if !rest.is_empty() => return Some(Node::new("set-option", rest)), + "exit" => return Some(Node::new("exit", rest)), + "return" => return Some(Node::new("return", rest)), + "source" | "." => return Some(Node::new("source", rest)), + _ => {} + } + + let mut children = vec![ + Node::new("word-list", words), + Node::new("redirect-list", redirects), + ]; + if !assignments.is_empty() { + // Prefixed assignments (`FOO=bar cmd`) scope an environment + // variable to one command; keep them attached so a rule can + // render them. + children.push(Node::new("assignment-list", assignments)); + } + Some(Node::new("command", children)) + } +} + +fn build_declaration(keyword: &str, words: Vec) -> Node { + let term = if keyword == "export" { + "export" + } else { + "local" + }; + let mut declarations: Vec = words + .into_iter() + .map(|word| { + let text = word.text.unwrap_or_default(); + match match_assignment(&text) { + Some((name, value)) => Node::with_text(term, vec![Node::leaf("word", value)], name), + None => Node::with_text(term, vec![Node::leaf("word", "")], text), + } + }) + .collect(); + + if declarations.len() == 1 { + declarations.remove(0) + } else { + Node::new("block", declarations) + } +} + +/// Parses a shell script into a plain node tree rooted at a `script` node. +pub fn parse_shell_script(source: &str) -> Node { + ShellParser::new(source).parse_script() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn terms(node: &Node) -> Vec { + node.children + .iter() + .map(|child| child.term.clone()) + .collect() + } + + #[test] + fn parses_a_pipeline_into_stages() { + let script = parse_shell_script("ls | grep test | wc -l"); + assert_eq!(script.term, "script"); + assert_eq!(script.children[0].term, "pipeline"); + assert_eq!(script.children[0].children.len(), 3); + } + + #[test] + fn parses_and_or_left_associatively() { + let script = parse_shell_script("a && b || c"); + assert_eq!(script.children[0].term, "or"); + assert_eq!(script.children[0].children[0].term, "and"); + } + + #[test] + fn parses_control_flow_functions_and_case() { + let script = parse_shell_script( + "if [ -f x ]; then echo a; else echo b; fi\n\ + while true; do echo loop; done\n\ + for i in 1 2; do echo $i; done\n\ + greet() { echo hi; }\n\ + case \"$1\" in start) run ;; *) usage ;; esac\n", + ); + assert_eq!( + terms(&script), + vec!["if", "while", "for", "function", "case"] + ); + } + + #[test] + fn reserved_words_stay_arguments_outside_command_position() { + let script = parse_shell_script("echo done | tr a-z A-Z"); + let pipeline = &script.children[0]; + assert_eq!(pipeline.term, "pipeline"); + let words = &pipeline.children[0].children[0]; + assert_eq!(words.children.len(), 2); + } +} diff --git a/rust/src/fy/rules.rs b/rust/src/fy/rules.rs new file mode 100644 index 0000000..6fc2772 --- /dev/null +++ b/rust/src/fy/rules.rs @@ -0,0 +1,465 @@ +//! The Shell -> JavaScript substitution rules. +//! +//! Every construct the formalizer can produce has exactly one +//! `TranslationRule` here: a `LinkQuery` selecting the links it owns, plus one +//! template per target language. Nothing about the translation lives outside +//! this file — [`super::engine`] only walks the network and substitutes. +//! +//! Four target languages express the four contexts a node can appear in: +//! +//! | language | context | example | +//! |-------------------------|---------------------|------------------------| +//! | `JavaScript` | a statement | ``await $`ls`;`` | +//! | `JavaScript:command` | text inside ``$`…```| `ls` | +//! | `JavaScript:value` | text inside `` `…` `` | `ls` without quoting | +//! | `JavaScript:expression` | a JavaScript value | `process.env.HOME` | +//! +//! Placeholder syntax is documented in [`super::engine`]. + +use std::collections::HashMap; + +use meta_language::link_network::LinkType; +use meta_language::query::LinkQuery; +use meta_language::translation_rules::{TranslationRule, TranslationRuleSet}; + +use super::engine::Mode; +use super::formalizer::SHELL_LANGUAGE; + +pub const TARGET: &str = "JavaScript"; +pub const COMMAND: &str = "JavaScript:command"; +pub const VALUE: &str = "JavaScript:value"; +pub const EXPRESSION: &str = "JavaScript:expression"; + +/// A `:value` rendering falls back to the `:command` one when unquoting is a no-op. +pub fn language_fallbacks() -> HashMap> { + HashMap::from([(VALUE.to_string(), vec![COMMAND.to_string()])]) +} + +/// `set -e` and friends map onto `shell.set('e')`. +const SET_FLAGS: [&str; 3] = ["e", "v", "x"]; + +/// Escapes text so it survives inside a JavaScript template literal. +fn escape_template(text: &str) -> String { + let mut escaped = String::with_capacity(text.len()); + let mut characters = text.chars().peekable(); + while let Some(character) = characters.next() { + match character { + '`' | '\\' => { + escaped.push('\\'); + escaped.push(character); + } + '$' if characters.peek() == Some(&'{') => { + escaped.push_str("\\$"); + } + _ => escaped.push(character), + } + } + escaped +} + +/// Removes the shell quoting from a literal. +/// +/// Quotes are meaningful to the shell (word splitting, globbing) so they are +/// kept in `:command` context, but a JavaScript value must not contain them. +fn strip_shell_quotes(text: &str) -> String { + let characters: Vec = text.chars().collect(); + let mut output = String::new(); + let mut quote: Option = None; + let mut index = 0usize; + + while index < characters.len() { + let character = characters[index]; + if character == '\\' && quote != Some('\'') { + if let Some(&next) = characters.get(index + 1) { + output.push(next); + } + index += 2; + continue; + } + if quote.is_none() && (character == '"' || character == '\'') { + quote = Some(character); + index += 1; + continue; + } + if Some(character) == quote { + quote = None; + index += 1; + continue; + } + output.push(character); + index += 1; + } + output +} + +fn quote_json(text: &str) -> String { + let escaped = text.replace('\\', "\\\\").replace('"', "\\\""); + format!("\"{escaped}\"") +} + +fn as_string(text: &str) -> String { + format!("`{text}`") +} + +fn as_unquoted(text: &str) -> String { + escape_template(&strip_shell_quotes(text)) +} + +fn as_flag(text: &str) -> String { + let letter = text.trim_start_matches('-'); + if SET_FLAGS.contains(&letter) { + quote_json(letter) + } else { + quote_json(text) + } +} + +/// Placeholder modes available to the templates below. +pub fn modes() -> HashMap { + let mode = |language: Option<&str>, transform: Option String>| Mode { + language: language.map(str::to_string), + transform, + }; + HashMap::from([ + // Raw shell source text of the captured link. + ("text".to_string(), mode(None, None)), + // The captured link rendered as shell command text. + ("command".to_string(), mode(Some(COMMAND), None)), + // The captured link rendered as an unquoted value. + ("value".to_string(), mode(Some(VALUE), None)), + // The captured link rendered as a JavaScript expression. + ("expr".to_string(), mode(Some(EXPRESSION), None)), + // The captured link as a JavaScript string (a template literal). + ("string".to_string(), mode(Some(VALUE), Some(as_string))), + // Literal source text, escaped for a template literal. + ("literal".to_string(), mode(None, Some(escape_template))), + // Literal source text with shell quoting removed. + ("unquoted".to_string(), mode(None, Some(as_unquoted))), + // A `set` operand such as `-e`, as a `shell.set` argument. + ("flag".to_string(), mode(None, Some(as_flag))), + ]) +} + +fn syntax(term: &str) -> LinkQuery { + LinkQuery::new() + .with_link_type(LinkType::Syntax) + .with_language(SHELL_LANGUAGE) + .with_term(term) +} + +/// Declares one rule: the node term it owns, its capture name -> reference +/// index bindings, and one template per target language. +fn rule(term: &str, captures: &[(&str, usize)], templates: &[(&str, &str)]) -> TranslationRule { + let mut created = TranslationRule::new(term, syntax(term)); + for (name, index) in captures { + created = created.with_reference_capture(*name, *index); + } + for (language, text) in templates { + created = created.with_template(*language, *text); + } + created +} + +/// A word expansion: one JavaScript expression, interpolated in command text. +fn expansion(term: &str, expression: &str) -> TranslationRule { + // `{{`/`}}` are literal braces, so this emits `${}`. + let interpolated = format!("${{{{{expression}}}}}"); + rule( + term, + &[("name", 0)], + &[ + (COMMAND, interpolated.as_str()), + (VALUE, interpolated.as_str()), + (EXPRESSION, expression), + ], + ) +} + +/// Builds the Shell -> JavaScript rule set. +/// +/// `track_exit_code` emits `exitCode = ...` for each command, which is only +/// needed when the script reads `$?`. +pub fn build_rule_set(track_exit_code: bool) -> TranslationRuleSet { + let run_command = if track_exit_code { + "exitCode = (await $`{.:command}`).code;" + } else { + "await $`{.:command}`;" + }; + + let mut rule_set = TranslationRuleSet::new("shell-to-javascript"); + for created in structure_rules(run_command) + .into_iter() + .chain(control_flow_rules()) + .chain(declaration_rules()) + .chain(expansion_rules()) + { + rule_set.add_rule(created); + } + rule_set +} + +/// Script structure, commands, pipelines and the `&&`/`||` operators. +fn structure_rules(run_command: &str) -> Vec { + vec![ + // ---- Script structure ---------------------------------------------- + rule( + "script", + &[], + &[(TARGET, "{*.|\\n}"), (COMMAND, "{*.:command|; }")], + ), + rule( + "block", + &[], + &[(TARGET, "{*.|\\n}"), (COMMAND, "{*.:command|; }")], + ), + rule("blank", &[], &[(TARGET, "")]), + rule("comment", &[("body", 0)], &[(TARGET, "// {body:text}")]), + // ---- Commands ------------------------------------------------------- + rule( + "command", + &[("words", 0), ("redirects", 1), ("prefix", 2)], + &[ + (TARGET, run_command), + ( + COMMAND, + "{?prefix}{prefix:command} {/prefix}{words:command}{redirects:command}", + ), + ], + ), + rule("word-list", &[], &[(COMMAND, "{*.:command| }")]), + rule("redirect-list", &[], &[(COMMAND, "{*.:command}")]), + rule( + "redirect", + &[("operator", 0), ("target", 1)], + &[(COMMAND, " {operator:text}{target:command}")], + ), + rule("assignment-list", &[], &[(COMMAND, "{*.:command| }")]), + rule( + "word", + &[], + &[(COMMAND, "{*.:command}"), (VALUE, "{*.:value}")], + ), + rule( + "pipeline", + &[], + &[(TARGET, run_command), (COMMAND, "{*.:command| | }")], + ), + rule( + "subshell", + &[("body", 0)], + &[(TARGET, run_command), (COMMAND, "({body:command})")], + ), + // ---- Operators ------------------------------------------------------ + // `a && b` is a conditional, not a boolean expression: `b` runs only + // when `a` exits zero. In command position the shell operator is kept. + rule( + "and", + &[("left", 0), ("right", 1)], + &[ + ( + TARGET, + "if ((await $`{left:command}`).code === 0) {\n {right}\n}", + ), + (COMMAND, "{left:command} && {right:command}"), + ], + ), + rule( + "or", + &[("left", 0), ("right", 1)], + &[ + ( + TARGET, + "if ((await $`{left:command}`).code !== 0) {\n {right}\n}", + ), + (COMMAND, "{left:command} || {right:command}"), + ], + ), + ] +} + +/// `if`, loops, `case` and function definitions. +fn control_flow_rules() -> Vec { + vec![ + rule( + "if", + &[("condition", 0), ("consequent", 1), ("alternative", 2)], + &[( + TARGET, + concat!( + "if ((await $`{condition:command}`).code === 0) {\n {consequent}\n}", + "{?alternative} else {\n {alternative}\n}{/alternative}" + ), + )], + ), + rule( + "while", + &[("condition", 0), ("body", 1)], + &[( + TARGET, + "while ((await $`{condition:command}`).code === 0) {\n {body}\n}", + )], + ), + rule( + "until", + &[("condition", 0), ("body", 1)], + &[( + TARGET, + "while ((await $`{condition:command}`).code !== 0) {\n {body}\n}", + )], + ), + rule( + "for", + &[("name", 0), ("items", 1), ("body", 2)], + &[( + TARGET, + "for (const {name:text} of [{*items:string|, }]) {\n {body}\n}", + )], + ), + rule( + "case", + &[("subject", 0), ("branches", 1)], + &[(TARGET, "switch ({subject:string}) {\n {*branches|\n }\n}")], + ), + rule( + "case-branch", + &[("patterns", 0), ("body", 1)], + &[(TARGET, "{patterns}\n {body}\n break;")], + ), + rule("pattern-list", &[], &[(TARGET, "{*.|\\n}")]), + rule( + "pattern", + &[("name", 0)], + &[(TARGET, "case {name:string}:")], + ), + // `*` is the shell's catch-all pattern. + rule("pattern-default", &[], &[(TARGET, "default:")]), + rule( + "function", + &[("name", 0), ("body", 1)], + &[(TARGET, "async function {name:text}(...args) {\n {body}\n}")], + ), + rule( + "name", + &[("name", 0)], + &[(TARGET, "{name:text}"), (COMMAND, "{name:text}")], + ), + ] +} + +/// Assignments, `export`/`local`, and the shell builtins with a JS equivalent. +fn declaration_rules() -> Vec { + vec![ + rule( + "assignment", + &[("name", 0), ("value", 1)], + &[(TARGET, "{name:text} = {value:string};")], + ), + rule( + "local", + &[("name", 0), ("value", 1)], + &[(TARGET, "let {name:text} = {value:string};")], + ), + rule( + "export", + &[("name", 0), ("value", 1)], + &[(TARGET, "process.env.{name:text} = {value:string};")], + ), + rule("set-option", &[], &[(TARGET, "shell.set({*.:flag|, });")]), + rule("exit", &[], &[(TARGET, "process.exit({*.:value|});")]), + rule( + "return", + &[("first", 0)], + &[(TARGET, "return{?first} {first:string}{/first};")], + ), + rule( + "source", + &[("file", 0)], + &[(TARGET, "await import({file:string});")], + ), + ] +} + +/// The leaves produced by the word expander. +fn expansion_rules() -> Vec { + vec![ + rule( + "literal", + &[("name", 0)], + &[ + (COMMAND, "{name:literal}"), + (VALUE, "{name:unquoted}"), + (EXPRESSION, "{name:string}"), + ], + ), + expansion("variable", "{name:text}"), + expansion("env-variable", "process.env.{name:text}"), + expansion("positional", "args[{name:text}]"), + expansion("script-name", "process.argv[1]"), + expansion("all-arguments", "args.join(' ')"), + expansion("argument-count", "args.length"), + expansion("exit-status", "exitCode"), + expansion("process-id", "process.pid"), + rule( + "default-expansion", + &[("name", 0), ("value", 1), ("fallback", 2)], + &[ + (COMMAND, "${{{value:expr} ?? {fallback:string}}}"), + (VALUE, "${{{value:expr} ?? {fallback:string}}}"), + (EXPRESSION, "({value:expr} ?? {fallback:string})"), + ], + ), + rule( + "substitution", + &[("body", 0)], + &[ + (COMMAND, "${{(await $`{body:command}`).stdout.trim()}}"), + (VALUE, "${{(await $`{body:command}`).stdout.trim()}}"), + (EXPRESSION, "(await $`{body:command}`).stdout.trim()"), + ], + ), + // Anything the expander could not classify stays a shell expansion: the + // escaped `\${...}` reaches command-stream's shell verbatim rather than + // being silently mistranslated. + rule( + "unsupported-expansion", + &[("name", 0)], + &[ + (COMMAND, "\\${{{name:text}}}"), + (VALUE, "\\${{{name:text}}}"), + ], + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn escapes_template_literal_syntax() { + assert_eq!(escape_template("a`b"), "a\\`b"); + assert_eq!(escape_template("${x}"), "\\${x}"); + } + + #[test] + fn strips_shell_quoting() { + assert_eq!(strip_shell_quotes("\"hi there\""), "hi there"); + assert_eq!(strip_shell_quotes("'a\\b'"), "a\\b"); + } + + #[test] + fn maps_set_flags() { + assert_eq!(as_flag("-e"), "\"e\""); + assert_eq!(as_flag("-o"), "\"-o\""); + } + + #[test] + fn every_rule_is_uniquely_named() { + let rule_set = build_rule_set(false); + let mut names: Vec<&str> = rule_set.rules().iter().map(TranslationRule::name).collect(); + let count = names.len(); + names.sort_unstable(); + names.dedup(); + assert_eq!(names.len(), count); + } +} diff --git a/rust/src/fy/word_expander.rs b/rust/src/fy/word_expander.rs new file mode 100644 index 0000000..c9bb3b8 --- /dev/null +++ b/rust/src/fy/word_expander.rs @@ -0,0 +1,312 @@ +//! Splits a shell word into typed parts so that every leaf of the network is +//! translated by a rule rather than by ad-hoc string surgery. +//! +//! A word such as `"${DIR}/$(basename "$1").log"` becomes: +//! literal(") variable(DIR) literal(/) substitution(...) literal(.log") + +use std::collections::HashSet; + +use super::parser::Node; + +/// Length of the leading `[A-Za-z_][A-Za-z0-9_]*` run, or 0 when there is none. +fn name_length(text: &[char]) -> usize { + let Some(&first) = text.first() else { + return 0; + }; + if !(first.is_ascii_alphabetic() || first == '_') { + return 0; + } + text.iter() + .position(|character| !(character.is_ascii_alphanumeric() || *character == '_')) + .unwrap_or(text.len()) +} + +/// Length of the parameter a `${...}` body starts with: a name, a positional +/// number, or one of the special parameters. +fn parameter_length(text: &[char]) -> usize { + let name = name_length(text); + if name > 0 { + return name; + } + let Some(&first) = text.first() else { + return 0; + }; + if first.is_ascii_digit() { + return text + .iter() + .position(|character| !character.is_ascii_digit()) + .unwrap_or(text.len()); + } + usize::from("@*#?$".contains(first)) +} + +/// Merges adjacent literal parts so the output stays compact. +fn compact(parts: Vec) -> Vec { + let mut merged: Vec = Vec::new(); + for part in parts { + if part.term == "literal" { + if let Some(previous) = merged.last_mut() { + if previous.term == "literal" { + let text = previous.text.get_or_insert_with(String::new); + text.push_str(part.text.as_deref().unwrap_or_default()); + continue; + } + } + } + merged.push(part); + } + merged.retain(|part| part.term != "literal" || part.text.as_deref() != Some("")); + merged +} + +/// Classifies a `$...` expansion. +/// +/// `declared` holds the names the script assigns itself. +fn expansion_node(name: &str, declared: &HashSet) -> Node { + if !name.is_empty() && name.chars().all(|character| character.is_ascii_digit()) { + // `$0` is the script itself; `$1` is the first user argument, which is + // index 0 of `std::env::args().skip(1)`. Doing the renumbering here + // keeps the translation rules free of arithmetic. + if name == "0" { + return Node::leaf("script-name", name); + } + let index = name.parse::().unwrap_or(1).saturating_sub(1); + return Node::leaf("positional", index.to_string()); + } + match name { + "@" | "*" => Node::leaf("all-arguments", name), + "#" => Node::leaf("argument-count", name), + "?" => Node::leaf("exit-status", name), + "$" => Node::leaf("process-id", name), + // A name the script assigns becomes a binding; anything else can only + // come from the environment. + _ => Node::leaf( + if declared.contains(name) { + "variable" + } else { + "env-variable" + }, + name, + ), + } +} + +/// `${NAME}`, `${NAME:-default}` and `${NAME:=default}` are supported directly; +/// anything more exotic is preserved verbatim as an environment lookup so no +/// information is silently dropped. +fn brace_expansion(body: &str, declared: &HashSet) -> Node { + let characters: Vec = body.chars().collect(); + let name = parameter_length(&characters); + if name == characters.len() && name > 0 { + return expansion_node(body, declared); + } + // `NAME:-default` / `NAME:=default` / `NAME-default` / `NAME=default` + if name > 0 { + let mut cursor = name; + if characters.get(cursor) == Some(&':') { + cursor += 1; + } + if matches!(characters.get(cursor), Some('-') | Some('=')) { + let variable: String = characters[..name].iter().collect(); + let fallback: String = characters[cursor + 1..].iter().collect(); + return Node::with_text( + "default-expansion", + vec![ + expansion_node(&variable, declared), + Node::leaf("literal", fallback), + ], + variable, + ); + } + } + Node::leaf("unsupported-expansion", body) +} + +fn matching_delimiter(text: &[char], start: usize, open: char, close: char) -> usize { + let mut depth = 0usize; + for (offset, &character) in text[start..].iter().enumerate() { + if character == open { + depth += 1; + } else if character == close { + depth -= 1; + if depth == 0 { + return start + offset; + } + } + } + text.len() +} + +/// Reads the expansion starting at `index`, if there is one. +/// +/// Returns the expansion node and the index just past it, or `None` when the +/// character is ordinary text. +fn expansion_at( + text: &[char], + index: usize, + declared: &HashSet, + parse_substitution: &dyn Fn(&str) -> Node, +) -> Option<(Node, usize)> { + let character = text[index]; + + if character == '`' { + let close = text[index + 1..] + .iter() + .position(|&c| c == '`') + .map_or(text.len(), |offset| index + 1 + offset); + let body: String = text[index + 1..close.min(text.len())].iter().collect(); + return Some(( + Node::new("substitution", vec![parse_substitution(&body)]), + close + 1, + )); + } + + if character != '$' { + return None; + } + + if text.get(index + 1) == Some(&'(') { + let close = matching_delimiter(text, index + 1, '(', ')'); + let body: String = text[(index + 2).min(close)..close].iter().collect(); + return Some(( + Node::new("substitution", vec![parse_substitution(&body)]), + close + 1, + )); + } + + if text.get(index + 1) == Some(&'{') { + let close = matching_delimiter(text, index + 1, '{', '}'); + let body: String = text[(index + 2).min(close)..close].iter().collect(); + return Some((brace_expansion(&body, declared), close + 1)); + } + + let rest = &text[index + 1..]; + let length = name_length(rest); + if length > 0 { + let name: String = rest[..length].iter().collect(); + return Some((expansion_node(&name, declared), index + 1 + length)); + } + let special = *rest.first()?; + if "@*#?$".contains(special) || special.is_ascii_digit() { + return Some((expansion_node(&special.to_string(), declared), index + 2)); + } + None +} + +/// Expands a shell word (quotes included) into part nodes. +/// +/// `declared` holds the variable names assigned by the script and +/// `parse_substitution` parses the body of a command substitution into a +/// statement node. +pub fn expand_word( + text: &str, + declared: &HashSet, + parse_substitution: &dyn Fn(&str) -> Node, +) -> Vec { + let text: Vec = text.chars().collect(); + let mut parts: Vec = Vec::new(); + let mut literal = String::new(); + let mut single_quoted = false; + let mut index = 0usize; + + while index < text.len() { + let character = text[index]; + + if character == '\'' { + single_quoted = !single_quoted; + literal.push(character); + index += 1; + continue; + } + // Inside single quotes the shell performs no expansion at all. + if single_quoted { + literal.push(character); + index += 1; + continue; + } + if character == '\\' { + literal.extend(text[index..(index + 2).min(text.len())].iter()); + index += 2; + continue; + } + + if let Some((part, end)) = expansion_at(&text, index, declared, parse_substitution) { + if !literal.is_empty() { + parts.push(Node::leaf("literal", std::mem::take(&mut literal))); + } + parts.push(part); + index = end; + continue; + } + + literal.push(character); + index += 1; + } + + if !literal.is_empty() { + parts.push(Node::leaf("literal", literal)); + } + compact(parts) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn no_substitution(source: &str) -> Node { + Node::leaf("word", source) + } + + fn expand(text: &str, declared: &[&str]) -> Vec<(String, Option)> { + let declared: HashSet = declared.iter().map(|name| name.to_string()).collect(); + expand_word(text, &declared, &no_substitution) + .into_iter() + .map(|part| (part.term, part.text)) + .collect() + } + + #[test] + fn splits_a_word_into_typed_parts() { + assert_eq!( + expand("\"${DIR}/log\"", &["DIR"]), + vec![ + ("literal".to_string(), Some("\"".to_string())), + ("variable".to_string(), Some("DIR".to_string())), + ("literal".to_string(), Some("/log\"".to_string())), + ] + ); + } + + #[test] + fn classifies_special_parameters() { + assert_eq!( + expand("$1 $# $? $HOME", &[]), + vec![ + ("positional".to_string(), Some("0".to_string())), + ("literal".to_string(), Some(" ".to_string())), + ("argument-count".to_string(), Some("#".to_string())), + ("literal".to_string(), Some(" ".to_string())), + ("exit-status".to_string(), Some("?".to_string())), + ("literal".to_string(), Some(" ".to_string())), + ("env-variable".to_string(), Some("HOME".to_string())), + ] + ); + } + + #[test] + fn keeps_single_quoted_text_unexpanded() { + assert_eq!( + expand("'$HOME'", &[]), + vec![("literal".to_string(), Some("'$HOME'".to_string()))] + ); + } + + #[test] + fn reads_a_default_expansion() { + let declared = HashSet::new(); + let parts = expand_word("${NAME:-anon}", &declared, &no_substitution); + assert_eq!(parts[0].term, "default-expansion"); + assert_eq!(parts[0].text.as_deref(), Some("NAME")); + assert_eq!(parts[0].children[1].text.as_deref(), Some("anon")); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 5544620..96040f8 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -64,6 +64,7 @@ // Modular utility modules (following JavaScript modular pattern) pub mod ansi; pub mod events; +pub mod fy; #[doc(hidden)] pub mod macros; pub mod pipeline; @@ -436,6 +437,7 @@ impl ProcessRunner { "yes" => Some(commands::yes(ctx).await), "seq" => Some(commands::seq(ctx).await), "test" => Some(commands::test(ctx).await), + "$fy" => Some(commands::fy(ctx).await), _ => None, } } diff --git a/rust/src/pipeline.rs b/rust/src/pipeline.rs index 8309ef8..75943fe 100644 --- a/rust/src/pipeline.rs +++ b/rust/src/pipeline.rs @@ -295,6 +295,7 @@ impl Pipeline { "yes" => Some(crate::commands::yes(ctx).await), "seq" => Some(crate::commands::seq(ctx).await), "test" => Some(crate::commands::test(ctx).await), + "$fy" => Some(crate::commands::fy(ctx).await), _ => None, } } diff --git a/rust/tests/fy.rs b/rust/tests/fy.rs new file mode 100644 index 0000000..f13b5fc --- /dev/null +++ b/rust/tests/fy.rs @@ -0,0 +1,41 @@ +//! Integration tests for the `$fy` translator. +//! +//! The golden fixture in `fixtures/fy/` is shared with the JavaScript suite +//! (`js/tests/fy-tool.test.mjs`), so both implementations are pinned to the +//! same bytes and cannot drift apart. + +use std::path::PathBuf; + +use command_stream::fy::{translate_shell_to_mjs, TranslateOptions}; + +fn fixture(name: &str) -> String { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../fixtures/fy") + .join(name); + std::fs::read_to_string(&path).unwrap_or_else(|error| panic!("read {path:?}: {error}")) +} + +#[test] +fn reproduces_the_shared_golden_fixture() { + let translation = translate_shell_to_mjs(&fixture("sample.sh"), &TranslateOptions::default()); + + assert_eq!(translation.diagnostics, Vec::::new()); + assert_eq!(translation.code, fixture("sample.mjs")); +} + +#[test] +fn translates_every_construct_of_the_fixture_structurally() { + let translation = translate_shell_to_mjs(&fixture("sample.sh"), &TranslateOptions::default()); + let code = translation.code; + + for expected in [ + "import { $, shell } from 'command-stream';", + "const args = process.argv.slice(2);", + "async function deploy(...args) {", + "for (const env of [`staging`, `prod`]) {", + "switch (`${TARGET}`) {", + "process.exit(0);", + ] { + assert!(code.contains(expected), "missing {expected:?} in:\n{code}"); + } +}