E2E testing

.github/workflows/test.yml

@@ -111,11 +111,74 @@ jobs:
         env:
           WABT: ${{ github.workspace }}/.tools
 
-      - name: Run example tests
-        run: npm run test:example
+  integration:
+    name: Integration tests
+    runs-on: ubuntu-latest
+    needs: [build-wabt, build-wdcli]
+    if: github.event.pull_request.draft == false
+    steps:
+      - uses: actions/checkout@v4
+
+      - run: npm i
+
+      - name: Prebuild files
+        run: npm run test:prebuild
+
+      - name: Download WAT tools
+        uses: actions/download-artifact@v4
+        with:
+          name: wabt-build-${{ github.run_id }}
+          path: .tools
+
+      - name: Download WARDuino CLI
+        uses: actions/download-artifact@v4
+        with:
+          name: warduino-build-${{ github.run_id }}
+          path: .warduino
+
+      - name: Configure test tools
+        run: |
+          chmod u+x "$GITHUB_WORKSPACE"/.tools/*
+          "$GITHUB_WORKSPACE"/.tools/wat2wasm --version
+          EMULATOR_PATH="$(find "$GITHUB_WORKSPACE"/.warduino -type f -name wdcli -print -quit)"
+          test -n "$EMULATOR_PATH"
+          chmod u+x "$EMULATOR_PATH"
+          echo "EMULATOR=$EMULATOR_PATH" >> "$GITHUB_ENV"
+
+      - name: Run integration tests
+        run: npm run test:integration
         env:
           WABT: ${{ github.workspace }}/.tools
 
+  end-to-end:
+    name: End-to-end tests
+    runs-on: ubuntu-latest
+    needs: build-wdcli
+    if: github.event.pull_request.draft == false
+    steps:
+      - uses: actions/checkout@v4
+
+      - run: npm i
+
+      - name: Prebuild files
+        run: npm run test:prebuild
+
+      - name: Download WARDuino CLI
+        uses: actions/download-artifact@v4
+        with:
+          name: warduino-build-${{ github.run_id }}
+          path: .warduino
+
+      - name: Configure emulator
+        run: |
+          EMULATOR_PATH="$(find "$GITHUB_WORKSPACE"/.warduino -type f -name wdcli -print -quit)"
+          test -n "$EMULATOR_PATH"
+          chmod u+x "$EMULATOR_PATH"
+          echo "EMULATOR=$EMULATOR_PATH" >> "$GITHUB_ENV"
+
+      - name: Run end-to-end tests
+        run: npm run test:end-to-end
+
   coverage:
     name: Code coverage
     runs-on: ubuntu-latest

package.json

@@ -42,7 +42,7 @@
     "test:all": "npm run test:prebuild && npm run test:ava && npm run test:integration && npm run test:end-to-end",
     "test:ava": "ava",
     "test:integration": "npx ts-node ./tests/integration/precision.ts",
-    "test:end-to-end": "npx ts-node ./tests/end-to-end/r3/r3.test.ts",
+    "test:end-to-end": "npx ts-node ./tests/end-to-end/end2end.test.ts",
     "test:example": "npx ts-node ./tests/examples/example.ts",
     "coverage:test:ava": "c8 --src src/ --all ava"
   },

src/framework/Framework.ts

@@ -138,6 +138,7 @@ export class Framework {
                     try {
                         const first: TestScenario = order[i][0];
                         await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));
+                        this.reportMetadata(runId, testee);
 
                         for (let j = i; j < order.length; j += suite.testees.length) {
                             await this.runSuite(result, testee, order[j], runId);
@@ -189,6 +190,7 @@ export class Framework {
         try {
             const first: TestScenario = order[0];
             await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));
+            this.reportMetadata(runId, testee);
             await this.runSuite(result, testee, order, runId);
         } catch (e) {
             result.error(e instanceof Error ? e.message : `${e}`);
@@ -209,6 +211,13 @@ export class Framework {
         return `${suite.title}:${testee.name}:${executionIndex}`;
     }
 
+    private reportMetadata(runId: string, testee: Testee): void {
+        const testbed = testee.bed();
+        if (testbed !== undefined) {
+            this.reporter.metadata?.(runId, testbed.meta());
+        }
+    }
+
     public static getImplementation() {
         if (!Framework.implementation) {
             Framework.implementation = new Framework();

src/framework/Testee.ts

@@ -278,9 +278,10 @@ export class Testee { // TODO unified with testbed interface
                                     await testee.initialize(description.program, description.args ?? []).catch((o) => {
                                         return Promise.reject(o)
                                     });
-                                }), 1).catch((e: string) => {
+                                }), 1).catch((e: unknown) => {
                                 const result = new StepOutcome(step);
-                                testee.states.set(description.title, result.update((e.includes('timeout')) ? Outcome.timedout : Outcome.error, e));
+                                const error = e instanceof Error ? e.toString() : String(e);
+                                testee.states.set(description.title, result.update((error.includes('timeout')) ? Outcome.timedout : Outcome.error, error));
                                 recordStep(result);
                             });
                         }

src/reporter/Reporter.ts

@@ -31,6 +31,8 @@ export interface Reporter {
 
     debug(text: string): void;
 
+    metadata?(runId: string, metadata: Promise<string>): void;
+
     finish(durationMs: number): void;
 
     close(): Promise<void>;

src/reporter/ink/App.tsx

@@ -1,7 +1,7 @@
 import {Box} from 'ink';
 import {ReporterSnapshot} from '../ReporterState';
 import {Verbosity} from '../index';
-import {RunHeader} from './RunHeader';
+import {Header} from './Header';
 import {ProgressSummary} from './ProgressSummary';
 import {SuiteList} from './SuiteList';
 import {LogPanel} from './LogPanel';
@@ -13,22 +13,24 @@ interface Props {
     snapshot: ReporterSnapshot;
     archive: string;
     verbosity: Verbosity;
+    metadata?: Promise<string>[];
+    metadataRevision?: number;
 }
 
-export function App({snapshot, archive, verbosity}: Props) {
+export function App({snapshot, archive, verbosity, metadata, metadataRevision}: Props) {
     if (snapshot.finished) {
         return (
             <Box flexDirection="column">
-                {showsDebugDetails(verbosity) ? <RunHeader archive={archive}/> : null}
-                <FinalSummary snapshot={snapshot} archive={archive} verbosity={verbosity}/>
+                <Header archive={archive} metadata={metadata} metadataRevision={metadataRevision}/>
+                <FinalSummary snapshot={snapshot} verbosity={verbosity}/>
                 {showsDebugDetails(verbosity) ? <LogPanel logs={snapshot.logs} verbosity={verbosity}/> : null}
             </Box>
         );
     }
 
     return (
         <Box flexDirection="column">
-            <RunHeader archive={archive}/>
+            <Header archive={archive} metadata={metadata} metadataRevision={metadataRevision}/>
             <SuiteList snapshot={snapshot} verbosity={verbosity}/>
             {verbosity === Verbosity.normal ? <ActiveFailures snapshot={snapshot}/> : null}
             <ProgressSummary snapshot={snapshot}/>

src/reporter/ink/FinalSummary.tsx

@@ -11,11 +11,10 @@ import {preservesFullHistory, showsActionDetails} from './verbosity';
 
 interface Props {
     snapshot: ReporterSnapshot;
-    archive: string;
     verbosity: Verbosity;
 }
 
-export function FinalSummary({snapshot, archive, verbosity}: Props) {
+export function FinalSummary({snapshot, verbosity}: Props) {
     if (!snapshot.finished) {
         return null;
     }
@@ -90,8 +89,6 @@ export function FinalSummary({snapshot, archive, verbosity}: Props) {
             <Text><Text bold>Suites</Text>     {alignRight(`${summary.suites.passing}`, firstCountWidth)} passed · {summary.suites.failing} failed</Text>
             <Text><Text bold>Scenarios</Text>  {alignRight(`${summary.scenarios.passing}`, firstCountWidth)} passed · {summary.scenarios.failing} failed · {summary.scenarios.errors} errors · {summary.scenarios.skipped} skipped</Text>
             <Text><Text bold>Actions</Text>    {alignRight(`${summary.actions.passing}`, firstCountWidth)} passed · {summary.actions.failing} failed · {summary.actions.errors} errors · {summary.actions.timeouts} timeouts</Text>
-            <Text> </Text>
-            <Text><Text bold>Archive</Text>  {archive}</Text>
         </Box>
     );
 }

src/reporter/ink/Header.tsx

@@ -0,0 +1,76 @@
+import {Box, Text} from 'ink';
+import {memo, useEffect, useState} from 'react';
+import {version} from '../../../package.json';
+import {Meta} from '../../testbeds/Testbed';
+
+interface Props {
+    archive: string;
+    metadata?: Promise<string>[];
+    metadataRevision?: number;
+}
+
+interface PlatformOverviewProps {
+    metadata?: Promise<string>[];
+    metadataRevision?: number;
+}
+
+const metadataPlaceholder = 'name [architecture] · vversion';
+const headerLabelWidth = 'Testbeds'.length;
+
+export const PlatformOverview = memo(function PlatformOverview({metadata = [], metadataRevision = 0}: PlatformOverviewProps) {
+    const [details, setDetails] = useState<string[]>([]);
+
+    useEffect(() => {
+        let mounted = true;
+        setDetails(previous => metadata.map((_, index) => previous[index] ?? metadataPlaceholder));
+
+        metadata.forEach((entry, index) => {
+            entry.then(raw => {
+                const meta = JSON.parse(raw) as Record<string, unknown>;
+                const values = [meta[Meta.Name], meta[Meta.Architecture], meta[Meta.Version]];
+
+                if (mounted && values.every((value): value is string => typeof value === 'string')) {
+                    const [name, architecture, version] = values;
+                    const detail = `${name} [${architecture}] · v${version}`;
+                    setDetails(previous => previous.map((current, detailIndex) => detailIndex === index ? detail : current));
+                }
+            }).catch(() => undefined);
+        });
+
+        return () => {
+            mounted = false;
+        };
+    }, [metadataRevision]);
+
+    const overview = Array.from(details.reduce((groups, detail) => {
+        groups.set(detail, (groups.get(detail) ?? 0) + 1);
+        return groups;
+    }, new Map<string, number>()).entries()).map(([detail, count]) =>
+        `${detail} · ${count} suite${count === 1 ? '' : 's'}`
+    );
+
+    const rows = overview.length > 0 ? overview : [metadataPlaceholder];
+
+    return (
+        <Box flexDirection="column">
+            {rows.map((row, index) => (
+                <Box key={row}>
+                    <Text bold>{index === 0 ? 'Testbeds' : ''.padEnd(headerLabelWidth)}</Text>
+                    <Text color="gray">  {row}</Text>
+                </Box>
+            ))}
+        </Box>
+    );
+}, (previous, next) => previous.metadataRevision === next.metadataRevision);
+
+export const Header = memo(function Header({archive, metadata, metadataRevision}: Props) {
+    return (
+        <Box flexDirection="column" marginBottom={1}>
+            <Box>
+                <Text bold>{'Latch'.padEnd(headerLabelWidth)}</Text>
+                <Text color="gray">  v{version} · archive {archive}</Text>
+            </Box>
+            <PlatformOverview metadata={metadata} metadataRevision={metadataRevision}/>
+        </Box>
+    );
+}, (previous, next) => previous.archive === next.archive && previous.metadataRevision === next.metadataRevision);

src/reporter/ink/InkReporter.ts

@@ -12,6 +12,8 @@ export class InkReporter implements Reporter {
     private readonly archiveWriter: ArchiveWriter;
     private instance?: Instance;
     private verbosityLevel: Verbosity;
+    private readonly testbedMetadata = new Map<string, Promise<string>>();
+    private metadataRevision = 0;
 
     constructor(verbosity: Verbosity = Verbosity.normal, archiveWriter: ArchiveWriter = new ArchiveWriter()) {
         this.verbosityLevel = verbosity;
@@ -24,6 +26,8 @@ export class InkReporter implements Reporter {
     }
 
     start() {
+        this.testbedMetadata.clear();
+        this.metadataRevision = 0;
         this.state.start();
         this.instance = render(this.element(), {patchConsole: true});
     }
@@ -70,6 +74,12 @@ export class InkReporter implements Reporter {
         }
     }
 
+    metadata(runId: string, metadata: Promise<string>) {
+        this.testbedMetadata.set(runId, metadata);
+        this.metadataRevision++;
+        this.rerender();
+    }
+
     finish(durationMs: number) {
         this.state.finish(durationMs);
         this.archiveWriter.write(durationMs, this.state.suites());
@@ -97,7 +107,9 @@ export class InkReporter implements Reporter {
         return React.createElement(App, {
             snapshot: this.state.snapshot(),
             archive: this.archiveWriter.archive,
-            verbosity: this.verbosityLevel
+            verbosity: this.verbosityLevel,
+            metadata: Array.from(this.testbedMetadata.values()),
+            metadataRevision: this.metadataRevision
         });
     }
 

src/reporter/ink/SuiteList.tsx

@@ -1,4 +1,4 @@
-import {Box} from 'ink';
+import {Box, Text} from 'ink';
 import {ReporterSnapshot} from '../ReporterState';
 import {Verbosity} from '../index';
 import {SuiteView} from './SuiteView';
@@ -13,6 +13,7 @@ export function SuiteList({snapshot, verbosity}: Props) {
         <Box flexDirection="column">
             {snapshot.completedRuns.map((run) => <SuiteView key={run.id} run={run} active={false}
                 verbosity={verbosity}/>)}
+            {snapshot.completedRuns.length > 0 && snapshot.activeRuns.length > 0 ? <Text> </Text> : null}
             {snapshot.activeRuns.map((run) => <SuiteView key={run.id} run={run} active={true} verbosity={verbosity}/>)}
         </Box>
     );

src/sourcemap/SourceMapFactory.ts

@@ -21,10 +21,10 @@ export class SourceMapFactory {
                 compiled = await this.compilerFactory.pickCompiler(source).compile(source);
                 return new WatMapper(compiled.out ?? '', tmpdir ?? path.dirname(compiled.file), WABT).mapping();
             case 'wasm' :
-                // Precompiled modules do not carry a WAT source mapping. The
-                // module can still be executed; requests that need source
-                // locations simply have no mapping to resolve against.
-                return new SourceMap.Mapping();
+                // Precompiled modules do not carry source locations, but their
+                // export section still provides the function names required by
+                // invoke requests.
+                return this.compilerFactory.pickCompiler(source).map(source).then(output => output.map!);
             case 'ts' :
                 return new AsScriptMapper(source ?? '', tmpdir ?? path.dirname(source)).mapping();
         }

src/testbeds/Arduino.ts

@@ -1,5 +1,8 @@
 import {Serial} from '../bridge/Serial';
 import {Platform} from './Platform';
+import {EMULATOR} from "../util/env";
+import {Meta} from "./Testbed";
+import {execFileAsync} from "../util/util";
 
 export class Arduino extends Platform {
     public readonly name: string = 'Hardware';
@@ -12,4 +15,19 @@ export class Arduino extends Platform {
 
         this.listen();
     }
-}
+
+    async meta(): Promise<string> {
+        const {stdout} = await execFileAsync(EMULATOR, ['--version']);
+        const version = stdout.match(/\d+\.\d+\.\d+/)?.[0];
+
+        if (version === undefined) {
+            throw new Error(`Unable to determine WARDuino version from: ${stdout.trim()}`);
+        }
+
+        return JSON.stringify({
+            [Meta.Name]: 'warduino',
+            [Meta.Architecture]: 'arduino',
+            [Meta.Version]: version
+        });
+    }
+}