diff --git a/src/target-go.ts b/src/target-go.ts index 5ff9948..7f125d8 100644 --- a/src/target-go.ts +++ b/src/target-go.ts @@ -213,19 +213,38 @@ ${pushHooks}\t\ttoks = append(toks, Tok{kind, text, off, end, pendingNl}); pendi \t_ = pushTok ` : `\tpushTok := func(kind, text string, off, end int) { toks = append(toks, Tok{kind, text, off, end, pendingNl}); pendingNl = false }\n\t_ = pushTok\n`; - return `${defs.length ? 'var _s string\n' + defs.join('\n') + '\n' : ''}func lex(src string) []Tok { + const pushTokAccFn = `\tpushTok := func(kind, text string, off, end int) { *acc = append(*acc, Tok{kind, text, off, end, pendingNl}); pendingNl = false } +\t_ = pushTok +`; + const loopBody = `${nlBoundary}\t\tc := int(src[pos]) +${nlWs}${tplDispatch}${toks} +${puncts} +\t\tpanic(fmt.Sprintf("lex error at %d", pos))`; + if (stateful) { + return `${defs.length ? 'var _s string\n' + defs.join('\n') + '\n' : ''}func lex(src string) []Tok { \ttoks := toks[:0] \tn := len(src) \tpos := 0 \tpendingNl := false \t_ = pendingNl ${rxState}${tplState}${nlState}${emitFn}${pushTokFn}${defs.length ? '\t_s = src\n' : ''}\tfor pos < n { -${nlBoundary}\t\tc := int(src[pos]) -${nlWs}${tplDispatch}${toks} -${puncts} -\t\tpanic(fmt.Sprintf("lex error at %d", pos)) +${loopBody} \t} \treturn toks +}`; + } + return `${defs.length ? 'var _s string\n' + defs.join('\n') + '\n' : ''}func lexFrom(src string, pos int, pendingNl bool, acc *[]Tok, limit int) (int, bool) { +\tn := len(src) +${pushTokAccFn}${defs.length ? '\t_s = src\n' : ''}\tbase := len(*acc) +\tfor pos < n && (limit <= 0 || len(*acc)-base < limit) { +${loopBody} +\t} +\treturn pos, pendingNl +} +func lex(src string) []Tok { +\tvar out []Tok +\tlexFrom(src, 0, false, &out, 0) +\treturn out }`; } @@ -373,6 +392,159 @@ ${r.nudSeqs.map((seq) => `\t{ save := pos; sb := len(scratch); nb := len(nodes); }`; } +function docEditBlockGo(ir: ParserIR): string { + const stateless = !(ir.regexCtx || ir.tpl || ir.newlineCfg); + const windowHelpers = stateless ? ` +func findTokAtOff(toks []alignMeta, off int) int { +\tlo, hi := 0, len(toks)-1 +\tfor lo <= hi { +\t\tmid := (lo + hi) >> 1 +\t\tif toks[mid].Off < off { lo = mid + 1 } else if toks[mid].Off > off { hi = mid - 1 } else { return mid } +\t} +\treturn -1 +} +func windowRelexStep(oldText string, oldToks []alignMeta, newText string, start, end int, ins string) ([]alignMeta, int) { +\tdelta := len(ins) - (end - start) +\teditEnd := start + len(ins) +\tmaxIdx := -1 +\tfor i := 0; i < len(oldToks); i++ { +\t\tif oldToks[i].End < start { maxIdx = i } else { break } +\t} +\trb := -1 +\tif maxIdx >= 0 { rb = maxIdx - 1 } +\tvar out []alignMeta +\tif rb >= 0 { out = append(out, oldToks[:rb+1]...) } +\tscanOff := 0 +\tif rb >= 0 { scanOff = oldToks[rb].End } +\tpendingNl := false +\tvar scratch []Tok +\trelexed := 0 +\tfor scanOff < len(newText) { +\t\tbefore := len(scratch) +\t\tscanOff, pendingNl = lexFrom(newText, scanOff, pendingNl, &scratch, 1) +\t\tif len(scratch) == before { break } +\t\tt := scratch[len(scratch)-1] +\t\tout = append(out, alignMeta{t.Kind, t.Off, t.End, t.Nl}) +\t\trelexed++ +\t\tif t.Off >= editEnd { +\t\t\toIdx := findTokAtOff(oldToks, t.Off-delta) +\t\t\tif oIdx >= 0 { +\t\t\t\to := oldToks[oIdx] +\t\t\t\tif o.Kind == t.Kind && o.End == t.End-delta && o.Nl == t.Nl && oldText[o.Off:o.End] == newText[t.Off:t.End] { +\t\t\t\t\tfor j := oIdx + 1; j < len(oldToks); j++ { +\t\t\t\t\t\tot := oldToks[j] +\t\t\t\t\t\tout = append(out, alignMeta{ot.Kind, ot.Off + delta, ot.End + delta, ot.Nl}) +\t\t\t\t\t} +\t\t\t\t\treturn out, relexed +\t\t\t\t} +\t\t\t} +\t\t} +\t} +\treturn out, relexed +} +` : ''; + const editBody = stateless + ? `\tcurText := d.text +\tcurToks := d.toks +\tfor _, e := range edits { +\t\tstepOldText, stepOldToks := curText, curToks +\t\tn := len(curText) +\t\tstart, end := e.Start, e.End +\t\tif start < 0 { start = 0 } +\t\tif start > n { start = n } +\t\tif end < start { end = start } +\t\tif end > n { end = n } +\t\tins := e.Text +\t\tcurText = curText[:start] + ins + curText[end:] +\t\tvar stepRelexed int +\t\tcurToks, stepRelexed = windowRelexStep(stepOldText, stepOldToks, curText, start, end, ins) +\t\trelexed += stepRelexed +\t} +\td.text = curText +\td.toks = curToks` + : `\tfor _, e := range edits { d.text = applyEdit(d.text, e) } +\tnewToks := tokenize(d.text) +\td.toks = toMeta(newToks) +\trelexed = len(d.toks)`; + return `type Edit struct { Start, End int; Text string } +type alignMeta struct { Kind string; Off, End int; Nl bool } +type Align struct { +\tOldN int \`json:"oldN"\` +\tNewN int \`json:"newN"\` +\tPrefix int \`json:"prefix"\` +\tSuffix int \`json:"suffix"\` +\tRelexed int \`json:"relexed"\` +\tStreamEq bool \`json:"streamEq"\` +} +func toMeta(toks []Tok) []alignMeta { +\tm := make([]alignMeta, len(toks)) +\tfor i, t := range toks { m[i] = alignMeta{t.Kind, t.Off, t.End, t.Nl} } +\treturn m +} +func computeAlignCore(oldText string, oldToks []alignMeta, newText string, newToks []alignMeta) (oldN, newN, prefix, suffix int) { +\toldN, newN = len(oldToks), len(newToks) +\tfor prefix < oldN && prefix < newN { +\t\to, n := oldToks[prefix], newToks[prefix] +\t\tif o.Kind != n.Kind || o.Off != n.Off || o.End != n.End || o.Nl != n.Nl { break } +\t\tif oldText[o.Off:o.End] != newText[n.Off:n.End] { break } +\t\tprefix++ +\t} +\tdelta := len(newText) - len(oldText) +\tminN := oldN; if newN < minN { minN = newN } +\tfor prefix+suffix < minN { +\t\to, n := oldToks[oldN-1-suffix], newToks[newN-1-suffix] +\t\tif o.Kind != n.Kind || o.Nl != n.Nl || n.Off != o.Off+delta || n.End != o.End+delta { break } +\t\tif oldText[o.Off:o.End] != newText[n.Off:n.End] { break } +\t\tsuffix++ +\t} +\treturn +} +func toksFromMeta(text string, meta []alignMeta) []Tok { +\tt := make([]Tok, len(meta)) +\tfor i, m := range meta { t[i] = Tok{m.Kind, text[m.Off:m.End], m.Off, m.End, m.Nl} } +\treturn t +} +func checkStreamEq(text string, meta []alignMeta) bool { +\tfresh := toMeta(tokenize(text)) +\tif len(fresh) != len(meta) { return false } +\tfor i := range fresh { +\t\tf, m := fresh[i], meta[i] +\t\tif f.Kind != m.Kind || f.Off != m.Off || f.End != m.End || f.Nl != m.Nl { return false } +\t\tif text[f.Off:f.End] != text[m.Off:m.End] { return false } +\t} +\treturn true +} +${windowHelpers}type Doc struct { text string; root int32; toks []alignMeta; align *Align } +func NewDoc(src string) *Doc { +\td := &Doc{text: src} +\td.toks = toMeta(tokenize(src)) +\td.root = parse(tokenize(src)) +\treturn d +} +func (d *Doc) Text() string { return d.text } +func (d *Doc) Root() int32 { return d.root } +func (d *Doc) Align() *Align { return d.align } +func applyEdit(text string, e Edit) string { +\tn := len(text) +\tstart, end := e.Start, e.End +\tif start < 0 { start = 0 } +\tif start > n { start = n } +\tif end < start { end = start } +\tif end > n { end = n } +\treturn text[:start] + e.Text + text[end:] +} +func (d *Doc) Edit(edits []Edit) int32 { +\toldText, oldToks := d.text, d.toks +\trelexed := 0 +${editBody} +\tstreamEq := checkStreamEq(d.text, d.toks) +\toldN, newN, prefix, suffix := computeAlignCore(oldText, oldToks, d.text, d.toks) +\td.align = &Align{oldN, newN, prefix, suffix, relexed, streamEq} +\td.root = parse(toksFromMeta(d.text, d.toks)) +\treturn d.root +}`; +} + export const goTarget: Target = { name: 'go', ext: 'go', @@ -551,68 +723,7 @@ func parse(t []Tok) int32 { \treturn parse${ir.entry}() } -type Edit struct { Start, End int; Text string } -type alignMeta struct { Kind string; Off, End int; Nl bool } -type Align struct { -\tOldN int \`json:"oldN"\` -\tNewN int \`json:"newN"\` -\tPrefix int \`json:"prefix"\` -\tSuffix int \`json:"suffix"\` -} -func toMeta(toks []Tok) []alignMeta { -\tm := make([]alignMeta, len(toks)) -\tfor i, t := range toks { m[i] = alignMeta{t.Kind, t.Off, t.End, t.Nl} } -\treturn m -} -func computeAlign(oldText string, oldToks []alignMeta, newText string, newToks []alignMeta) Align { -\toldN, newN := len(oldToks), len(newToks) -\tprefix := 0 -\tfor prefix < oldN && prefix < newN { -\t\to, n := oldToks[prefix], newToks[prefix] -\t\tif o.Kind != n.Kind || o.Off != n.Off || o.End != n.End || o.Nl != n.Nl { break } -\t\tif oldText[o.Off:o.End] != newText[n.Off:n.End] { break } -\t\tprefix++ -\t} -\tdelta := len(newText) - len(oldText) -\tminN := oldN; if newN < minN { minN = newN } -\tsuffix := 0 -\tfor prefix+suffix < minN { -\t\to, n := oldToks[oldN-1-suffix], newToks[newN-1-suffix] -\t\tif o.Kind != n.Kind || o.Nl != n.Nl || n.Off != o.Off+delta || n.End != o.End+delta { break } -\t\tif oldText[o.Off:o.End] != newText[n.Off:n.End] { break } -\t\tsuffix++ -\t} -\treturn Align{oldN, newN, prefix, suffix} -} -type Doc struct { text string; root int32; toks []alignMeta; align *Align } -func NewDoc(src string) *Doc { -\td := &Doc{text: src} -\td.toks = toMeta(tokenize(src)) -\td.root = parse(tokenize(src)) -\treturn d -} -func (d *Doc) Text() string { return d.text } -func (d *Doc) Root() int32 { return d.root } -func (d *Doc) Align() *Align { return d.align } -func applyEdit(text string, e Edit) string { -\tn := len(text) -\tstart, end := e.Start, e.End -\tif start < 0 { start = 0 } -\tif start > n { start = n } -\tif end < start { end = start } -\tif end > n { end = n } -\treturn text[:start] + e.Text + text[end:] -} -func (d *Doc) Edit(edits []Edit) int32 { -\toldText, oldToks := d.text, d.toks -\tfor _, e := range edits { d.text = applyEdit(d.text, e) } -\tnewToks := tokenize(d.text) -\td.toks = toMeta(newToks) -\ta := computeAlign(oldText, oldToks, d.text, d.toks) -\td.align = &a -\td.root = parse(newToks) -\treturn d.root -} +${docEditBlockGo(ir)} `; }, emitRunner(): string { diff --git a/src/target-rust.ts b/src/target-rust.ts index d5b808b..34df964 100644 --- a/src/target-rust.ts +++ b/src/target-rust.ts @@ -220,18 +220,35 @@ ${emitHooks} if pos + 2 < n && b[pos] == 0xE2 && b[pos + 1] == 0x80 && (b[pos + 2] == 0xA8 || b[pos + 2] == 0xA9) { ${nlVar} = true; pos += 3; continue; } // LS/PS (UTF-8) if c == 10 || c == 13 { ${nlVar} = true; pos += 1; continue; } // LF/CR `; - return `${defs.length ? defs.join('\n') + '\n' : ''}${rxConsts}${tplFn}${stateImpl}fn lex<'a>(src: &'a str) -> Vec> { + const loopBody = `${nlBoundary} let c = b[pos] as u32; +${nlWs}${tplDispatch}${toks} +${puncts} + panic!("lex error at {}", pos);`; + if (stateful) { + return `${defs.length ? defs.join('\n') + '\n' : ''}${rxConsts}${tplFn}${stateImpl}fn lex<'a>(src: &'a str) -> Vec> { let b = src.as_bytes(); let n = b.len(); ${open} let mut pos = 0usize; while pos < n { -${nlBoundary} let c = b[pos] as u32; -${nlWs}${tplDispatch}${toks} -${puncts} - panic!("lex error at {}", pos); +${loopBody} } ${stateful ? 'st.toks' : 'toks'} +}`; + } + return `${defs.length ? defs.join('\n') + '\n' : ''}${rxConsts}${tplFn}fn lex_from<'a>(src: &'a str, mut pos: usize, mut pending_nl: bool, acc: &mut Vec>, limit: usize) -> (usize, bool) { + let b = src.as_bytes(); + let n = b.len(); + let base = acc.len(); + while pos < n && (limit == 0 || acc.len() - base < limit) { +${loopBody.replace(/pending_nl/g, 'pending_nl').replace(/toks\.push/g, 'acc.push')} + } + (pos, pending_nl) +} +fn lex<'a>(src: &'a str) -> Vec> { + let mut toks = Vec::new(); + lex_from(src, 0, false, &mut toks, 0); + toks }`; } @@ -398,6 +415,157 @@ ${r.nudSeqs.map((seq) => ` { let save = self.pos; let sb = self.scratch.l }`; } +function docEditBlockRust(ir: ParserIR): string { + const stateless = !(ir.regexCtx || ir.tpl || ir.newlineCfg); + const windowHelpers = stateless ? ` +fn find_tok_at_off(toks: &[AlignMeta], off: usize) -> Option { + let mut lo = 0usize; + let mut hi = toks.len(); + while lo < hi { + let mid = (lo + hi) / 2; + if toks[mid].off < off { lo = mid + 1; } else { hi = mid; } + } + if lo < toks.len() && toks[lo].off == off { Some(lo) } else { None } +} +fn window_relex_step(old_text: &str, old_toks: &[AlignMeta], new_text: &str, start: usize, end: usize, ins: &str) -> (Vec, usize) { + let delta = ins.len() as isize - (end - start) as isize; + let edit_end = start + ins.len(); + let mut max_idx = None::; + for (i, t) in old_toks.iter().enumerate() { + if t.end < start { max_idx = Some(i); } else { break; } + } + let rb = max_idx.map(|i| i as isize - 1).unwrap_or(-1); + let mut out: Vec = if rb >= 0 { old_toks[..=rb as usize].to_vec() } else { Vec::new() }; + let mut scan_off = if rb >= 0 { old_toks[rb as usize].end } else { 0 }; + let mut pending_nl = false; + let mut scratch: Vec> = Vec::new(); + let mut relexed = 0usize; + while scan_off < new_text.len() { + let before = scratch.len(); + (scan_off, pending_nl) = lex_from(new_text, scan_off, pending_nl, &mut scratch, 1); + if scratch.len() == before { break; } + let t = &scratch[scratch.len() - 1]; + out.push(AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl }); + relexed += 1; + if t.off >= edit_end { + if let Some(o_idx) = find_tok_at_off(old_toks, (t.off as isize - delta) as usize) { + let o = &old_toks[o_idx]; + if o.kind == t.kind && o.end == (t.end as isize - delta) as usize && o.nl == t.nl && old_text[o.off..o.end] == new_text[t.off..t.end] { + for ot in &old_toks[o_idx + 1..] { + out.push(AlignMeta { kind: ot.kind, off: (ot.off as isize + delta) as usize, end: (ot.end as isize + delta) as usize, nl: ot.nl }); + } + return (out, relexed); + } + } + } + } + (out, relexed) +} +fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { + let fresh = to_meta(&lex(text)); + if fresh.len() != meta.len() { return false; } + for (f, m) in fresh.iter().zip(meta.iter()) { + if f.kind != m.kind || f.off != m.off || f.end != m.end || f.nl != m.nl { return false; } + if text[f.off..f.end] != text[m.off..m.end] { return false; } + } + true +} +` : ` +fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool { + let fresh = to_meta(&lex(text)); + if fresh.len() != meta.len() { return false; } + for (f, m) in fresh.iter().zip(meta.iter()) { + if f.kind != m.kind || f.off != m.off || f.end != m.end || f.nl != m.nl { return false; } + if text[f.off..f.end] != text[m.off..m.end] { return false; } + } + true +} +`; + const editBody = stateless + ? ` let mut cur_text = self.text.clone(); + let mut cur_toks = self.toks.clone(); + for e in edits { + let step_old_text = cur_text.clone(); + let step_old_toks = cur_toks.clone(); + let n = cur_text.len(); + let start = e.start.min(n); + let end = e.end.max(start).min(n); + let ins = e.text.clone(); + cur_text = format!("{}{}{}", &cur_text[..start], ins, &cur_text[end..]); + let (toks, step_relexed) = window_relex_step(&step_old_text, &step_old_toks, &cur_text, start, end, &ins); + cur_toks = toks; + relexed += step_relexed; + } + self.text = cur_text; + self.toks = cur_toks;` + : ` for e in edits { + let n = self.text.len(); + let start = e.start.min(n); + let end = e.end.max(start).min(n); + self.text = format!("{}{}{}", &self.text[..start], e.text, &self.text[end..]); + } + self.toks = to_meta(&lex(&self.text)); + relexed = self.toks.len();`; + return `pub struct Edit { pub start: usize, pub end: usize, pub text: String } +#[derive(Clone)] +struct AlignMeta { kind: &'static str, off: usize, end: usize, nl: bool } +struct Align { old_n: usize, new_n: usize, prefix: usize, suffix: usize, relexed: usize, stream_eq: bool } +fn to_meta(toks: &[Tok<'_>]) -> Vec { + toks.iter().map(|t| AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl }).collect() +} +fn compute_align_core(old_text: &str, old_toks: &[AlignMeta], new_text: &str, new_toks: &[AlignMeta]) -> (usize, usize, usize, usize) { + let old_n = old_toks.len(); + let new_n = new_toks.len(); + let mut prefix = 0usize; + while prefix < old_n && prefix < new_n { + let o = &old_toks[prefix]; + let n = &new_toks[prefix]; + if o.kind != n.kind || o.off != n.off || o.end != n.end || o.nl != n.nl { break; } + if old_text[o.off..o.end] != new_text[n.off..n.end] { break; } + prefix += 1; + } + let delta = new_text.len() as isize - old_text.len() as isize; + let min_n = old_n.min(new_n); + let mut suffix = 0usize; + while prefix + suffix < min_n { + let o = &old_toks[old_n - 1 - suffix]; + let n = &new_toks[new_n - 1 - suffix]; + if o.kind != n.kind || o.nl != n.nl { break; } + if n.off != (o.off as isize + delta) as usize || n.end != (o.end as isize + delta) as usize { break; } + if old_text[o.off..o.end] != new_text[n.off..n.end] { break; } + suffix += 1; + } + (old_n, new_n, prefix, suffix) +} +fn toks_from_meta<'a>(text: &'a str, meta: &[AlignMeta]) -> Vec> { + meta.iter().map(|m| Tok { kind: m.kind, text: &text[m.off..m.end], off: m.off, end: m.end, nl: m.nl }).collect() +} +${windowHelpers}pub struct Doc { text: String, toks: Vec, align: Option } +impl Doc { + pub fn new(text: String) -> Doc { Doc { text: text.clone(), toks: to_meta(&lex(&text)), align: None } } + pub fn text(&self) -> &str { &self.text } + pub fn alignment(&self) -> Option<&Align> { self.align.as_ref() } + pub fn edit(&mut self, edits: &[Edit]) { + let old_text = self.text.clone(); + let old_toks = self.toks.clone(); + let mut relexed = 0usize; +${editBody} + let stream_eq = check_stream_eq(&self.text, &self.toks); + let (old_n, new_n, prefix, suffix) = compute_align_core(&old_text, &old_toks, &self.text, &self.toks); + self.align = Some(Align { old_n, new_n, prefix, suffix, relexed, stream_eq }); + } + pub fn parse(&self) -> Option<(Parser<'_>, i32)> { + let toks = toks_from_meta(&self.text, &self.toks); + let n = toks.len(); + let mut p = Parser { toks, pos: 0, capped: false, suppress_next: Vec::new(), suppress_cur: Vec::new(), src: &self.text, nodes: Vec::new(), kids: Vec::new(), scratch: Vec::new() }; + match p.parse_${ir.entry}() { + Some(root) if p.pos == n => Some((p, root)), + _ => None, + } + } +}`; +} + export const rustTarget: Target = { name: 'rust', ext: 'rs', @@ -547,67 +715,7 @@ fn parse<'a>(tokens: Tokens<'a>) -> Option<(Parser<'a>, i32)> { } } -pub struct Edit { pub start: usize, pub end: usize, pub text: String } -#[derive(Clone)] -struct AlignMeta { kind: &'static str, off: usize, end: usize, nl: bool } -struct Align { old_n: usize, new_n: usize, prefix: usize, suffix: usize } -fn to_meta(toks: &[Tok<'_>]) -> Vec { - toks.iter().map(|t| AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl }).collect() -} -fn compute_align(old_text: &str, old_toks: &[AlignMeta], new_text: &str, new_toks: &[AlignMeta]) -> Align { - let old_n = old_toks.len(); - let new_n = new_toks.len(); - let mut prefix = 0usize; - while prefix < old_n && prefix < new_n { - let o = &old_toks[prefix]; - let n = &new_toks[prefix]; - if o.kind != n.kind || o.off != n.off || o.end != n.end || o.nl != n.nl { break; } - if old_text[o.off..o.end] != new_text[n.off..n.end] { break; } - prefix += 1; - } - let delta = new_text.len() as isize - old_text.len() as isize; - let min_n = old_n.min(new_n); - let mut suffix = 0usize; - while prefix + suffix < min_n { - let o = &old_toks[old_n - 1 - suffix]; - let n = &new_toks[new_n - 1 - suffix]; - if o.kind != n.kind || o.nl != n.nl { break; } - if n.off != (o.off as isize + delta) as usize || n.end != (o.end as isize + delta) as usize { break; } - if old_text[o.off..o.end] != new_text[n.off..n.end] { break; } - suffix += 1; - } - Align { old_n, new_n, prefix, suffix } -} -fn toks_from_meta<'a>(text: &'a str, meta: &[AlignMeta]) -> Vec> { - meta.iter().map(|m| Tok { kind: m.kind, text: &text[m.off..m.end], off: m.off, end: m.end, nl: m.nl }).collect() -} -pub struct Doc { text: String, toks: Vec, align: Option } -impl Doc { - pub fn new(text: String) -> Doc { Doc { text: text.clone(), toks: to_meta(&lex(&text)), align: None } } - pub fn text(&self) -> &str { &self.text } - pub fn alignment(&self) -> Option<&Align> { self.align.as_ref() } - pub fn edit(&mut self, edits: &[Edit]) { - let old_text = self.text.clone(); - let old_toks = self.toks.clone(); - for e in edits { - let n = self.text.len(); - let start = e.start.min(n); - let end = e.end.max(start).min(n); - self.text = format!("{}{}{}", &self.text[..start], e.text, &self.text[end..]); - } - self.toks = to_meta(&lex(&self.text)); - self.align = Some(compute_align(&old_text, &old_toks, &self.text, &self.toks)); - } - pub fn parse(&self) -> Option<(Parser<'_>, i32)> { - let toks = toks_from_meta(&self.text, &self.toks); - let n = toks.len(); - let mut p = Parser { toks, pos: 0, capped: false, suppress_next: Vec::new(), suppress_cur: Vec::new(), src: &self.text, nodes: Vec::new(), kids: Vec::new(), scratch: Vec::new() }; - match p.parse_${ir.entry}() { - Some(root) if p.pos == n => Some((p, root)), - _ => None, - } - } -} +${docEditBlockRust(ir)} `; }, emitRunner(): string { @@ -717,7 +825,7 @@ fn main() { doc.edit(&edits); } if let Some(a) = doc.alignment() { - eprintln!("{{\\"oldN\\":{},\\"newN\\":{},\\"prefix\\":{},\\"suffix\\":{}}}", a.old_n, a.new_n, a.prefix, a.suffix); + eprintln!("{{\\"oldN\\":{},\\"newN\\":{},\\"prefix\\":{},\\"suffix\\":{},\\"relexed\\":{},\\"streamEq\\":{}}}", a.old_n, a.new_n, a.prefix, a.suffix, a.relexed, a.stream_eq); } match doc.parse() { Some((p, root)) => { diff --git a/src/target-ts.ts b/src/target-ts.ts index 08a811e..08af15a 100644 --- a/src/target-ts.ts +++ b/src/target-ts.ts @@ -208,18 +208,34 @@ ${pushHooks} toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = }; ` : ' const push = (kind: string, text: string, off: number, end: number) => { toks.push({ kind, text, off, end, nl: pendingNl }); pendingNl = false; };\n'; - return `${defs.length ? 'let _s = "";\n' + defs.join('\n') + '\n' : ''}function lex(src: string): Tok[] { + const loopBody = `${nlBoundary} const c = src.charCodeAt(pos); + // JS line terminators LF/CR/LS/PS set newline-before, matching the interpreter (gen-lexer.ts). +${nlWs}${tplDispatch}${toks} +${puncts} + throw new Error('lex error at ' + pos + ': ' + JSON.stringify(src[pos]));`; + if (stateful) { + return `${defs.length ? 'let _s = "";\n' + defs.join('\n') + '\n' : ''}function lex(src: string): Tok[] { const toks: Tok[] = []; const n = src.length; let pos = 0; let pendingNl = false; -${defs.length ? ' _s = src;\n' : ''}${rxState}${tplState}${nlState}${stateful ? emitFn : pushFnDef} while (pos < n) { -${nlBoundary} const c = src.charCodeAt(pos); - // JS line terminators LF/CR/LS/PS set newline-before, matching the interpreter (gen-lexer.ts). -${nlWs}${tplDispatch}${toks} -${puncts} - throw new Error('lex error at ' + pos + ': ' + JSON.stringify(src[pos])); +${defs.length ? ' _s = src;\n' : ''}${rxState}${tplState}${nlState}${emitFn} while (pos < n) { +${loopBody} + } + return toks; +}`; + } + return `${defs.length ? 'let _s = "";\n' + defs.join('\n') + '\n' : ''}function lexFrom(src: string, pos: number, pendingNl: boolean, toks: Tok[], limit?: number): { pos: number; pendingNl: boolean } { + const n = src.length; + const base = toks.length; +${defs.length ? ' _s = src;\n' : ''}${pushFnDef} while (pos < n && (limit === undefined || toks.length - base < limit)) { +${loopBody} } + return { pos, pendingNl }; +} +function lex(src: string): Tok[] { + const toks: Tok[] = []; + lexFrom(src, 0, false, toks); return toks; }`; } @@ -353,6 +369,137 @@ ${r.nudSeqs.map((seq) => ` { const save = pos; const kids: Cst[] = []; if (${se }`; } +function docEditBlock(ir: ParserIR): string { + const stateless = !(ir.regexCtx || ir.tpl || ir.newlineCfg); + const windowHelpers = stateless ? ` +function findTokAtOff(toks: AlignMeta[], off: number): number { + let lo = 0, hi = toks.length - 1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + if (toks[mid].off < off) lo = mid + 1; + else if (toks[mid].off > off) hi = mid - 1; + else return mid; + } + return -1; +} +function windowRelexStep(oldText: string, oldToks: AlignMeta[], newText: string, start: number, end: number, ins: string): { toks: AlignMeta[]; relexed: number } { + const delta = ins.length - (end - start); + const editEnd = start + ins.length; + let maxIdx = -1; + for (let i = 0; i < oldToks.length; i++) { + if (oldToks[i].end < start) maxIdx = i; + else break; + } + const rb = maxIdx >= 0 ? maxIdx - 1 : -1; + const out: AlignMeta[] = rb >= 0 ? oldToks.slice(0, rb + 1) : []; + let scanOff = rb >= 0 ? oldToks[rb].end : 0; + let pendingNl = false; + const scratch: Tok[] = []; + let relexed = 0; + while (scanOff < newText.length) { + const before = scratch.length; + ({ pos: scanOff, pendingNl } = lexFrom(newText, scanOff, pendingNl, scratch, 1)); + if (scratch.length === before) break; + const t = scratch[scratch.length - 1]; + out.push({ kind: t.kind, off: t.off, end: t.end, nl: t.nl }); + relexed++; + if (t.off >= editEnd) { + const oIdx = findTokAtOff(oldToks, t.off - delta); + if (oIdx >= 0) { + const o = oldToks[oIdx]; + if (o.kind === t.kind && o.end === t.end - delta && o.nl === t.nl && oldText.slice(o.off, o.end) === newText.slice(t.off, t.end)) { + for (let j = oIdx + 1; j < oldToks.length; j++) { + const ot = oldToks[j]; + out.push({ kind: ot.kind, off: ot.off + delta, end: ot.end + delta, nl: ot.nl }); + } + return { toks: out, relexed }; + } + } + } + } + return { toks: out, relexed }; +} +` : ''; + const editBody = stateless + ? ` let curText = text; + let curToks = oldToks; + for (const e of edits) { + const stepOldText = curText; + const stepOldToks = curToks; + const n = curText.length, start = Math.max(0, Math.min(e.start, n)), end = Math.max(start, Math.min(e.end, n)); + const ins = e.text; + curText = curText.slice(0, start) + ins + curText.slice(end); + const wr = windowRelexStep(stepOldText, stepOldToks, curText, start, end, ins); + curToks = wr.toks; + relexed += wr.relexed; + } + text = curText; + prevToks = curToks;` + : ` for (const e of edits) { + const n = text.length, start = Math.max(0, Math.min(e.start, n)), end = Math.max(start, Math.min(e.end, n)); + text = text.slice(0, start) + e.text + text.slice(end); + } + prevToks = toMeta(tokenize(text)); + relexed = prevToks.length;`; + return `export type Edit = { start: number; end: number; text: string }; +type AlignMeta = { kind: string; off: number; end: number; nl: boolean }; +type Align = { oldN: number; newN: number; prefix: number; suffix: number; relexed: number; streamEq: boolean }; +const toMeta = (toks: Tok[]): AlignMeta[] => toks.map((t) => ({ kind: t.kind, off: t.off, end: t.end, nl: t.nl })); +function computeAlign(oldText: string, oldToks: AlignMeta[], newText: string, newToks: AlignMeta[]): Omit { + const oldN = oldToks.length, newN = newToks.length; + let prefix = 0; + while (prefix < oldN && prefix < newN) { + const o = oldToks[prefix], n = newToks[prefix]; + if (o.kind !== n.kind || o.off !== n.off || o.end !== n.end || o.nl !== n.nl) break; + if (oldText.slice(o.off, o.end) !== newText.slice(n.off, n.end)) break; + prefix++; + } + const delta = newText.length - oldText.length; + const minN = Math.min(oldN, newN); + let suffix = 0; + while (prefix + suffix < minN) { + const o = oldToks[oldN - 1 - suffix], n = newToks[newN - 1 - suffix]; + if (o.kind !== n.kind || o.nl !== n.nl || n.off !== o.off + delta || n.end !== o.end + delta) break; + if (oldText.slice(o.off, o.end) !== newText.slice(n.off, n.end)) break; + suffix++; + } + return { oldN, newN, prefix, suffix }; +} +function toksFromMeta(text: string, meta: AlignMeta[]): Tok[] { + return meta.map((m) => ({ kind: m.kind, text: text.slice(m.off, m.end), off: m.off, end: m.end, nl: m.nl })); +} +function checkStreamEq(text: string, meta: AlignMeta[]): boolean { + const fresh = toMeta(tokenize(text)); + if (fresh.length !== meta.length) return false; + for (let i = 0; i < fresh.length; i++) { + const f = fresh[i], t = meta[i]; + if (f.kind !== t.kind || f.off !== t.off || f.end !== t.end || f.nl !== t.nl) return false; + if (text.slice(f.off, f.end) !== text.slice(t.off, t.end)) return false; + } + return true; +} +${windowHelpers}export function createDoc(src: string): { text(): string; root(): Node | null; align(): Align | null; edit(edits: Edit[]): Node | null } { + let text = src; + let prevToks = toMeta(tokenize(src)); + let align: Align | null = null; + let root: Node | null = parse(tokenize(src)); + return { + text(): string { return text; }, + root(): Node | null { return root; }, + align(): Align | null { return align; }, + edit(edits: Edit[]): Node | null { + const oldText = text, oldToks = prevToks; + let relexed = 0; +${editBody} + const streamEq = checkStreamEq(text, prevToks); + align = { ...computeAlign(oldText, oldToks, text, prevToks), relexed, streamEq }; + root = parse(toksFromMeta(text, prevToks)); + return root; + }, + }; +}`; +} + export const tsTarget: Target = { name: 'typescript', ext: 'ts', @@ -472,53 +619,7 @@ export function parse(tokens: Tok[]): Cst | null { return root !== null && pos === toks.length ? root : null; } -export type Edit = { start: number; end: number; text: string }; -type AlignMeta = { kind: string; off: number; end: number; nl: boolean }; -type Align = { oldN: number; newN: number; prefix: number; suffix: number }; -const toMeta = (toks: Tok[]): AlignMeta[] => toks.map((t) => ({ kind: t.kind, off: t.off, end: t.end, nl: t.nl })); -function computeAlign(oldText: string, oldToks: AlignMeta[], newText: string, newToks: AlignMeta[]): Align { - const oldN = oldToks.length, newN = newToks.length; - let prefix = 0; - while (prefix < oldN && prefix < newN) { - const o = oldToks[prefix], n = newToks[prefix]; - if (o.kind !== n.kind || o.off !== n.off || o.end !== n.end || o.nl !== n.nl) break; - if (oldText.slice(o.off, o.end) !== newText.slice(n.off, n.end)) break; - prefix++; - } - const delta = newText.length - oldText.length; - const minN = Math.min(oldN, newN); - let suffix = 0; - while (prefix + suffix < minN) { - const o = oldToks[oldN - 1 - suffix], n = newToks[newN - 1 - suffix]; - if (o.kind !== n.kind || o.nl !== n.nl || n.off !== o.off + delta || n.end !== o.end + delta) break; - if (oldText.slice(o.off, o.end) !== newText.slice(n.off, n.end)) break; - suffix++; - } - return { oldN, newN, prefix, suffix }; -} -export function createDoc(src: string): { text(): string; root(): Node | null; align(): Align | null; edit(edits: Edit[]): Node | null } { - let text = src; - let prevToks = toMeta(tokenize(src)); - let align: Align | null = null; - let root: Node | null = parse(tokenize(src)); - return { - text(): string { return text; }, - root(): Node | null { return root; }, - align(): Align | null { return align; }, - edit(edits: Edit[]): Node | null { - const oldText = text, oldToks = prevToks; - for (const e of edits) { - const n = text.length, start = Math.max(0, Math.min(e.start, n)), end = Math.max(start, Math.min(e.end, n)); - text = text.slice(0, start) + e.text + text.slice(end); - } - const newToks = tokenize(text); - prevToks = toMeta(newToks); - align = computeAlign(oldText, oldToks, text, prevToks); - root = parse(newToks); - return root; - }, - }; -} +${docEditBlock(ir)} `; }, emitRunner(): string { diff --git a/test/portable-targets.ts b/test/portable-targets.ts index a11df1c..7e594a1 100644 --- a/test/portable-targets.ts +++ b/test/portable-targets.ts @@ -271,7 +271,7 @@ function runProc(cmd: string, args: string[], src: string): Outcome { try { return { ok: true, cst: canon(JSON.parse(execFileSync(cmd, args, { input: src, stdio: ['pipe', 'pipe', 'pipe'] }).toString())) }; } catch { return { ok: false }; } } -type Align = { oldN: number; newN: number; prefix: number; suffix: number }; +type Align = { oldN: number; newN: number; prefix: number; suffix: number; relexed?: number; streamEq?: boolean }; type EditOutcome = Outcome & { align?: Align }; function parseAlignStderr(stderr: string): Align | undefined { for (const line of stderr.split('\n')) { @@ -325,21 +325,28 @@ const expectAlign = (g: CstGrammar, init: string, batches: EditBatch[]): Align = }; type EditBatch = [number, number, string][]; -type EditScenario = { init: string; batches: EditBatch[]; large?: boolean }; +type EditScenario = { init: string; batches: EditBatch[]; large?: boolean; maxRelexed?: number; fullRelex?: boolean }; +const calcLargeInit = '1+2*3+'.repeat(199) + '1+2*3'; const EDIT_SCENARIOS: Record = { calc: [ { init: '1+2*3', batches: [[[3, 3, '4']]] }, { init: '1+2*3', batches: [[[1, 3, '']]] }, { init: '1+2*3', batches: [[[2, 5, '(7-8)']]] }, { init: '1+2*3', batches: [[[0, 0, '9-']], [[7, 8, '']]] }, - { init: '1+2*3+'.repeat(199) + '1+2*3', batches: [[[600, 601, '9']]], large: true }, + { init: calcLargeInit, batches: [[[600, 601, '9']]], large: true, maxRelexed: 8 }, + { init: '1 2;', batches: [[[1, 2, '']]] }, + { init: calcLargeInit, batches: [[[calcLargeInit.length, calcLargeInit.length, '+4']]], maxRelexed: 5 }, + { init: calcLargeInit, batches: [[[0, 1, '9']]], maxRelexed: 8 }, ], javascript: [ - { init: 'let a = 1;\nf(a);', batches: [[[8, 9, '42']]] }, - { init: 'let a = 1;\nf(a);', batches: [[[11, 16, '']]] }, - { init: 'let a = 1;\nf(a);', batches: [[[4, 5, 'b'], [12, 13, 'b']]] }, - { init: 'let a = 1;\nf(a);', batches: [[[16, 16, '\ng(b);']], [[0, 4, 'var']]] }, - { init: 'let a = 1;\nf(a);\n'.repeat(80), batches: [[[688, 689, '9']]], large: true }, + { init: 'let a = 1;\nf(a);', batches: [[[8, 9, '42']]], fullRelex: true }, + { init: 'let a = 1;\nf(a);', batches: [[[11, 16, '']]], fullRelex: true }, + { init: 'let a = 1;\nf(a);', batches: [[[4, 5, 'b'], [12, 13, 'b']]], fullRelex: true }, + { init: 'let a = 1;\nf(a);', batches: [[[16, 16, '\ng(b);']], [[0, 4, 'var']]], fullRelex: true }, + { init: 'let a = 1;\nf(a);\n'.repeat(80), batches: [[[688, 689, '9']]], large: true, fullRelex: true }, + ], + envspec: [ + { init: 'A=1\nB=2', batches: [[[2, 2, '3']]], fullRelex: true }, ], }; const applyEdits = (init: string, batches: EditBatch[]): string => { @@ -433,6 +440,18 @@ for (const c of CASES) { failures++; console.log(` ${c.grammar}/${r.label}: large-doc align too narrow prefix+suffix=${want.prefix + want.suffix} oldN=${want.oldN}`); } + if (a.align.streamEq !== true) { + failures++; + console.log(` ${c.grammar}/${r.label}: streamEq want=true got=${JSON.stringify(a.align.streamEq)}`); + } + if (sc.maxRelexed !== undefined && (a.align.relexed === undefined || a.align.relexed > sc.maxRelexed)) { + failures++; + console.log(` ${c.grammar}/${r.label}: relexed want<=${sc.maxRelexed} got=${JSON.stringify(a.align.relexed)}`); + } + if (sc.fullRelex && a.align.relexed !== want.newN) { + failures++; + console.log(` ${c.grammar}/${r.label}: fullRelex want relexed=${want.newN} got=${JSON.stringify(a.align.relexed)}`); + } } else { failures++; console.log(` ${c.grammar}/${r.label}: token-align mismatch want=${JSON.stringify(want)} got=${JSON.stringify(a.align)}`);