From 0328c1a3eacdc73285a2a7087135cca8fa4391c9 Mon Sep 17 00:00:00 2001 From: skid-dev <62094231+skid-dev@users.noreply.github.com> Date: Mon, 4 May 2026 17:13:30 +1000 Subject: [PATCH] fix: Fix memory issues with large diffs --- bun.lock | 6 ++ package.json | 2 + .../events/injects/display_history.ts | 22 ++--- .../functions/calculate_rev_metrics.ts | 89 +++---------------- src/background/pull_feed.ts | 8 +- 5 files changed, 37 insertions(+), 90 deletions(-) diff --git a/bun.lock b/bun.lock index 2b08466..38e8d78 100644 --- a/bun.lock +++ b/bun.lock @@ -10,10 +10,12 @@ "@babel/preset-react": "^7.27.1", "@babel/preset-typescript": "^7.27.1", "@types/chrome": "^0.1.38", + "@types/diff": "^8.0.0", "@types/html-to-text": "^9.0.4", "babel-loader": "^10.0.0", "copy-webpack-plugin": "^13.0.0", "css-loader": "^7.1.2", + "diff": "^9.0.0", "fast-xml-parser": "^5.2.3", "fuse.js": "^7.1.0", "html-to-text": "^9.0.5", @@ -315,6 +317,8 @@ "@types/chrome": ["@types/chrome@0.1.38", "", { "dependencies": { "@types/filesystem": "*", "@types/har-format": "*" } }, "sha512-5aK4m9wZqoWAoB98aElESLm/5pXpqJnFWMNoiCs/XdPsXR6wNdVkJFSdQ9Wr4PnTuUrxD0SuNuDHh3EG5QeBzA=="], + "@types/diff": ["@types/diff@8.0.0", "", { "dependencies": { "diff": "*" } }, "sha512-o7jqJM04gfaYrdCecCVMbZhNdG6T1MHg/oQoRFdERLV+4d+V7FijhiEAbFu0Usww84Yijk9yH58U4Jk4HbtzZw=="], + "@types/eslint": ["@types/eslint@9.6.1", "", { "dependencies": { "@types/estree": "*", "@types/json-schema": "*" } }, "sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag=="], "@types/eslint-scope": ["@types/eslint-scope@3.7.7", "", { "dependencies": { "@types/eslint": "*", "@types/estree": "*" } }, "sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg=="], @@ -505,6 +509,8 @@ "deepmerge": ["deepmerge@4.3.1", "", {}, "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A=="], + "diff": ["diff@9.0.0", "", {}, "sha512-svtcdpS8CgJyqAjEQIXdb3OjhFVVYjzGAPO8WGCmRbrml64SPw/jJD4GoE98aR7r25A0XcgrK3F02yw9R/vhQw=="], + "dir-glob": ["dir-glob@3.0.1", "", { "dependencies": { "path-type": "^4.0.0" } }, "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA=="], "dom-converter": ["dom-converter@0.2.0", "", { "dependencies": { "utila": "~0.4" } }, "sha512-gd3ypIPfOMr9h5jIKq8E3sHOTCjeirnl0WK5ZdS1AW0Odt0b1PaWaHdJ4Qk4klv+YB9aJBS7mESXjFoDQPu6DA=="], diff --git a/package.json b/package.json index c7debb8..bdcc767 100644 --- a/package.json +++ b/package.json @@ -25,10 +25,12 @@ "@babel/preset-react": "^7.27.1", "@babel/preset-typescript": "^7.27.1", "@types/chrome": "^0.1.38", + "@types/diff": "^8.0.0", "@types/html-to-text": "^9.0.4", "babel-loader": "^10.0.0", "copy-webpack-plugin": "^13.0.0", "css-loader": "^7.1.2", + "diff": "^9.0.0", "fast-xml-parser": "^5.2.3", "fuse.js": "^7.1.0", "html-to-text": "^9.0.5", diff --git a/src/background/events/injects/display_history.ts b/src/background/events/injects/display_history.ts index 0f30a97..31a7e5f 100644 --- a/src/background/events/injects/display_history.ts +++ b/src/background/events/injects/display_history.ts @@ -1,28 +1,28 @@ -import { Module } from "../../../types/module"; +import { Module } from "../../../types/module" -export default { +export default { setting: s => { return !!s.recents_list_module }, condition: async (base, settings, helper_fns) => { - return (await helper_fns.url_begins_with("/news")) + return await helper_fns.url_begins_with("/news") }, action: async (base, settings, helper_fns) => { await chrome.scripting.executeScript({ - target: {tabId: base.tab_id}, - files: ["display_history.js"] + target: { tabId: base.tab_id }, + files: ["display_history.js"], }) await chrome.scripting.insertCSS({ - target: {tabId: base.tab_id}, - files: ["news_history.css"] + target: { tabId: base.tab_id }, + files: ["news_history.css"], }) if (settings.schooltape_compatibility) { await chrome.scripting.insertCSS({ - target: {tabId: base.tab_id}, - files: ["schooltape/post_history_styles.css"] + target: { tabId: base.tab_id }, + files: ["schooltape/post_history_styles.css"], }) } - } -} \ No newline at end of file + }, +} diff --git a/src/background/functions/calculate_rev_metrics.ts b/src/background/functions/calculate_rev_metrics.ts index d919ac6..907e6b2 100644 --- a/src/background/functions/calculate_rev_metrics.ts +++ b/src/background/functions/calculate_rev_metrics.ts @@ -1,85 +1,24 @@ import { RevisionData } from "../../types/rev_history" +import { diffChars } from "diff" export async function calculate_new_metrics(old_data: RevisionData, new_data: RevisionData) { - const old_title = old_data.title || "" - const new_title = new_data.title || "" + let added_chars = 0 + let removed_chars = 0 + let unchanged_chars = 0 - let new_lines = 0 - let modified_lines = 0 - let deleted_lines = 0 + const changes = diffChars(old_data.content.toLowerCase(), new_data.content.toLowerCase()) - if (old_title != new_title) { - modified_lines++ - } - - // split into individual chars - const old_text = [...old_data.content.toLowerCase()] - const new_text = [...new_data.content.toLowerCase()] - - const m = old_text.length - const n = new_text.length - - // create a DP table of size (m+1) x (n+1) - // dp[i][j] represents the edit distance between old_text[0..i-1] and new_text[0..j-1] - const dp: number[][] = Array.from({ length: m + 1 }, () => Array(n + 1).fill(0)) - - // base cases - for (let i = 0; i <= m; i++) { - dp[i][0] = i // deleting all lines from old_text - } - for (let j = 0; j <= n; j++) { - dp[0][j] = j // adding all lines from new_text - } - - // build the DP table - for (let i = 1; i <= m; i++) { - for (let j = 1; j <= n; j++) { - if (old_text[i - 1] === new_text[j - 1]) { - dp[i][j] = dp[i - 1][j - 1] // no change - } else { - dp[i][j] = Math.min( - dp[i - 1][j] + 1, // delete old line - dp[i][j - 1] + 1, // add new line - dp[i - 1][j - 1] + 1 // modify line - ) - } - } - } - - // backtrack to count the number of new, modified, and deleted lines - let i = m - let j = n - while (i > 0 && j > 0) { - if (old_text[i - 1] === new_text[j - 1]) { - // if lines are the same, move diagonally - i-- - j-- - } else if (dp[i][j] === dp[i - 1][j] + 1) { - // line deleted from old_text - deleted_lines++ - i-- - } else if (dp[i][j] === dp[i][j - 1] + 1) { - // line added in new_text - new_lines++ - j-- + for (const part of changes) { + if (part.added) { + added_chars += part.count || 0 + } else if (part.removed) { + removed_chars += part.count || 0 } else { - modified_lines++ - i-- - j-- + unchanged_chars += part.count || 0 } } - // if there are remaining lines in old_text, they are deleted - while (i > 0) { - deleted_lines++ - i-- - } - - // if there are remaining lines in new_text, they are added - while (j > 0) { - new_lines++ - j-- - } - - return { new_lines, modified_lines, deleted_lines } + // A "modification" at the character level isn't a standard metric, + // it's usually just represented as a removal followed by an addition. + return { added_chars, removed_chars, unchanged_chars } } diff --git a/src/background/pull_feed.ts b/src/background/pull_feed.ts index 5696404..f1eca90 100644 --- a/src/background/pull_feed.ts +++ b/src/background/pull_feed.ts @@ -65,7 +65,7 @@ async function append_revision( } const now = Date.now() - const { new_lines, modified_lines, deleted_lines } = await calculate_new_metrics( + const { added_chars, removed_chars, unchanged_chars } = await calculate_new_metrics( prev_data_obj, rev_object ) @@ -74,9 +74,9 @@ async function append_revision( guid, prev_data_obj, now, - new_lines, - modified_lines, - deleted_lines + added_chars, + removed_chars, + unchanged_chars ) return data.rev_id