fix: Fix memory issues with large diffs

package.json

@@ -25,10 +25,12 @@
     "@babel/preset-react": "^7.27.1",
     "@babel/preset-typescript": "^7.27.1",
     "@types/chrome": "^0.1.38",
+    "@types/diff": "^8.0.0",
     "@types/html-to-text": "^9.0.4",
     "babel-loader": "^10.0.0",
     "copy-webpack-plugin": "^13.0.0",
     "css-loader": "^7.1.2",
+    "diff": "^9.0.0",
     "fast-xml-parser": "^5.2.3",
     "fuse.js": "^7.1.0",
     "html-to-text": "^9.0.5",

src/background/events/injects/display_history.ts

@@ -1,28 +1,28 @@
-import { Module } from "../../../types/module";
+import { Module } from "../../../types/module"
 
-export default <Module> {
+export default <Module>{
     setting: s => {
         return !!s.recents_list_module
     },
     condition: async (base, settings, helper_fns) => {
-       return (await helper_fns.url_begins_with("/news"))
+        return await helper_fns.url_begins_with("/news")
     },
     action: async (base, settings, helper_fns) => {
         await chrome.scripting.executeScript({
-            target: {tabId: base.tab_id},
-            files: ["display_history.js"]
+            target: { tabId: base.tab_id },
+            files: ["display_history.js"],
         })
 
         await chrome.scripting.insertCSS({
-            target: {tabId: base.tab_id},
-            files: ["news_history.css"]
+            target: { tabId: base.tab_id },
+            files: ["news_history.css"],
         })
 
         if (settings.schooltape_compatibility) {
             await chrome.scripting.insertCSS({
-                target: {tabId: base.tab_id},
-                files: ["schooltape/post_history_styles.css"]
+                target: { tabId: base.tab_id },
+                files: ["schooltape/post_history_styles.css"],
             })
         }
-    }
-}
+    },
+}

src/background/functions/calculate_rev_metrics.ts

@@ -1,85 +1,24 @@
 import { RevisionData } from "../../types/rev_history"
+import { diffChars } from "diff"
 
 export async function calculate_new_metrics(old_data: RevisionData, new_data: RevisionData) {
-    const old_title = old_data.title || ""
-    const new_title = new_data.title || ""
+    let added_chars = 0
+    let removed_chars = 0
+    let unchanged_chars = 0
 
-    let new_lines = 0
-    let modified_lines = 0
-    let deleted_lines = 0
+    const changes = diffChars(old_data.content.toLowerCase(), new_data.content.toLowerCase())
 
-    if (old_title != new_title) {
-        modified_lines++
-    }
-
-    // split into individual chars
-    const old_text = [...old_data.content.toLowerCase()]
-    const new_text = [...new_data.content.toLowerCase()]
-
-    const m = old_text.length
-    const n = new_text.length
-
-    // create a DP table of size (m+1) x (n+1)
-    // dp[i][j] represents the edit distance between old_text[0..i-1] and new_text[0..j-1]
-    const dp: number[][] = Array.from({ length: m + 1 }, () => Array(n + 1).fill(0))
-
-    // base cases
-    for (let i = 0; i <= m; i++) {
-        dp[i][0] = i // deleting all lines from old_text
-    }
-    for (let j = 0; j <= n; j++) {
-        dp[0][j] = j // adding all lines from new_text
-    }
-
-    // build the DP table
-    for (let i = 1; i <= m; i++) {
-        for (let j = 1; j <= n; j++) {
-            if (old_text[i - 1] === new_text[j - 1]) {
-                dp[i][j] = dp[i - 1][j - 1] // no change
-            } else {
-                dp[i][j] = Math.min(
-                    dp[i - 1][j] + 1, // delete old line
-                    dp[i][j - 1] + 1, // add new line
-                    dp[i - 1][j - 1] + 1 // modify line
-                )
-            }
-        }
-    }
-
-    // backtrack to count the number of new, modified, and deleted lines
-    let i = m
-    let j = n
-    while (i > 0 && j > 0) {
-        if (old_text[i - 1] === new_text[j - 1]) {
-            // if lines are the same, move diagonally
-            i--
-            j--
-        } else if (dp[i][j] === dp[i - 1][j] + 1) {
-            // line deleted from old_text
-            deleted_lines++
-            i--
-        } else if (dp[i][j] === dp[i][j - 1] + 1) {
-            // line added in new_text
-            new_lines++
-            j--
+    for (const part of changes) {
+        if (part.added) {
+            added_chars += part.count || 0
+        } else if (part.removed) {
+            removed_chars += part.count || 0
         } else {
-            modified_lines++
-            i--
-            j--
+            unchanged_chars += part.count || 0
         }
     }
 
-    // if there are remaining lines in old_text, they are deleted
-    while (i > 0) {
-        deleted_lines++
-        i--
-    }
-
-    // if there are remaining lines in new_text, they are added
-    while (j > 0) {
-        new_lines++
-        j--
-    }
-
-    return { new_lines, modified_lines, deleted_lines }
+    // A "modification" at the character level isn't a standard metric,
+    // it's usually just represented as a removal followed by an addition.
+    return { added_chars, removed_chars, unchanged_chars }
 }

src/background/pull_feed.ts

@@ -65,7 +65,7 @@ async function append_revision(
     }
 
     const now = Date.now()
-    const { new_lines, modified_lines, deleted_lines } = await calculate_new_metrics(
+    const { added_chars, removed_chars, unchanged_chars } = await calculate_new_metrics(
         prev_data_obj,
         rev_object
     )
@@ -74,9 +74,9 @@ async function append_revision(
         guid,
         prev_data_obj,
         now,
-        new_lines,
-        modified_lines,
-        deleted_lines
+        added_chars,
+        removed_chars,
+        unchanged_chars
     )
 
     return data.rev_id