From 818c29ef94b7b5b825f8eb20d52676e572bdfaf6 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Wed, 10 Jun 2026 22:40:54 +0300 Subject: [PATCH 01/11] Make arrays from `__key` values --- lib/core.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/core.js b/lib/core.js index 3e3e77bb8..624d58ef2 100644 --- a/lib/core.js +++ b/lib/core.js @@ -887,6 +887,16 @@ if (!(key in context) || key === 'whitelistRecord') { context[key] = r.data[key]; hasNewData = true; + } else { + // Make arrays from `__key` values. + if (key.startsWith('__')) { + if (!Array.isArray(context[key])) { + // Init array from non array value. + context[key] = [context[key]]; + } + context[key].push(r.data[key]); + hasNewData = true; + } } } } From 268acc526ff38dd31bbce586c9a5808bc827d1c9 Mon Sep 17 00:00:00 2001 From: Ivan Paramonau Date: Fri, 12 Jun 2026 09:39:47 -0400 Subject: [PATCH 02/11] change getSources to getContent --- config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.js b/config.js index c4dd73afa..0e42e379a 100644 --- a/config.js +++ b/config.js @@ -372,7 +372,7 @@ GET_VARS_METHODS: { 'getSignals': 'signals', 'getPolicy': 'policy', - 'getSources': 'sources' + 'getContent': 'content', } }; From 2244a40b4513d202752dd1e91063ffca87e10d06 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Fri, 12 Jun 2026 16:58:16 +0300 Subject: [PATCH 03/11] using parameters destructuring for `findPluginMethods` --- lib/core.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/core.js b/lib/core.js index 624d58ef2..39e0df18f 100644 --- a/lib/core.js +++ b/lib/core.js @@ -38,7 +38,7 @@ /* * Recursively finds plugin methods to run, including 'mixins' and dependencies (up and down by tree). * */ - function findPluginMethods(pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams) { + function findPluginMethods({pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams}) { /* * Params: @@ -120,7 +120,7 @@ var mixins = plugin.module.mixins; if (mixins) { for(var i = 0; i < mixins.length; i++) { - findPluginMethods(mixins[i], loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains); + findPluginMethods({pluginId: mixins[i], loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains}); } } } @@ -186,7 +186,7 @@ // Find absent params in other plugins 'provides' attribute. for(var j = 0; j < pluginsId.length; j++) { var foundPluginId = pluginsId[j]; - findPluginMethods(foundPluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains); + findPluginMethods({pluginId: foundPluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains}); } } else { @@ -376,7 +376,7 @@ // Find methods in each required plugin. for(var i = 0; i < requiredPlugins.length; i++) { var plugin = requiredPlugins[i]; - findPluginMethods(plugin.id, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains); + findPluginMethods({pluginId: plugin.id, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains}); } // If has new unused params (mandatoryParams) - then find plugins which can use them. @@ -387,7 +387,7 @@ // // Find methods in plugins, which can use mandatory params. // for(var i = 0; i < secondaryPlugins.length; i++) { // var pluginId = secondaryPlugins[i]; - // findPluginMethods(pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams); + // findPluginMethods({pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams}); // } // } From 02787a6b92461dca0de21be2d4f8019be3d51548 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Fri, 12 Jun 2026 16:59:35 +0300 Subject: [PATCH 04/11] using parameters destructuring for `runPluginsIteration` --- lib/core.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/core.js b/lib/core.js index 39e0df18f..0b8992719 100644 --- a/lib/core.js +++ b/lib/core.js @@ -361,7 +361,7 @@ /* * Single iteration of run plugins wave. * */ - function runPluginsIteration(requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb) { + function runPluginsIteration({requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb}) { var loadedParams = Object.keys(context); @@ -2058,7 +2058,7 @@ // Run all available plugins again with new data. var hasRuns = 0; if (hasNewData || error === 'initial') { - hasRuns = runPluginsIteration(requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb); + hasRuns = runPluginsIteration({requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb}); } if (hasRuns === 0) { @@ -2098,7 +2098,7 @@ isDomainPluginsMode = pluginsSet.isDomainPluginsMode; // Recursive call with same context to prevent same data fetching. - hasRuns = runPluginsIteration(requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb); + hasRuns = runPluginsIteration({requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb}); if (hasRuns === 0) { finish(); From 63c68b6727297a94c5b69ad6c4286ef45982e101 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Fri, 12 Jun 2026 17:04:36 +0300 Subject: [PATCH 05/11] add `isVarMethod` --- lib/core.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/core.js b/lib/core.js index 0b8992719..2b718171b 100644 --- a/lib/core.js +++ b/lib/core.js @@ -35,6 +35,10 @@ cache }; + function isVarMethod(name) { + return name === "getVars" || CONFIG.GET_VARS_METHODS && name in CONFIG.GET_VARS_METHODS + } + /* * Recursively finds plugin methods to run, including 'mixins' and dependencies (up and down by tree). * */ @@ -191,6 +195,10 @@ } else { + if (isVarMethod(r.method.name)) { + + } + // This branch goes down by the tree. // Mark method used but not finished. @@ -962,7 +970,7 @@ } } } - } else if (r.method.name === "getVars" || CONFIG.GET_VARS_METHODS && r.method.name in CONFIG.GET_VARS_METHODS) { + } else if (isVarMethod(r.method.name)) { var result_data = r.data; From 3819c58935abed1dc6a85a564c266415d3987746 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Fri, 12 Jun 2026 17:10:44 +0300 Subject: [PATCH 06/11] simplify `mandatoryParams` recursive calls --- lib/core.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/core.js b/lib/core.js index 2b718171b..9bc2a710a 100644 --- a/lib/core.js +++ b/lib/core.js @@ -42,7 +42,8 @@ /* * Recursively finds plugin methods to run, including 'mixins' and dependencies (up and down by tree). * */ - function findPluginMethods({pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams}) { + function findPluginMethods(opts) { + const {pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams} = opts; /* * Params: @@ -124,7 +125,7 @@ var mixins = plugin.module.mixins; if (mixins) { for(var i = 0; i < mixins.length; i++) { - findPluginMethods({pluginId: mixins[i], loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains}); + findPluginMethods({...opts, pluginId: mixins[i]}); } } } @@ -190,7 +191,7 @@ // Find absent params in other plugins 'provides' attribute. for(var j = 0; j < pluginsId.length; j++) { var foundPluginId = pluginsId[j]; - findPluginMethods({pluginId: foundPluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains}); + findPluginMethods({...opts, pluginId: foundPluginId, mandatoryParams: undefined}); } } else { From f56e06caaa24243049eb6854d1acf6b0c6e52967 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Fri, 12 Jun 2026 18:26:20 +0300 Subject: [PATCH 07/11] last iteration workaround --- lib/core.js | 61 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/lib/core.js b/lib/core.js index 9bc2a710a..33dd8bc45 100644 --- a/lib/core.js +++ b/lib/core.js @@ -43,7 +43,7 @@ * Recursively finds plugin methods to run, including 'mixins' and dependencies (up and down by tree). * */ function findPluginMethods(opts) { - const {pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams} = opts; + const {pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, getVarsInfo, mandatoryParams} = opts; /* * Params: @@ -196,21 +196,24 @@ } else { - if (isVarMethod(r.method.name)) { + if (!getVarsInfo.lastIteration && isVarMethod(method)) { - } - - // This branch goes down by the tree. + getVarsInfo.pendingMethods = true; - // Mark method used but not finished. - usedPluginMethods[method] = 1; + } else { - // Method will run, store in result set. - methods.push({ - pluginId: pluginId, - name: method, - handle: plugin.module[method] - }); + // This branch goes down by the tree. + + // Mark method used but not finished. + usedPluginMethods[method] = 1; + + // Method will run, store in result set. + methods.push({ + pluginId: pluginId, + name: method, + handle: plugin.module[method] + }); + } } } } @@ -370,7 +373,7 @@ /* * Single iteration of run plugins wave. * */ - function runPluginsIteration({requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb}) { + function runPluginsIteration({requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, getVarsInfo, options, asyncMethodCb}) { var loadedParams = Object.keys(context); @@ -385,7 +388,7 @@ // Find methods in each required plugin. for(var i = 0; i < requiredPlugins.length; i++) { var plugin = requiredPlugins[i]; - findPluginMethods({pluginId: plugin.id, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains}); + findPluginMethods({pluginId: plugin.id, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, getVarsInfo}); } // If has new unused params (mandatoryParams) - then find plugins which can use them. @@ -1892,6 +1895,8 @@ usedDomains = pluginsSet.usedDomains, isDomainPluginsMode = pluginsSet.isDomainPluginsMode, + getVarsInfo = {}, + pluginsContexts = {}, allResults = { @@ -2066,8 +2071,19 @@ // Run all available plugins again with new data. var hasRuns = 0; - if (hasNewData || error === 'initial') { - hasRuns = runPluginsIteration({requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb}); + var runOpts = { + requiredPlugins, + context, + pluginsUrlMatches, + usedMethods, + usedParams, + usedDomains, + getVarsInfo, + options, + asyncMethodCb + }; + if (hasNewData || error === 'initial' || error === 'lastIteration') { + hasRuns = runPluginsIteration(runOpts); } if (hasRuns === 0) { @@ -2083,6 +2099,14 @@ } } + if (getVarsInfo.pendingMethods) { + // Make last iteration. All `getVars` argumenets must be loaded at this moment. + getVarsInfo.pendingMethods = false; + getVarsInfo.lastIteration = true; + // Run last iteration. + return asyncMethodCb('lastIteration'); + } + // If no data from domain plugins - try fallback to generic plugins. if (!options.mixAllWithDomainPlugin && isDomainPluginsMode && !resultsHasDomainData(requiredPlugins, allResults.allData)) { @@ -2105,9 +2129,10 @@ pluginsUrlMatches = pluginsSet.pluginsUrlMatches; usedDomains = pluginsSet.usedDomains; // Will be null. isDomainPluginsMode = pluginsSet.isDomainPluginsMode; + getVarsInfo = {}; // Also reset to restart process. // Recursive call with same context to prevent same data fetching. - hasRuns = runPluginsIteration({requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb}); + hasRuns = runPluginsIteration({requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, getVarsInfo, options, asyncMethodCb}); if (hasRuns === 0) { finish(); From f2ec6631bf655e2a29953a9bc8f4920624e2d4b6 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Fri, 12 Jun 2026 18:38:07 +0300 Subject: [PATCH 08/11] debug log --- lib/core.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/core.js b/lib/core.js index 33dd8bc45..02ad8af2c 100644 --- a/lib/core.js +++ b/lib/core.js @@ -338,6 +338,7 @@ } // Call method. + // console.log('-- run method:', method.pluginId, method.name) var result = method.handle.apply(plugin.module, args); } catch(ex) { From 367b95578b6be18dfeeb500ca73384692daa5ac6 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Fri, 12 Jun 2026 21:30:21 +0300 Subject: [PATCH 09/11] do not duplicate values in data array --- lib/core.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/core.js b/lib/core.js index 02ad8af2c..04895ab0a 100644 --- a/lib/core.js +++ b/lib/core.js @@ -895,10 +895,12 @@ allResults.h2 = r.data.htmlparser.h2; } + const dataValue = r.data[key]; + // First data has priority, do not override it. // whitelistRecord - exception for oembed plugin. if (!(key in context) || key === 'whitelistRecord') { - context[key] = r.data[key]; + context[key] = dataValue; hasNewData = true; } else { // Make arrays from `__key` values. @@ -907,7 +909,10 @@ // Init array from non array value. context[key] = [context[key]]; } - context[key].push(r.data[key]); + // Do not insert existing value. + if (!context[key].includes(dataValue)) { + context[key].push(dataValue); + } hasNewData = true; } } From bb2a7fa0f048277a445df500cc29ac8006c5743b Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Fri, 12 Jun 2026 21:30:32 +0300 Subject: [PATCH 10/11] better function name `isGetVarsMethod` --- lib/core.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/core.js b/lib/core.js index 04895ab0a..37f0f3be3 100644 --- a/lib/core.js +++ b/lib/core.js @@ -35,7 +35,7 @@ cache }; - function isVarMethod(name) { + function isGetVarsMethod(name) { return name === "getVars" || CONFIG.GET_VARS_METHODS && name in CONFIG.GET_VARS_METHODS } @@ -196,7 +196,7 @@ } else { - if (!getVarsInfo.lastIteration && isVarMethod(method)) { + if (!getVarsInfo.lastIteration && isGetVarsMethod(method)) { getVarsInfo.pendingMethods = true; @@ -980,7 +980,7 @@ } } } - } else if (isVarMethod(r.method.name)) { + } else if (isGetVarsMethod(r.method.name)) { var result_data = r.data; From f7f4789db6d4d38f47114e4bd772b0196074fe12 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Mon, 15 Jun 2026 16:24:09 +0300 Subject: [PATCH 11/11] for __data values allow join arrays --- lib/core.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/core.js b/lib/core.js index 37f0f3be3..c57967d09 100644 --- a/lib/core.js +++ b/lib/core.js @@ -909,11 +909,18 @@ // Init array from non array value. context[key] = [context[key]]; } - // Do not insert existing value. - if (!context[key].includes(dataValue)) { + if (Array.isArray(dataValue)) { + // Join arrays without duplicating existing items. + const uniqueValues = dataValue.filter(value => !context[key].includes(value)); + if (uniqueValues.length) { + context[key] = context[key].concat(uniqueValues); + hasNewData = true; + } + } else if (!context[key].includes(dataValue)) { + // Do not insert existing value. context[key].push(dataValue); + hasNewData = true; } - hasNewData = true; } } }