Skip to content

Put getVars processing after all iterations finished #673

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nleush wants to merge 11 commits into
base: dev
Choose a base branch
from
Open

Put getVars processing after all iterations finished #673

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
818c29e
Make arrays from `__key` values
nleush Jun 10, 2026
268acc5
change getSources to getContent
iparamonau Jun 12, 2026
2244a40
using parameters destructuring for `findPluginMethods`
nleush Jun 12, 2026
02787a6
using parameters destructuring for `runPluginsIteration`
nleush Jun 12, 2026
63c68b6
add `isVarMethod`
nleush Jun 12, 2026
3819c58
simplify `mandatoryParams` recursive calls
nleush Jun 12, 2026
f56e06c
last iteration workaround
nleush Jun 12, 2026
f2ec663
debug log
nleush Jun 12, 2026
367b955
do not duplicate values in data array
nleush Jun 12, 2026
bb2a7fa
better function name `isGetVarsMethod`
nleush Jun 12, 2026
f7f4789
for __data values allow join arrays
nleush Jun 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,7 +372,7 @@
GET_VARS_METHODS: {
'getSignals': 'signals',
'getPolicy': 'policy',
'getSources': 'sources'
'getContent': 'content',
}
};

Expand Down
97 changes: 77 additions & 20 deletions lib/core.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,15 @@
cache
};

function isGetVarsMethod(name) {
return name === "getVars" || CONFIG.GET_VARS_METHODS && name in CONFIG.GET_VARS_METHODS
}

/*
* Recursively finds plugin methods to run, including 'mixins' and dependencies (up and down by tree).
* */
function findPluginMethods(pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams) {
function findPluginMethods(opts) {
const {pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, getVarsInfo, mandatoryParams} = opts;

/*
* Params:
Expand Down Expand Up @@ -120,7 +125,7 @@
var mixins = plugin.module.mixins;
if (mixins) {
for(var i = 0; i < mixins.length; i++) {
findPluginMethods(mixins[i], loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains);
findPluginMethods({...opts, pluginId: mixins[i]});
}
}
}
Expand Down Expand Up @@ -186,22 +191,29 @@
// Find absent params in other plugins 'provides' attribute.
for(var j = 0; j < pluginsId.length; j++) {
var foundPluginId = pluginsId[j];
findPluginMethods(foundPluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains);
findPluginMethods({...opts, pluginId: foundPluginId, mandatoryParams: undefined});
}

} else {

// This branch goes down by the tree.
if (!getVarsInfo.lastIteration && isGetVarsMethod(method)) {

// Mark method used but not finished.
usedPluginMethods[method] = 1;
getVarsInfo.pendingMethods = true;

// Method will run, store in result set.
methods.push({
pluginId: pluginId,
name: method,
handle: plugin.module[method]
});
} else {

// This branch goes down by the tree.

// Mark method used but not finished.
usedPluginMethods[method] = 1;

// Method will run, store in result set.
methods.push({
pluginId: pluginId,
name: method,
handle: plugin.module[method]
});
}
}
}
}
Expand Down Expand Up @@ -326,6 +338,7 @@
}

// Call method.
// console.log('-- run method:', method.pluginId, method.name)
var result = method.handle.apply(plugin.module, args);

} catch(ex) {
Expand Down Expand Up @@ -361,7 +374,7 @@
/*
* Single iteration of run plugins wave.
* */
function runPluginsIteration(requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb) {
function runPluginsIteration({requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, getVarsInfo, options, asyncMethodCb}) {

var loadedParams = Object.keys(context);

Expand All @@ -376,7 +389,7 @@
// Find methods in each required plugin.
for(var i = 0; i < requiredPlugins.length; i++) {
var plugin = requiredPlugins[i];
findPluginMethods(plugin.id, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains);
findPluginMethods({pluginId: plugin.id, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, getVarsInfo});
}

// If has new unused params (mandatoryParams) - then find plugins which can use them.
Expand All @@ -387,7 +400,7 @@
// // Find methods in plugins, which can use mandatory params.
// for(var i = 0; i < secondaryPlugins.length; i++) {
// var pluginId = secondaryPlugins[i];
// findPluginMethods(pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams);
// findPluginMethods({pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams});
// }
// }

Expand Down Expand Up @@ -882,11 +895,33 @@
allResults.h2 = r.data.htmlparser.h2;
}

const dataValue = r.data[key];

// First data has priority, do not override it.
// whitelistRecord - exception for oembed plugin.
if (!(key in context) || key === 'whitelistRecord') {
context[key] = r.data[key];
context[key] = dataValue;
hasNewData = true;
} else {
// Make arrays from `__key` values.
if (key.startsWith('__')) {
if (!Array.isArray(context[key])) {
// Init array from non array value.
context[key] = [context[key]];
}
if (Array.isArray(dataValue)) {
// Join arrays without duplicating existing items.
const uniqueValues = dataValue.filter(value => !context[key].includes(value));
if (uniqueValues.length) {
context[key] = context[key].concat(uniqueValues);
hasNewData = true;
}
} else if (!context[key].includes(dataValue)) {
// Do not insert existing value.
context[key].push(dataValue);
hasNewData = true;
}
}
}
}
}
Expand Down Expand Up @@ -952,7 +987,7 @@
}
}
}
} else if (r.method.name === "getVars" || CONFIG.GET_VARS_METHODS && r.method.name in CONFIG.GET_VARS_METHODS) {
} else if (isGetVarsMethod(r.method.name)) {

var result_data = r.data;

Expand Down Expand Up @@ -1873,6 +1908,8 @@
usedDomains = pluginsSet.usedDomains,
isDomainPluginsMode = pluginsSet.isDomainPluginsMode,

getVarsInfo = {},

pluginsContexts = {},

allResults = {
Expand Down Expand Up @@ -2047,8 +2084,19 @@

// Run all available plugins again with new data.
var hasRuns = 0;
if (hasNewData || error === 'initial') {
hasRuns = runPluginsIteration(requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb);
var runOpts = {
requiredPlugins,
context,
pluginsUrlMatches,
usedMethods,
usedParams,
usedDomains,
getVarsInfo,
options,
asyncMethodCb
};
if (hasNewData || error === 'initial' || error === 'lastIteration') {
hasRuns = runPluginsIteration(runOpts);
}

if (hasRuns === 0) {
Expand All @@ -2064,6 +2112,14 @@
}
}

if (getVarsInfo.pendingMethods) {
// Make last iteration. All `getVars` argumenets must be loaded at this moment.
getVarsInfo.pendingMethods = false;
getVarsInfo.lastIteration = true;
// Run last iteration.
return asyncMethodCb('lastIteration');
}

// If no data from domain plugins - try fallback to generic plugins.
if (!options.mixAllWithDomainPlugin && isDomainPluginsMode && !resultsHasDomainData(requiredPlugins, allResults.allData)) {

Expand All @@ -2086,9 +2142,10 @@
pluginsUrlMatches = pluginsSet.pluginsUrlMatches;
usedDomains = pluginsSet.usedDomains; // Will be null.
isDomainPluginsMode = pluginsSet.isDomainPluginsMode;
getVarsInfo = {}; // Also reset to restart process.

// Recursive call with same context to prevent same data fetching.
hasRuns = runPluginsIteration(requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb);
hasRuns = runPluginsIteration({requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, getVarsInfo, options, asyncMethodCb});

if (hasRuns === 0) {
finish();
Expand Down
Loading