diff --git a/config.js b/config.js index c4dd73afa..0e42e379a 100644 --- a/config.js +++ b/config.js @@ -372,7 +372,7 @@ GET_VARS_METHODS: { 'getSignals': 'signals', 'getPolicy': 'policy', - 'getSources': 'sources' + 'getContent': 'content', } }; diff --git a/lib/core.js b/lib/core.js index 3e3e77bb8..c57967d09 100644 --- a/lib/core.js +++ b/lib/core.js @@ -35,10 +35,15 @@ cache }; + function isGetVarsMethod(name) { + return name === "getVars" || CONFIG.GET_VARS_METHODS && name in CONFIG.GET_VARS_METHODS + } + /* * Recursively finds plugin methods to run, including 'mixins' and dependencies (up and down by tree). * */ - function findPluginMethods(pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams) { + function findPluginMethods(opts) { + const {pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, getVarsInfo, mandatoryParams} = opts; /* * Params: @@ -120,7 +125,7 @@ var mixins = plugin.module.mixins; if (mixins) { for(var i = 0; i < mixins.length; i++) { - findPluginMethods(mixins[i], loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains); + findPluginMethods({...opts, pluginId: mixins[i]}); } } } @@ -186,22 +191,29 @@ // Find absent params in other plugins 'provides' attribute. for(var j = 0; j < pluginsId.length; j++) { var foundPluginId = pluginsId[j]; - findPluginMethods(foundPluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains); + findPluginMethods({...opts, pluginId: foundPluginId, mandatoryParams: undefined}); } } else { - // This branch goes down by the tree. + if (!getVarsInfo.lastIteration && isGetVarsMethod(method)) { - // Mark method used but not finished. - usedPluginMethods[method] = 1; + getVarsInfo.pendingMethods = true; - // Method will run, store in result set. - methods.push({ - pluginId: pluginId, - name: method, - handle: plugin.module[method] - }); + } else { + + // This branch goes down by the tree. + + // Mark method used but not finished. + usedPluginMethods[method] = 1; + + // Method will run, store in result set. + methods.push({ + pluginId: pluginId, + name: method, + handle: plugin.module[method] + }); + } } } } @@ -326,6 +338,7 @@ } // Call method. + // console.log('-- run method:', method.pluginId, method.name) var result = method.handle.apply(plugin.module, args); } catch(ex) { @@ -361,7 +374,7 @@ /* * Single iteration of run plugins wave. * */ - function runPluginsIteration(requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb) { + function runPluginsIteration({requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, getVarsInfo, options, asyncMethodCb}) { var loadedParams = Object.keys(context); @@ -376,7 +389,7 @@ // Find methods in each required plugin. for(var i = 0; i < requiredPlugins.length; i++) { var plugin = requiredPlugins[i]; - findPluginMethods(plugin.id, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains); + findPluginMethods({pluginId: plugin.id, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, getVarsInfo}); } // If has new unused params (mandatoryParams) - then find plugins which can use them. @@ -387,7 +400,7 @@ // // Find methods in plugins, which can use mandatory params. // for(var i = 0; i < secondaryPlugins.length; i++) { // var pluginId = secondaryPlugins[i]; - // findPluginMethods(pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams); + // findPluginMethods({pluginId, loadedParams, pluginsUrlMatches, usedMethods, usedParams, methods, scannedPluginsIds, usedDomains, mandatoryParams}); // } // } @@ -882,11 +895,33 @@ allResults.h2 = r.data.htmlparser.h2; } + const dataValue = r.data[key]; + // First data has priority, do not override it. // whitelistRecord - exception for oembed plugin. if (!(key in context) || key === 'whitelistRecord') { - context[key] = r.data[key]; + context[key] = dataValue; hasNewData = true; + } else { + // Make arrays from `__key` values. + if (key.startsWith('__')) { + if (!Array.isArray(context[key])) { + // Init array from non array value. + context[key] = [context[key]]; + } + if (Array.isArray(dataValue)) { + // Join arrays without duplicating existing items. + const uniqueValues = dataValue.filter(value => !context[key].includes(value)); + if (uniqueValues.length) { + context[key] = context[key].concat(uniqueValues); + hasNewData = true; + } + } else if (!context[key].includes(dataValue)) { + // Do not insert existing value. + context[key].push(dataValue); + hasNewData = true; + } + } } } } @@ -952,7 +987,7 @@ } } } - } else if (r.method.name === "getVars" || CONFIG.GET_VARS_METHODS && r.method.name in CONFIG.GET_VARS_METHODS) { + } else if (isGetVarsMethod(r.method.name)) { var result_data = r.data; @@ -1873,6 +1908,8 @@ usedDomains = pluginsSet.usedDomains, isDomainPluginsMode = pluginsSet.isDomainPluginsMode, + getVarsInfo = {}, + pluginsContexts = {}, allResults = { @@ -2047,8 +2084,19 @@ // Run all available plugins again with new data. var hasRuns = 0; - if (hasNewData || error === 'initial') { - hasRuns = runPluginsIteration(requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb); + var runOpts = { + requiredPlugins, + context, + pluginsUrlMatches, + usedMethods, + usedParams, + usedDomains, + getVarsInfo, + options, + asyncMethodCb + }; + if (hasNewData || error === 'initial' || error === 'lastIteration') { + hasRuns = runPluginsIteration(runOpts); } if (hasRuns === 0) { @@ -2064,6 +2112,14 @@ } } + if (getVarsInfo.pendingMethods) { + // Make last iteration. All `getVars` argumenets must be loaded at this moment. + getVarsInfo.pendingMethods = false; + getVarsInfo.lastIteration = true; + // Run last iteration. + return asyncMethodCb('lastIteration'); + } + // If no data from domain plugins - try fallback to generic plugins. if (!options.mixAllWithDomainPlugin && isDomainPluginsMode && !resultsHasDomainData(requiredPlugins, allResults.allData)) { @@ -2086,9 +2142,10 @@ pluginsUrlMatches = pluginsSet.pluginsUrlMatches; usedDomains = pluginsSet.usedDomains; // Will be null. isDomainPluginsMode = pluginsSet.isDomainPluginsMode; + getVarsInfo = {}; // Also reset to restart process. // Recursive call with same context to prevent same data fetching. - hasRuns = runPluginsIteration(requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, options, asyncMethodCb); + hasRuns = runPluginsIteration({requiredPlugins, context, pluginsUrlMatches, usedMethods, usedParams, usedDomains, getVarsInfo, options, asyncMethodCb}); if (hasRuns === 0) { finish();