diff --git a/lib/core.js b/lib/core.js index 5e3f5af8c..4dd99bccb 100644 --- a/lib/core.js +++ b/lib/core.js @@ -1154,7 +1154,7 @@ return hasDomainData; } - const BIG_CONTEXT = ['readability', 'decode', 'cheerio']; + const BIG_CONTEXT = ['readability', '$fulltext', 'decode', 'cheerio']; function prepareResultData(uri, result, options) { @@ -1692,6 +1692,11 @@ } } + // Read from CONFIG. + if (typeof value === 'undefined' && CONFIG.domainOptions) { + value = searchParamInObj(0, bits, CONFIG.domainOptions); + } + if ( typeof defaultValue !== "undefined" && typeof value !== typeof defaultValue @@ -1702,12 +1707,7 @@ } else { return defaultValue; } - } - - // Read from CONFIG. - if (typeof value === 'undefined' && CONFIG.domainOptions) { - value = searchParamInObj(0, bits, CONFIG.domainOptions); - } + } return typeof value !== 'undefined' ? value : defaultValue; }; diff --git a/package.json b/package.json index 2935f3b41..102846666 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "jslint": "^0.12.1", "json5": "^2.2.3", "jsontoxml": "^1.0.1", + "marked": "^18.0.4", "memcached": "2.2.2", "moment": "^2.30.1", "node-cache": "^1.1.0", @@ -46,7 +47,8 @@ "redis": "^4.6.14", "redis-clustr": "1.7.0", "sax": "^1.2.4", - "send": "^1.2.0" + "send": "^1.2.0", + "turndown": "^7.2.4" }, "devDependencies": { "chai": "^6.2.0", diff --git a/plugins/links/article/article.js b/plugins/links/article/article.js deleted file mode 100644 index 505547520..000000000 --- a/plugins/links/article/article.js +++ /dev/null @@ -1,25 +0,0 @@ -import * as cheerio from 'cheerio'; - -export default { - - provides: 'articlebody', // if not yet provided from LD articlebody - - getData: function(__readabilityEnabled, readability, meta, utils) { - - const articleHtml = utils.encodeText(meta.charset, readability.getHTML()); - const $p = cheerio.load(articleHtml)('p'); - const articleText = $p.text(); - - if (articleText) { - return { - articlebody: __readabilityEnabled === 'html' ? articleHtml : articleText.replace(/\.([^\.\d\s\n\r\'\"\”\)\]])/g, '. $1') - } - } - }, - - getVars: function(articlebody) { - return { - articlebody: articlebody - }; - } -}; \ No newline at end of file diff --git a/plugins/links/article/check-article.js b/plugins/links/article/check-article.js deleted file mode 100644 index f7a0cdb87..000000000 --- a/plugins/links/article/check-article.js +++ /dev/null @@ -1,33 +0,0 @@ -export default { - - provides: [ - "__readabilityEnabled", - "articlebody" - ], - - getData: function(meta, options) { - - const ld = meta.ld?.newsarticle || meta.ld?.article || meta.ld?.blogposting || meta.ld?.reportagenewsarticle || meta.ld?.socialmediaposting; - - if ((ld - || (meta.og && (meta.og.type === "article" || meta.og.type === "blog" || meta.og.type === 'website') - || meta.twitter?.card === 'summary_large_image' - || meta.article)) - - && (options.getRequestOptions('readability.articlebody') || CONFIG.providerOptions?.app?.allow_readability === true)) { - - const article_format = (options.getRequestOptions('readability.articlebody') === 'html' - || CONFIG.providerOptions?.app?.allow_readability === true) ? 'html' : 'txt'; - - if (ld?.articlebody && (article_format !== 'html' || /\/>/.test(ld.articlebody))) { - return { - articlebody: ld.articlebody - } - } else if (options.getProviderOptions('app.allow_readability')) { - return { - __readabilityEnabled: article_format - } - } - } - } -}; \ No newline at end of file diff --git a/plugins/links/article/reader.js b/plugins/links/article/reader.js deleted file mode 100644 index 3d496baa7..000000000 --- a/plugins/links/article/reader.js +++ /dev/null @@ -1,10 +0,0 @@ -export default { - - getData: function(__readabilityEnabled, articlebody) { - if (__readabilityEnabled === 'html' && CONFIG.providerOptions?.app?.allow_readability === true && !CONFIG.SKIP_IFRAMELY_RENDERS) { - return { - safe_html: articlebody - } - } - } -}; \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6ad584193..f127aac8e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -72,6 +72,9 @@ importers: jsontoxml: specifier: ^1.0.1 version: 1.0.1 + marked: + specifier: ^18.0.4 + version: 18.0.4 memcached: specifier: 2.2.2 version: 2.2.2 @@ -105,6 +108,9 @@ importers: send: specifier: ^1.2.0 version: 1.2.0 + turndown: + specifier: ^7.2.4 + version: 7.2.4 devDependencies: chai: specifier: ^6.2.0 @@ -131,6 +137,9 @@ packages: resolution: {integrity: sha512-Zak2kPJuIdg9UQQfUgNm848vRAg2pdOqYYU+7DkCYWO+SgZiMV+qy99BpO1geDiP2rQ2M7JH5oNXRTEvEWglRQ==} engines: {node: '>=14.16'} + '@mixmark-io/domino@2.2.0': + resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==} + '@mongodb-js/saslprep@1.3.2': resolution: {integrity: sha512-QgA5AySqB27cGTXBFmnpifAi7HxoGUeezwo6p9dI03MuDB6Pp33zgclqVb6oVK3j6I9Vesg0+oojW2XxB59SGg==} @@ -817,6 +826,11 @@ packages: resolution: {integrity: sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==} engines: {node: '>=12'} + marked@18.0.4: + resolution: {integrity: sha512-c/BTaKzg0G6ezQx97DAkYU7k0HM6ys0FqYeKBL6hlBByZwy+ycA1+f0vDdjMHKKeEjdgkx0GOv9Il6D+85cOqA==} + engines: {node: '>= 20'} + hasBin: true + math-intrinsics@1.1.0: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} @@ -1244,6 +1258,10 @@ packages: resolution: {integrity: sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==} engines: {node: '>=14'} + turndown@7.2.4: + resolution: {integrity: sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==} + engines: {node: '>=18', npm: '>=9'} + type-fest@4.32.0: resolution: {integrity: sha512-rfgpoi08xagF3JSdtJlCwMq9DGNDE0IMh3Mkpc1wUypg9vPi786AiqeBBKcqvIkq42azsBM85N490fyZjeUftw==} engines: {node: '>=16'} @@ -1337,6 +1355,8 @@ snapshots: transitivePeerDependencies: - supports-color + '@mixmark-io/domino@2.2.0': {} + '@mongodb-js/saslprep@1.3.2': dependencies: sparse-bitfield: 3.0.3 @@ -2044,6 +2064,8 @@ snapshots: lru-cache@7.18.3: {} + marked@18.0.4: {} + math-intrinsics@1.1.0: {} media-typer@1.1.0: {} @@ -2523,6 +2545,10 @@ snapshots: dependencies: punycode: 2.3.0 + turndown@7.2.4: + dependencies: + '@mixmark-io/domino': 2.2.0 + type-fest@4.32.0: {} type-is@2.0.1: