Merge pull request #762 from microlinkhq/next

feat: add `omitPropNames`

Author: Kikobeats

README.md

@@ -285,6 +285,31 @@ Default: `true`
 
 Ensure the URL provided is validated as a [WHATWG URL](https://nodejs.org/api/url.html#url_the_whatwg_url_api) API compliant.
 
+#### omitPropNames
+
+Type: `Set`<br>
+Default: `undefined`
+
+A set of property names that should be omitted. When specified, these properties will be missing in the returned metadata objects, and rules related to that will not be computed.
+
+```js
+const metascraper = require('metascraper')([
+  require('metascraper-title')(),
+  require('metascraper-image')(),
+  require('metascraper-description')()
+])
+
+const html = '<title>Example</title><meta property="og:image" content="image.jpg">'
+const url = 'https://example.com'
+
+// Omit the image property
+const omitPropNames = new Set(['image'])
+const metadata = await metascraper({ url, html, omitPropNames })
+
+console.log(metadata)
+// Output: { title: 'Example', image: null, description: null }
+```
+
 ## Environment Variables
 
 #### METASCRAPER_RE2

packages/metascraper/benchmark/index.js renamed to packages/metascraper/benchmark/merge-rules.js

@@ -52,7 +52,7 @@ const benchmark = (fn, iterations = CONFIG.BENCHMARK_ITERATIONS) => {
 }
 
 console.log('┌────────────────────────────────────────────────┐')
-console.log('│            MergeRules Benchmark                │')
+console.log('│           MergeRules Benchmark                 │')
 console.log('└────────────────────────────────────────────────┘\n')
 
 // Large comprehensive test case

packages/metascraper/package.json

@@ -61,7 +61,6 @@
     "@metascraper/helpers": "workspace:*",
     "cheerio": "~1.1.0",
     "debug-logfmt": "~1.2.3",
-    "lodash": "~4.17.21",
     "whoops": "~5.0.1"
   },
   "devDependencies": {

packages/metascraper/src/get-data.js

@@ -2,22 +2,19 @@
 
 const debug = require('debug-logfmt')('metascraper:get-data')
 const { findRule, has } = require('@metascraper/helpers')
-const { map, fromPairs } = require('lodash')
 
-const normalizeValue = value => (has(value) ? value : null)
-
-const getData = async ({ rules, name, ...props }) => {
+const getData = async ({ rules, ...props }) => {
   const data = await Promise.all(
-    map(rules, async ([propName, innerRules]) => {
+    rules.map(async ([propName, innerRules]) => {
       const duration = debug.duration()
       const value = await findRule(innerRules, props, propName)
-      const normalizedValue = normalizeValue(value)
+      const normalizedValue = has(value) ? value : null
       duration(`${propName}=${normalizedValue} rules=${innerRules.length}`)
       return [propName, normalizedValue]
     })
   )
 
-  return fromPairs(data)
+  return Object.fromEntries(data)
 }
 
 module.exports = getData

packages/metascraper/src/index.d.ts

@@ -34,6 +34,11 @@ declare namespace createMetascraper {
      * Ensure the URL provided is validated as a WHATWG URL API compliant.
      */
     validateUrl?: boolean;
+    /**
+     * A Set of property names to omit from the metadata extraction process.
+     * These properties will be filtered out before processing the rules.
+     */
+    omitPropNames?: Set<string>;
   }
 
   export interface Metadata {

packages/metascraper/src/index.js

@@ -17,6 +17,7 @@ module.exports = rules => {
     htmlDom,
     rules: inlineRules,
     validateUrl = true,
+    omitPropNames = new Set(),
     ...props
   } = {}) => {
     if (validateUrl && !isUrl(url)) {
@@ -29,7 +30,7 @@ module.exports = rules => {
     return getData({
       url,
       htmlDom: htmlDom ?? load(html, { baseURI: url }),
-      rules: mergeRules(inlineRules, loadedRules),
+      rules: mergeRules(inlineRules, loadedRules, omitPropNames),
       ...props
     })
   }

packages/metascraper/src/rules.js

@@ -1,57 +1,69 @@
 'use strict'
 
-const {
-  cloneDeep,
-  concat,
-  first,
-  findIndex,
-  forEach,
-  chain,
-  castArray,
-  has,
-  set
-} = require('lodash')
-
-const forEachRule = (collection, fn) => forEach(castArray(collection), fn)
-
-const loadRules = rulesBundle =>
-  chain(rulesBundle)
-    .reduce((acc, { test, pkgName, ...rules }) => {
-      forEach(rules, (innerRules, propName) => {
-        forEachRule(innerRules, rule => {
+const castArray = value => (Array.isArray(value) ? value : [value])
+
+const loadRules = rulesBundle => {
+  const acc = {}
+
+  for (const { test, pkgName, ...rules } of rulesBundle) {
+    for (const [propName, innerRules] of Object.entries(rules)) {
+      const processedRules = castArray(innerRules)
+      if (test || pkgName) {
+        for (const rule of processedRules) {
           if (test) rule.test = test
-          rule.pkgName = pkgName ?? 'uknown'
-        })
-
-        set(
-          acc,
-          propName,
-          has(acc, propName)
-            ? concat(acc[propName], innerRules)
-            : concat(innerRules)
-        )
-
-        return acc
-      })
-      return acc
-    }, {})
-    .toPairs()
-    .value()
-
-const mergeRules = (rules, baseRules) =>
-  chain(rules)
-    .reduce((acc, { test, ...rules }) => {
-      forEach(rules, (innerRules, propName) => {
-        if (test) forEachRule(innerRules, rule => (rule.test = test))
-        // find the rules associated with `propName`
-        const index = findIndex(acc, item => first(item) === propName)
-        // if `propName` has more rule, add the new rule from the end
-        if (index !== -1) acc[index][1] = concat(innerRules, ...acc[index][1])
-        // otherwise, create an array of rules
-        else acc.push([propName, castArray(innerRules)])
-      })
-      return acc
-    }, cloneDeep(baseRules))
-    .value()
+          if (pkgName) rule.pkgName = pkgName ?? 'unknown'
+        }
+      }
+
+      if (acc[propName]) {
+        acc[propName].push(...processedRules)
+      } else {
+        acc[propName] = processedRules
+      }
+    }
+  }
+  return Object.entries(acc)
+}
+
+const mergeRules = (rules, baseRules, omitPropNames = new Set()) => {
+  const result = {}
+
+  // Process base rules first (shallow clone arrays only)
+  for (const [propName, ruleArray] of baseRules) {
+    if (!omitPropNames.has(propName)) {
+      result[propName] = [...ruleArray] // Shallow clone array
+    }
+  }
+
+  // Handle case where rules might be null/undefined or not an array
+  if (!rules || !Array.isArray(rules)) {
+    return Object.entries(result)
+  }
+
+  // Process inline rules
+  for (const { test, ...ruleSet } of rules) {
+    for (const [propName, innerRules] of Object.entries(ruleSet)) {
+      if (omitPropNames.has(propName)) continue
+
+      const processedRules = Array.isArray(innerRules)
+        ? [...innerRules]
+        : [innerRules]
+      if (test) {
+        for (const rule of processedRules) {
+          rule.test = test
+        }
+      }
+
+      if (result[propName]) {
+        // Prepend new rules to match original concat(innerRules, existing) behavior
+        result[propName] = [...processedRules, ...result[propName]]
+      } else {
+        result[propName] = processedRules
+      }
+    }
+  }
+
+  return Object.entries(result)
+}
 
 module.exports = { mergeRules, loadRules }

packages/metascraper/test/integration/transistor/index.js

@@ -25,6 +25,8 @@ const url = 'https://share.transistor.fm/e/70c487ed'
 
 test('transistor.fm', async t => {
   const html = await readFile(resolve(__dirname, 'input.html'))
-  const metadata = await metascraper({ html, url })
+  const { logo, ...metadata } = await metascraper({ html, url })
   t.snapshot(metadata)
+  t.is(typeof logo, 'string')
+  t.true(new URL(logo).hostname.endsWith('.gstatic.com'), logo)
 })

packages/metascraper/test/integration/transistor/snapshots/index.js.md

@@ -15,7 +15,6 @@ Generated by [AVA](https://avajs.dev).
       description: null,
       image: 'https://images.transistor.fm/file/transistor/images/episode/263474/medium_1590257639-artwork.jpg',
       lang: null,
-      logo: 'https://t1.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL&url=https://share.transistor.fm/e/70c487ed&size=128',
       publisher: '#032 – Before and After Product-Market Fit with Peter and Calvin from Segment',
       title: 'Transistor Embed | #032 – Before and After Product-Market Fit with Peter and Calvin from Segment',
       url: 'https://share.transistor.fm/e/70c487ed',