Skip to content

Commit ecb8131

Browse files
inaridiyinaridiy
authored
Merge pull request #56 from inaridiy/feat/improve-extraction
Feat/improve extraction
2 parents da5f403 + ea8b326 commit ecb8131

13 files changed

Lines changed: 262 additions & 52 deletions

File tree

‎.changeset/rich-mayflies-divide.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
"ai-learning": minor
3+
"webforai": minor
4+
"simple": minor
5+
"site": minor
6+
---
7+
8+
Add minimal filter extractor

‎.clinerules‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
{
2+
"project": {
3+
"name": "WebForAI",
4+
"description": "A library that converts HTML to Markdown with various loaders and extractors for AI consumption",
5+
"repository": "https://github.com/inaridiy/webforai",
6+
"homepage": "https://webforai.dev/"
7+
},
8+
"structure": {
9+
"monorepo": true,
10+
"packageManager": "pnpm",
11+
"mainPackage": "packages/webforai",
12+
"directories": {
13+
"packages": {
14+
"description": "Contains the main WebForAI package",
15+
"patterns": ["packages/**"]
16+
},
17+
"examples": {
18+
"description": "Example projects demonstrating WebForAI usage",
19+
"patterns": ["examples/**"]
20+
},
21+
"site": {
22+
"description": "Documentation website",
23+
"patterns": ["site/**"]
24+
},
25+
"apps": {
26+
"description": "Application implementations",
27+
"patterns": ["apps/**"]
28+
}
29+
}
30+
},
31+
"capabilities": {
32+
"core": [
33+
"HTML to Markdown conversion",
34+
"HTML to MDAST conversion",
35+
"MDAST to Markdown conversion",
36+
"Web content loading via various methods"
37+
],
38+
"loaders": ["Playwright", "Puppeteer", "Cloudflare Puppeteer", "Fetch API"],
39+
"extractors": ["Content extraction presets", "Custom extraction pipelines"]
40+
},
41+
"algorithms": {
42+
"htmlToMarkdown": {
43+
"description": "Main conversion pipeline that transforms HTML to Markdown",
44+
"flow": "HTML → HAST → MDAST → Markdown",
45+
"steps": [
46+
"Parse HTML into HAST (HTML Abstract Syntax Tree)",
47+
"Apply content extractors to clean and focus on main content",
48+
"Transform HAST to MDAST (Markdown Abstract Syntax Tree)",
49+
"Convert MDAST to Markdown text with formatting options"
50+
]
51+
},
52+
"contentExtraction": {
53+
"description": "Intelligent algorithms to extract the main content from web pages",
54+
"implementations": [
55+
{
56+
"name": "takumiExtractor",
57+
"description": "Advanced content extractor inspired by Mozilla Readability",
58+
"techniques": [
59+
"Metadata filtering to remove scripts, styles, and other non-content elements",
60+
"Universal element filtering to remove navigation, asides, and hidden content",
61+
"Content selection using common article selectors",
62+
"Link density analysis to identify content-rich areas",
63+
"Language-specific content length thresholds"
64+
]
65+
}
66+
]
67+
},
68+
"mdastHandlers": {
69+
"description": "Custom handlers for transforming specific HTML elements to Markdown",
70+
"handlers": [
71+
"customAHandler: Enhanced link handling with text-only option",
72+
"customCodeHandler: Code block handling with language detection",
73+
"customDivHandler: Special div element processing",
74+
"customImgHandler: Image handling with hide option",
75+
"customTableHandler: Table processing with text-only option",
76+
"mathHandler: Mathematical notation conversion"
77+
]
78+
},
79+
"linkProcessing": {
80+
"description": "Utilities for handling and transforming links",
81+
"features": ["Relative to absolute URL conversion", "Base URL integration", "Link text extraction"]
82+
}
83+
},
84+
"development": {
85+
"nodeVersion": ">=18.0.0",
86+
"commands": {
87+
"build": "pnpm run --r --filter \"./packages/**\" build",
88+
"test": "vitest",
89+
"format": "biome format .",
90+
"lint": "biome check ."
91+
},
92+
"tools": ["TypeScript", "Biome", "Vitest", "Changesets"]
93+
},
94+
"customModes": [
95+
{
96+
"slug": "webforai-dev",
97+
"name": "WebForAI Developer",
98+
"roleDefinition": "You are Roo, a specialized developer for the WebForAI library. You understand HTML parsing, Markdown generation, and web content extraction techniques. You're familiar with the project's architecture including loaders, extractors, and MDAST/HAST transformations.",
99+
"groups": ["read", "edit", "browser", "command", "mcp"]
100+
},
101+
{
102+
"slug": "webforai-docs",
103+
"name": "WebForAI Documentation",
104+
"roleDefinition": "You are Roo, a documentation specialist for the WebForAI library. You excel at creating clear, concise documentation with practical examples. You understand the library's capabilities and can explain complex concepts in an accessible way.",
105+
"groups": [
106+
"read",
107+
["edit", { "fileRegex": "\\.(md|mdx)$", "description": "Markdown and MDX files only" }],
108+
"browser",
109+
"command",
110+
"mcp"
111+
]
112+
}
113+
]
114+
}

‎examples/ai-learning/src/manual.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ await fs.mkdirSync(".output", { recursive: true });
5252
await fs.writeFileSync(".output/html.html", html);
5353

5454
const rawContent = await htmlToMarkdown(html, { baseUrl: url, extractors: false });
55-
const cleanedContent = await htmlToMarkdown(html, { baseUrl: url, extractors: "takumi" });
55+
const cleanedContent = await htmlToMarkdown(html, { baseUrl: url });
5656

5757
await fs.writeFileSync(".output/raw.md", rawContent);
5858
await fs.writeFileSync(".output/cleaned.md", cleanedContent);

‎examples/simple/src/index.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,16 @@ await fs.mkdir(".output", { recursive: true });
77

88
const args = arg({ "--url": String });
99

10-
const url = args["--url"] || "https://ja.wikipedia.org/wiki/%E5%BE%A1%E5%9D%82%E7%BE%8E%E7%90%B4";
10+
const url = args["--url"] ?? "https://webforai.dev/";
1111

1212
const html = await loadHtml(url);
13-
const markdown = htmlToMarkdown(html, { baseUrl: url, linkAsText: true, hideImage: true });
13+
14+
await fs.writeFile(".output/output.html", html);
15+
16+
const rawMarkdown = htmlToMarkdown(html, { baseUrl: url, extractors: false });
17+
18+
await fs.writeFile(".output/output.raw.md", rawMarkdown);
19+
20+
const markdown = htmlToMarkdown(html, { baseUrl: url });
1421

1522
await fs.writeFile(".output/output.md", markdown);

‎packages/webforai/package.json‎

Lines changed: 7 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,7 @@
44
"description": "A library that provides a web interface for AI",
55
"author": "inaridiy",
66
"license": "Apache-2.0",
7-
"keywords": [
8-
"web",
9-
"ai",
10-
"html",
11-
"html2md",
12-
"markdown",
13-
"mdast",
14-
"hast"
15-
],
7+
"keywords": ["web", "ai", "html", "html2md", "markdown", "mdast", "hast"],
168
"repository": {
179
"type": "git",
1810
"url": "https://github.com/inaridiy/webforai.git"
@@ -26,10 +18,7 @@
2618
"prerelease": "pnpm build",
2719
"release": "np"
2820
},
29-
"files": [
30-
"dist",
31-
"!dist/types/**/*.js"
32-
],
21+
"files": ["dist", "!dist/types/**/*.js"],
3322
"main": "dist/cjs/index.js",
3423
"type": "module",
3524
"module": "dist/index.js",
@@ -69,21 +58,11 @@
6958
},
7059
"typesVersions": {
7160
"*": {
72-
"types": [
73-
"./dist/types/index.d.ts"
74-
],
75-
"loaders/playwright": [
76-
"./dist/types/loaders/playwright.d.ts"
77-
],
78-
"loaders/cf-puppeteer": [
79-
"./dist/types/loaders/cf-puppeteer.d.ts"
80-
],
81-
"loaders/fetch": [
82-
"./dist/types/loaders/fetch.d.ts"
83-
],
84-
"loaders/puppeteer": [
85-
"./dist/types/loaders/puppeteer.d.ts"
86-
]
61+
"types": ["./dist/types/index.d.ts"],
62+
"loaders/playwright": ["./dist/types/loaders/playwright.d.ts"],
63+
"loaders/cf-puppeteer": ["./dist/types/loaders/cf-puppeteer.d.ts"],
64+
"loaders/fetch": ["./dist/types/loaders/fetch.d.ts"],
65+
"loaders/puppeteer": ["./dist/types/loaders/puppeteer.d.ts"]
8766
}
8867
},
8968
"peerDependencies": {

‎packages/webforai/src/constants.ts‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,3 @@
11
import { takumiExtractor } from "./extractors/presets/takumi";
22

3-
export const PRESET_EXTRACTORS = {
4-
takumi: takumiExtractor,
5-
};
6-
7-
export const DEFAULT_EXTRACTORS = ["takumi" as const];
3+
export const DEFAULT_EXTRACTORS = [takumiExtractor];

‎packages/webforai/src/extractors/index.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ export {
33
pipeExtractors,
44
type ExtractorSelectors,
55
type ExtractorSelector,
6-
type PresetExtractors,
76
} from "./pipeExtractors";
87
export { takumiExtractor } from "./presets/takumi";
98
export { type ExtractParams, type Extractor } from "./types";

‎packages/webforai/src/extractors/pipeExtractors.ts‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
import type { Nodes as Hast } from "hast";
2-
import { DEFAULT_EXTRACTORS, PRESET_EXTRACTORS } from "../constants";
2+
import { DEFAULT_EXTRACTORS } from "../constants";
33
import type { ExtractParams, Extractor } from "./types";
44

5-
export type PresetExtractors = keyof typeof PRESET_EXTRACTORS;
6-
export type ExtractorSelector = Extractor | false | PresetExtractors;
5+
export type ExtractorSelector = Extractor | false;
76
export type ExtractorSelectors = ExtractorSelector | ExtractorSelector[];
87

98
export const pipeExtractors = (params: ExtractParams, extractors: ExtractorSelectors = DEFAULT_EXTRACTORS): Hast => {
@@ -15,9 +14,6 @@ export const pipeExtractors = (params: ExtractParams, extractors: ExtractorSelec
1514
if (extractor === false) {
1615
return acc;
1716
}
18-
if (typeof extractor === "string" && extractor in PRESET_EXTRACTORS) {
19-
return PRESET_EXTRACTORS[extractor]({ hast: acc, lang });
20-
}
2117
if (typeof extractor === "function") {
2218
return extractor({ hast: acc, lang });
2319
}

‎packages/webforai/src/extractors/presets/minimal-filter.ts‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
import type { Element, Nodes as Hast } from "hast";
2+
import { select } from "hast-util-select";
3+
import { toString as hastToString } from "hast-util-to-string";
4+
import { filter } from "unist-util-filter";
5+
import type { ExtractParams } from "../types";
6+
import { classnames, isStrInclude, matchString } from "./utils";
7+
8+
const UNLIKELY_ROLES = ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"];
9+
10+
/*
11+
* This section of the code is influenced by @mozilla/readability, licensed under Apache License 2.0.
12+
* Original copyright (c) 2010 Arc90 Inc
13+
* See https://github.com/mozilla/readability for the full license text.
14+
* Modifications made by inaridiy
15+
* - Added and edited some regular expressions.
16+
*/
17+
const REGEXPS = {
18+
hidden: /hidden|invisible|fallback-image/i,
19+
byline: /byline|author|dateline|writtenby|p-author/i,
20+
specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
21+
unlikelyCandidates:
22+
/-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
23+
okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i,
24+
};
25+
26+
const metadataFilter = (node: Hast) => {
27+
return !(
28+
["comment", "doctype"].includes(node.type) ||
29+
(node.type === "element" && ["script", "style", "link", "meta", "noscript", "svg", "title"].includes(node.tagName))
30+
);
31+
};
32+
33+
const universalElementFilter = (node: Hast) => {
34+
if (node.type !== "element") {
35+
return true;
36+
}
37+
const element = node as Element;
38+
39+
if (["aside", "nav"].includes(element.tagName)) {
40+
return false;
41+
}
42+
43+
// Remove elements with hidden properties
44+
if (["hidden", "aria-hidden"].some((key) => element.properties[key])) {
45+
return false;
46+
}
47+
if (classnames(element).some((classname) => REGEXPS.hidden.test(classname))) {
48+
return false;
49+
}
50+
51+
// Remove dialog elements
52+
if (element.tagName === "dialog") {
53+
return false;
54+
}
55+
if (element.properties.role === "dialog" && element.properties["aria-modal"]) {
56+
return false;
57+
}
58+
59+
// Remove byline elements
60+
if (element.properties.rel === "author" && isStrInclude(element.properties.itemprop, "author")) {
61+
return false;
62+
}
63+
if (REGEXPS.byline.test(matchString(element))) {
64+
return false;
65+
}
66+
67+
// Remove unlikely roles
68+
if (element.properties.role && UNLIKELY_ROLES.includes(element.properties.role as string)) {
69+
return false;
70+
}
71+
72+
return true;
73+
};
74+
75+
/**
76+
* Simple filter to remove unwanted elements from the HAST tree.
77+
*
78+
* @param params - {@link ExtractParams}
79+
* @returns The HAST tree.
80+
*/
81+
export const minimalFilter = (params: ExtractParams): Hast => {
82+
const { hast } = params;
83+
const body = select("body", hast) ?? hast;
84+
85+
const metadataFilteredHast = filter(body, (node) => metadataFilter(node as Hast));
86+
const metadataFilteredHastText = metadataFilteredHast && hastToString(metadataFilteredHast);
87+
if (!(metadataFilteredHast && metadataFilteredHastText)) {
88+
return body;
89+
}
90+
91+
const baseFilterd = filter(metadataFilteredHast, (node) => universalElementFilter(node as Hast));
92+
const baseFilterdText = baseFilterd ? hastToString(baseFilterd) : "";
93+
94+
const isOverExtracted = baseFilterdText.length > metadataFilteredHastText.length / 3 || baseFilterdText.length > 5000;
95+
const baseTree = isOverExtracted && baseFilterd ? baseFilterd : metadataFilteredHast;
96+
97+
return baseTree;
98+
};

‎packages/webforai/src/extractors/presets/takumi.ts‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,19 @@ const REGEXPS = {
1919
byline: /byline|author|dateline|writtenby|p-author/i,
2020
specialUnlikelyCandidates: /frb-|uls-menu|language-link/i,
2121
unlikelyCandidates:
22-
/-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
22+
/-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|tooltip|disqus|extra|footer|gdpr|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote|speechify-ignore|avatar/i,
2323
okMaybeItsaCandidate: /and|article|body|column|content|main|shadow|code/i,
2424
};
2525

2626
const BODY_SELECTORS = ["article", "#article", ".article_body", ".article-body", "#content", ".entry"];
2727

2828
const PARAGRAPH_TAGS = ["a", "p", "div", "section", "article", "main", "ul", "ol", "li"];
2929

30-
const BASE_MINIMAL_LENGTH = {
31-
ja: 200,
32-
en: 500,
33-
};
30+
const CONTENTABLE_TAGS = ["article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"];
31+
32+
const TEXTABLE_TAGS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "li", "ul"];
33+
34+
const BASE_MINIMAL_LENGTH = { ja: 200, en: 500 };
3435

3536
const metadataFilter = (node: Hast) => {
3637
return !(
@@ -88,9 +89,10 @@ const unlikelyElementFilter = (node: Hast) => {
8889
const element = node as Element;
8990

9091
// Skip main content elements
91-
if (["body", "article", "main", "section", "h1", "h2", "h3", "h4", "h5", "h6", "p"].includes(element.tagName)) {
92+
if (CONTENTABLE_TAGS.includes(element.tagName)) {
9293
return true;
9394
}
95+
9496
const match = matchString(element);
9597

9698
if (REGEXPS.specialUnlikelyCandidates.test(match)) {
@@ -119,6 +121,10 @@ const removeEmptyFilter = (node: Hast, _lang: string) => {
119121
return false;
120122
}
121123

124+
if (TEXTABLE_TAGS.includes(element.tagName) && hastToString(element).length === 0) {
125+
return false;
126+
}
127+
122128
return true;
123129
};
124130

0 commit comments

Comments
 (0)