ArchiveBox/archivebox/extractors/htmltotext.py
Ross Williams 310b4d1242 Add htmltotext extractor
Saves HTML text nodes and selected element attributes in
`htmltotext.txt` for each Snapshot. Primarily intended to be used
for search indexing.
2023-10-23 21:42:32 -04:00

154 lines
4.5 KiB
Python

__package__ = 'archivebox.extractors'
from html.parser import HTMLParser
import io
from pathlib import Path
from typing import Optional
from ..config import (
SAVE_HTMLTOTEXT,
TIMEOUT,
VERSION,
)
from ..index.schema import Link, ArchiveResult, ArchiveError
from ..logging_util import TimedProgress
from ..system import atomic_write
from ..util import (
enforce_types,
is_static_file,
)
from .title import get_html
class HTMLTextExtractor(HTMLParser):
TEXT_ATTRS = [
"alt", "cite", "href", "label",
"list", "placeholder", "title", "value"
]
NOTEXT_TAGS = ["script", "style", "template"]
NOTEXT_HREF = ["data:", "javascript:", "#"]
def __init__(self):
super().__init__()
self.output = io.StringIO()
self._tag_stack = []
def _is_text_attr(self, name, value):
if not isinstance(value, str):
return False
if name == "href" and any(map(lambda p: value.startswith(p), self.NOTEXT_HREF)):
return False
if name in self.TEXT_ATTRS:
return True
return False
def _parent_tag(self):
try:
return self._tag_stack[-1]
except IndexError:
return None
def _in_notext_tag(self):
return any([t in self._tag_stack for t in self.NOTEXT_TAGS])
def handle_starttag(self, tag, attrs):
self._tag_stack.append(tag)
# Don't write out attribute values if any ancestor
# is in NOTEXT_TAGS
if self._in_notext_tag():
return
for name, value in attrs:
if self._is_text_attr(name, value):
self.output.write(f"({value.strip()}) ")
def handle_endtag(self, tag):
orig_stack = self._tag_stack.copy()
try:
# Keep popping tags until we find the nearest
# ancestor matching this end tag
while tag != self._tag_stack.pop():
pass
# Write a space after every tag, to ensure that tokens
# in tag text aren't concatenated. This may result in
# excess spaces, which should be ignored by search tokenizers.
if not self._in_notext_tag() and tag not in self.NOTEXT_TAGS:
self.output.write(" ")
except IndexError:
# Got to the top of the stack, but somehow missed
# this end tag -- maybe malformed markup -- restore the
# stack
self._tag_stack = orig_stack
def handle_data(self, data):
# Don't output text data if any ancestor is in NOTEXT_TAGS
if self._in_notext_tag():
return
data = data.lstrip()
len_before_rstrip = len(data)
data = data.rstrip()
spaces_rstripped = len_before_rstrip - len(data)
if data:
self.output.write(data)
if spaces_rstripped:
# Add back a single space if 1 or more
# whitespace characters were stripped
self.output.write(' ')
def __str__(self):
return self.output.getvalue()
@enforce_types
def should_save_htmltotext(link: Link, out_dir: Optional[Path]=None, overwrite: Optional[bool]=False) -> bool:
if is_static_file(link.url):
return False
out_dir = out_dir or Path(link.link_dir)
if not overwrite and (out_dir / 'htmltotext.txt').exists():
return False
return SAVE_HTMLTOTEXT
@enforce_types
def save_htmltotext(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -> ArchiveResult:
"""extract search-indexing-friendly text from an HTML document"""
out_dir = Path(out_dir or link.link_dir)
output = "htmltotext.txt"
timer = TimedProgress(timeout, prefix=' ')
extracted_text = None
try:
extractor = HTMLTextExtractor()
document = get_html(link, out_dir)
if not document:
raise ArchiveError('htmltotext could not find HTML to parse for article text')
extractor.feed(document)
extractor.close()
extracted_text = str(extractor)
atomic_write(str(out_dir / output), extracted_text)
except (Exception, OSError) as err:
status = 'failed'
output = err
cmd = ['(internal) archivebox.extractors.htmltotext', './{singlefile,dom}.html']
finally:
timer.end()
return ArchiveResult(
cmd=cmd,
pwd=str(out_dir),
cmd_version=VERSION,
output=output,
status=status,
index_texts=[extracted_text] if extracted_text else [],
**timer.stats,
)