ArchiveBox/archivebox/parsers/generic_rss.py
jim winstead 9f462a87a8 Use feedparser for RSS parsing in generic_rss and pinboard_rss parsers
The feedparser packages has 20 years of history and is very good at parsing
RSS and Atom, so use that instead of ad-hoc regex and XML parsing.

The medium_rss and shaarli_rss parsers weren't touched because they are
probably unnecessary. (The special parse for pinboard is just needing because
of how tags work.)

Doesn't include tests because I haven't figured out how to run them in the
docker development setup.

Fixes #1171
2024-03-01 11:25:45 -08:00

46 lines
1.1 KiB
Python

__package__ = 'archivebox.parsers'
from typing import IO, Iterable
from time import mktime
from feedparser import parse as feedparser
from ..index.schema import Link
from ..util import (
htmldecode,
enforce_types
)
@enforce_types
def parse_generic_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
"""Parse RSS XML-format files into links"""
rss_file.seek(0)
feed = feedparser(rss_file.read())
for item in feed.entries:
url = item.link
title = item.title
time = mktime(item.updated_parsed)
try:
tags = ','.join(map(lambda tag: tag.term, item.tags))
except AttributeError:
tags = ''
if url is None:
# Yielding a Link with no URL will
# crash on a URL validation assertion
continue
yield Link(
url=htmldecode(url),
timestamp=str(time),
title=htmldecode(title) or None,
tags=tags,
sources=[rss_file.name],
)
KEY = 'rss'
NAME = 'Generic RSS'
PARSER = parse_generic_rss_export