""" Everything related to parsing links from input sources. For a list of supported services, see the README.md. For examples of supported import formats see tests/. Link: { 'url': 'https://example.com/example/?abc=123&xyc=345#lmnop', 'timestamp': '1544212312.4234', 'title': 'Example.com Page Title', 'tags': 'abc,def', 'sources': [ 'output/sources/ril_export.html', 'output/sources/getpocket.com-1523422111.txt', 'output/sources/stdin-234234112312.txt' ] } """ import re import json from typing import Tuple, List, IO, Iterable from datetime import datetime import xml.etree.ElementTree as etree from config import TIMEOUT from util import ( htmldecode, str_between, URL_REGEX, check_url_parsing_invariants, TimedProgress, Link, enforce_types, ) @enforce_types def parse_links(source_file: str) -> Tuple[List[Link], str]: """parse a list of URLs with their metadata from an RSS feed, bookmarks export, or text file """ check_url_parsing_invariants() PARSERS = ( # Specialized parsers ('Pocket HTML', parse_pocket_html_export), ('Pinboard RSS', parse_pinboard_rss_export), ('Shaarli RSS', parse_shaarli_rss_export), ('Medium RSS', parse_medium_rss_export), # General parsers ('Netscape HTML', parse_netscape_html_export), ('Generic RSS', parse_rss_export), ('Generic JSON', parse_json_export), # Fallback parser ('Plain Text', parse_plain_text_export), ) timer = TimedProgress(TIMEOUT * 4) with open(source_file, 'r', encoding='utf-8') as file: for parser_name, parser_func in PARSERS: try: links = list(parser_func(file)) if links: timer.end() return links, parser_name except Exception as err: # Parsers are tried one by one down the list, and the first one # that succeeds is used. To see why a certain parser was not used # due to error or format incompatibility, uncomment this line: # print('[!] Parser {} failed: {} {}'.format(parser_name, err.__class__.__name__, err)) pass timer.end() return [], 'Failed to parse' ### Import Parser Functions @enforce_types def parse_pocket_html_export(html_file: IO[str]) -> Iterable[Link]: """Parse Pocket-format bookmarks export files (produced by getpocket.com/export/)""" html_file.seek(0) pattern = re.compile("^\\s*
  • (.+)
  • ", re.UNICODE) for line in html_file: # example line #
  • example title
  • match = pattern.search(line) if match: url = match.group(1).replace('http://www.readability.com/read?url=', '') # remove old readability prefixes to get original url time = datetime.fromtimestamp(float(match.group(2))) tags = match.group(3) title = match.group(4).replace(' — Readability', '').replace('http://www.readability.com/read?url=', '') yield Link( url=url, timestamp=str(time.timestamp()), title=title or None, tags=tags or '', sources=[html_file.name], ) @enforce_types def parse_json_export(json_file: IO[str]) -> Iterable[Link]: """Parse JSON-format bookmarks export files (produced by pinboard.in/export/, or wallabag)""" json_file.seek(0) links = json.load(json_file) json_date = lambda s: datetime.strptime(s, '%Y-%m-%dT%H:%M:%S%z') for link in links: # example line # {"href":"http:\/\/www.reddit.com\/r\/example","description":"title here","extended":"","meta":"18a973f09c9cc0608c116967b64e0419","hash":"910293f019c2f4bb1a749fb937ba58e3","time":"2014-06-14T15:51:42Z","shared":"no","toread":"no","tags":"reddit android"}] if link: # Parse URL url = link.get('href') or link.get('url') or link.get('URL') if not url: raise Exception('JSON must contain URL in each entry [{"url": "http://...", ...}, ...]') # Parse the timestamp ts_str = str(datetime.now().timestamp()) if link.get('timestamp'): # chrome/ff histories use a very precise timestamp ts_str = str(link['timestamp'] / 10000000) elif link.get('time'): ts_str = str(json_date(link['time'].split(',', 1)[0]).timestamp()) elif link.get('created_at'): ts_str = str(json_date(link['created_at']).timestamp()) elif link.get('created'): ts_str = str(json_date(link['created']).timestamp()) elif link.get('date'): ts_str = str(json_date(link['date']).timestamp()) elif link.get('bookmarked'): ts_str = str(json_date(link['bookmarked']).timestamp()) elif link.get('saved'): ts_str = str(json_date(link['saved']).timestamp()) # Parse the title title = None if link.get('title'): title = link['title'].strip() elif link.get('description'): title = link['description'].replace(' — Readability', '').strip() elif link.get('name'): title = link['name'].strip() yield Link( url=url, timestamp=ts_str, title=htmldecode(title) or None, tags=link.get('tags') or '', sources=[json_file.name], ) @enforce_types def parse_rss_export(rss_file: IO[str]) -> Iterable[Link]: """Parse RSS XML-format files into links""" rss_file.seek(0) items = rss_file.read().split('') items = items[1:] if items else [] for item in items: # example item: # # <![CDATA[How JavaScript works: inside the V8 engine]]> # Unread # https://blog.sessionstack.com/how-javascript-works-inside # https://blog.sessionstack.com/how-javascript-works-inside # Mon, 21 Aug 2017 14:21:58 -0500 # trailing_removed = item.split('', 1)[0] leading_removed = trailing_removed.split('', 1)[-1].strip() rows = leading_removed.split('\n') def get_row(key): return [r for r in rows if r.strip().startswith('<{}>'.format(key))][0] url = str_between(get_row('link'), '', '') ts_str = str_between(get_row('pubDate'), '', '') time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %z") title = str_between(get_row('title'), ' Iterable[Link]: """Parse Shaarli-specific RSS XML-format files into links""" rss_file.seek(0) entries = rss_file.read().split('')[1:] for entry in entries: # example entry: # # Aktuelle Trojaner-Welle: Emotet lauert in gefälschten Rechnungsmails | heise online # # https://demo.shaarli.org/?cEV4vw # 2019-01-30T06:06:01+00:00 # 2019-01-30T06:06:01+00:00 #

    Permalink

    ]]>
    #
    trailing_removed = entry.split('
    ', 1)[0] leading_removed = trailing_removed.strip() rows = leading_removed.split('\n') def get_row(key): return [r.strip() for r in rows if r.strip().startswith('<{}'.format(key))][0] title = str_between(get_row('title'), '', '').strip() url = str_between(get_row('link'), '') ts_str = str_between(get_row('published'), '', '') time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z") yield Link( url=url, timestamp=str(time.timestamp()), title=htmldecode(title) or None, tags='', sources=[rss_file.name], ) @enforce_types def parse_netscape_html_export(html_file: IO[str]) -> Iterable[Link]: """Parse netscape-format bookmarks export files (produced by all browsers)""" html_file.seek(0) pattern = re.compile("]*>(.+)", re.UNICODE | re.IGNORECASE) for line in html_file: # example line #
    example bookmark title match = pattern.search(line) if match: url = match.group(1) time = datetime.fromtimestamp(float(match.group(2))) title = match.group(3).strip() yield Link( url=url, timestamp=str(time.timestamp()), title=htmldecode(title) or None, tags='', sources=[html_file.name], ) @enforce_types def parse_pinboard_rss_export(rss_file: IO[str]) -> Iterable[Link]: """Parse Pinboard RSS feed files into links""" rss_file.seek(0) root = etree.parse(rss_file).getroot() items = root.findall("{http://purl.org/rss/1.0/}item") for item in items: url = item.find("{http://purl.org/rss/1.0/}link").text tags = item.find("{http://purl.org/dc/elements/1.1/}subject").text if item.find("{http://purl.org/dc/elements/1.1/}subject") else None title = item.find("{http://purl.org/rss/1.0/}title").text.strip() if item.find("{http://purl.org/rss/1.0/}title").text.strip() else None ts_str = item.find("{http://purl.org/dc/elements/1.1/}date").text if item.find("{http://purl.org/dc/elements/1.1/}date").text else None # Pinboard includes a colon in its date stamp timezone offsets, which # Python can't parse. Remove it: if ts_str and ts_str[-3:-2] == ":": ts_str = ts_str[:-3]+ts_str[-2:] if ts_str: time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z") else: time = datetime.now() yield Link( url=url, timestamp=str(time.timestamp()), title=htmldecode(title) or None, tags=tags or '', sources=[rss_file.name], ) @enforce_types def parse_medium_rss_export(rss_file: IO[str]) -> Iterable[Link]: """Parse Medium RSS feed files into links""" rss_file.seek(0) root = etree.parse(rss_file).getroot() items = root.find("channel").findall("item") for item in items: url = item.find("link").text title = item.find("title").text.strip() ts_str = item.find("pubDate").text time = datetime.strptime(ts_str, "%a, %d %b %Y %H:%M:%S %Z") yield Link( url=url, timestamp=str(time.timestamp()), title=htmldecode(title) or None, tags='', sources=[rss_file.name], ) @enforce_types def parse_plain_text_export(text_file: IO[str]) -> Iterable[Link]: """Parse raw links from each line in a text file""" text_file.seek(0) for line in text_file.readlines(): urls = re.findall(URL_REGEX, line) if line.strip() else () for url in urls: yield Link( url=url, timestamp=str(datetime.now().timestamp()), title=None, tags='', sources=[text_file.name], )