fix source filenames missing domain

This commit is contained in:
Nick Sweeting 2020-08-18 04:39:29 -04:00
parent ec1cce283e
commit 12f3977da5

View file

@ -550,7 +550,7 @@ def add(urls: Union[str, List[str]],
if new_links and depth == 1:
log_crawl_started(new_links)
for new_link in new_links:
downloaded_file = save_file_as_source(new_link.url, filename='{ts}-crawl-{basename}.txt', out_dir=out_dir)
downloaded_file = save_file_as_source(new_link.url, filename=f'{new_link.timestamp}-crawl-{new_link.domain}.txt', out_dir=out_dir)
new_links_depth += parse_links_from_source(downloaded_file)
imported_links = new_links + new_links_depth