mirror of
https://github.com/nathom/streamrip.git
synced 2024-09-19 11:18:45 -04:00
Manually yield for better performance (#648)
This commit is contained in:
parent
affdaa8fab
commit
8757956636
1 changed files with 15 additions and 9 deletions
|
@ -17,6 +17,7 @@ from typing import Any, Callable, Optional
|
||||||
import aiofiles
|
import aiofiles
|
||||||
import aiohttp
|
import aiohttp
|
||||||
import m3u8
|
import m3u8
|
||||||
|
import requests
|
||||||
from Cryptodome.Cipher import AES, Blowfish
|
from Cryptodome.Cipher import AES, Blowfish
|
||||||
from Cryptodome.Util import Counter
|
from Cryptodome.Util import Counter
|
||||||
|
|
||||||
|
@ -41,7 +42,7 @@ class Downloadable(ABC):
|
||||||
session: aiohttp.ClientSession
|
session: aiohttp.ClientSession
|
||||||
url: str
|
url: str
|
||||||
extension: str
|
extension: str
|
||||||
chunk_size = 1024
|
chunk_size = 2**17
|
||||||
_size: Optional[int] = None
|
_size: Optional[int] = None
|
||||||
|
|
||||||
async def download(self, path: str, callback: Callable[[int], Any]):
|
async def download(self, path: str, callback: Callable[[int], Any]):
|
||||||
|
@ -71,19 +72,24 @@ class BasicDownloadable(Downloadable):
|
||||||
self.extension = extension
|
self.extension = extension
|
||||||
self._size = None
|
self._size = None
|
||||||
|
|
||||||
async def _download(self, path: str, callback: Callable[[int], None]):
|
async def _download(self, path: str, callback):
|
||||||
async with self.session.get(self.url, allow_redirects=True) as response:
|
# Attempt to fix async performance issues by manually and infrequently
|
||||||
response.raise_for_status()
|
# yielding to event loop selector
|
||||||
async with aiofiles.open(path, "wb") as file:
|
counter = 0
|
||||||
async for chunk in response.content.iter_chunked(self.chunk_size):
|
yield_every = 16
|
||||||
await file.write(chunk)
|
with open(path, "wb") as file:
|
||||||
# typically a bar.update()
|
with requests.get(self.url, allow_redirects=True, stream=True) as resp:
|
||||||
|
for chunk in resp.iter_content(chunk_size=self.chunk_size):
|
||||||
|
file.write(chunk)
|
||||||
callback(len(chunk))
|
callback(len(chunk))
|
||||||
|
if counter % yield_every == 0:
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
counter += 1
|
||||||
|
|
||||||
|
|
||||||
class DeezerDownloadable(Downloadable):
|
class DeezerDownloadable(Downloadable):
|
||||||
is_encrypted = re.compile("/m(?:obile|edia)/")
|
is_encrypted = re.compile("/m(?:obile|edia)/")
|
||||||
chunk_size = 2048 * 3
|
# chunk_size = 2048 * 3
|
||||||
|
|
||||||
def __init__(self, session: aiohttp.ClientSession, info: dict):
|
def __init__(self, session: aiohttp.ClientSession, info: dict):
|
||||||
logger.debug("Deezer info for downloadable: %s", info)
|
logger.debug("Deezer info for downloadable: %s", info)
|
||||||
|
|
Loading…
Reference in a new issue