diff --git a/Dockerfile b/Dockerfile index 8cf2da30..bb750721 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,13 +79,13 @@ WORKDIR "$CODE_DIR" ENV PATH="${PATH}:$VENV_PATH/bin" RUN python -m venv --clear --symlinks "$VENV_PATH" \ && pip install --upgrade --quiet pip setuptools -ADD ./pip_dist/archivebox.egg-info/requires.txt "$CODE_DIR/pip_dist/archivebox.egg-info/requires.txt" +ADD "./setup.py" "$CODE_DIR/" +ADD "./README.md" "./package.json" "$CODE_DIR/archivebox/" RUN apt-get update -qq \ && apt-get install -qq -y --no-install-recommends \ build-essential python-dev python3-dev \ - # && pip install --upgrade pip \ - && grep -B 1000 -E '^$' "$CODE_DIR/pip_dist/archivebox.egg-info/requires.txt" | pip install --quiet -r /dev/stdin \ - && pip install --quiet "sonic-client==0.0.5" \ + && python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.install_requires + result.extras_require["sonic"]))' > /tmp/requirements.txt \ + && pip install --quiet -r /tmp/requirements.txt \ && apt-get purge -y build-essential python-dev python3-dev \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/* diff --git a/archivebox/config.py b/archivebox/config.py index 3d48344f..b8acb3f7 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -1075,6 +1075,11 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG, call_command("migrate", interactive=False, verbosity=0) else: django.setup() + + # Enable WAL mode in sqlite3 + from django.db import connection + with connection.cursor() as cursor: + cursor.execute("PRAGMA journal_mode=wal;") if check_db: sql_index_path = Path(output_dir) / SQL_INDEX_FILENAME diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py index 853a4863..e73c93d9 100644 --- a/archivebox/core/settings.py +++ b/archivebox/core/settings.py @@ -107,9 +107,6 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': DATABASE_NAME, - 'OPTIONS': { - 'init_command': 'PRAGMA journal_mode=wal;', - } } } diff --git a/setup.py b/setup.py index 962db8d8..3ab4f238 100755 --- a/setup.py +++ b/setup.py @@ -27,6 +27,47 @@ PACKAGE_DIR = ROOT_DIR / PKG_NAME README = (PACKAGE_DIR / "README.md").read_text(encoding='utf-8', errors='ignore') VERSION = json.loads((PACKAGE_DIR / "package.json").read_text().strip())['version'] +PYTHON_REQUIRES = ">=3.7" +SETUP_REQUIRES = ["wheel"] +INSTALL_REQUIRES = [ + # only add things here that have corresponding apt python3-packages available + # anything added here also needs to be added to our package dependencies in + # stdeb.cfg (apt), archivebox.rb (brew), Dockerfile, etc. + # if there is no apt python3-package equivalent, then vendor it instead in + # ./archivebox/vendor/ + "requests>=2.24.0", + "atomicwrites>=1.4.0", + "mypy-extensions>=0.4.3", + "django>=3.1.3", + "django-extensions>=3.0.3", + "dateparser", + "ipython", + "youtube-dl", + "python-crontab>=2.5.1", + "croniter>=0.3.34", + "w3lib>=1.22.0", +] +EXTRAS_REQUIRE = { + 'sonic': [ + "sonic-client>=0.0.5", + ], + 'dev': [ + "setuptools", + "twine", + "wheel", + "flake8", + "ipdb", + "mypy", + "django-stubs", + "sphinx", + "sphinx-rtd-theme", + "recommonmark", + "pytest", + "bottle", + "stdeb", + ], +} + # To see when setup.py gets called (uncomment for debugging): # import sys # print(PACKAGE_DIR, f" (v{VERSION})") @@ -36,7 +77,9 @@ VERSION = json.loads((PACKAGE_DIR / "package.json").read_text().strip())['versio class DisabledTestCommand(test): def run(self): # setup.py test is deprecated, disable it here by force so stdeb doesnt run it - print('Use the ./bin/test.sh script to run tests, not setup.py test.') + print() + print('[X] Running tests via setup.py test is deprecated.') + print(' Hint: Use the ./bin/test.sh script or pytest instead') setuptools.setup( @@ -50,45 +93,10 @@ setuptools.setup( long_description_content_type="text/markdown", url=REPO_URL, project_urls=PROJECT_URLS, - python_requires=">=3.7", - setup_requires=[ - "wheel", - ], - install_requires=[ - # only add things here that have corresponding apt python3-packages available - # anything added here also needs to be added to our package dependencies in - # stdeb.cfg (apt), archivebox.rb (brew), Dockerfile, etc. - # if there is no apt python3-package equivalent, then vendor it instead in - # ./archivebox/vendor/ - "requests==2.24.0", - "atomicwrites==1.4.0", - "mypy-extensions==0.4.3", - "django==3.1.3", - "django-extensions==3.0.3", - "dateparser", - "ipython", - "youtube-dl", - "python-crontab==2.5.1", - "croniter==0.3.34", - "w3lib==1.22.0", - ], - extras_require={ - 'dev': [ - "setuptools", - "twine", - "wheel", - "flake8", - "ipdb", - "mypy", - "django-stubs", - "sphinx", - "sphinx-rtd-theme", - "recommonmark", - "pytest", - "bottle", - "stdeb", - ], - }, + python_requires=PYTHON_REQUIRES, + setup_requires=SETUP_REQUIRES, + install_requires=INSTALL_REQUIRES, + extras_require=EXTRAS_REQUIRE, packages=[PKG_NAME], include_package_data=True, # see MANIFEST.in entry_points={