diff --git a/archivebox/playbooks/install_npm.yml b/archivebox/playbooks/install_npm.yml new file mode 100644 index 00000000..57d94450 --- /dev/null +++ b/archivebox/playbooks/install_npm.yml @@ -0,0 +1,94 @@ +--- +- name: "Install node, npm, and npx" + hosts: localhost + gather_facts: no + vars: + DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4' + LIB_DIR: '{{DATA_DIR}}/lib' + LIB_DIR_BIN: '{{LIB_DIR}}/bin' + LIB_DIR_NPM: '{{LIB_DIR}}/npm' + LIB_DIR_NPM_BIN: '{{LIB_DIR_NPM}}/node_modules/.bin' + MIN_NODE_VERSION: '22.7.0' + MIN_NPM_VERSION: '10.8.3' + state: 'latest' + packages: [] + tasks: + # - package: update_cache=yes + # when: ansible_facts['os_family'] == "Debian" + + - name: Make sure lib folders exist + file: + path: '{{item}}' + state: directory + recurse: true + loop: + - '{{LIB_DIR_NPM_BIN}}' + - '{{LIB_DIR_BIN}}' + + ################################################################################### + - name: "Install npm on system: [node, npm]" + ansible.builtin.package: + name: node + state: "{{state}}" + + - name: "Install npm packages: {{packages}}" + community.general.npm: + name: '{{item}}' + state: "{{state}}" + path: '{{LIB_DIR_NPM}}' + loop: '{{packages}}' + + ################################################################################### + - name: Get installed node abspath + command: 'which node' + register: NODE_ABSPATH_FULL + changed_when: False + + - name: Get installed node version + command: 'node --version' + register: NODE_VERSION_FULL + changed_when: False + + - name: Get installed npm abspath + command: 'which npm' + register: NPM_ABSPATH_FULL + changed_when: False + + - name: Get installed npm version + command: 'npm --version' + register: NPM_VERSION_FULL + changed_when: False + + - set_fact: + NODE_ABSPATH: "{{NODE_ABSPATH_FULL.stdout_lines|first}}" + NODE_VERSION: "{{NODE_VERSION_FULL.stdout_lines|first|regex_replace('^.*?v?([\\d\\.]+).*$', '\\1')}}" + NPM_ABSPATH: "{{NPM_ABSPATH_FULL.stdout_lines|first}}" + NPM_VERSION: "{{NPM_VERSION_FULL.stdout_lines|first|regex_replace('^.*?v?([\\d\\.]+).*$', '\\1')}}" + + - name: Check that installed Chrome matches expected version + assert: + that: + - NODE_VERSION is version(MIN_NODE_VERSION, '<=') + - NPM_VERSION is version(MIN_NPM_VERSION, '<=') + quiet: true + + ################################################################################### + - set_fact: + NODE_BINARIES: + node: + abspath: "{{NODE_ABSPATH}}" + version: "{{NODE_VERSION}}" + npm: + abspath: "{{NPM_ABSPATH}}" + version: "{{NPM_VERSION}}" + NODE_BINPROVIDERS: + npm: + installer_abspath: "{{NPM_ABSPATH}}" + installer_version: "{{NPM_VERSION}}" + PATH: "{{LIB_DIR_NPM_BIN}}" + - set_fact: + BINARIES: "{{ BINARIES | default({}) | combine(NODE_BINARIES) }}" + BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(NODE_BINPROVIDERS) }}" + + - debug: + msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}" diff --git a/archivebox/playbooks/install_package.yml b/archivebox/playbooks/install_package.yml new file mode 100644 index 00000000..e0257962 --- /dev/null +++ b/archivebox/playbooks/install_package.yml @@ -0,0 +1,63 @@ +--- +- name: "Install using apt/brew/pkg/yum/etc." + hosts: localhost + gather_facts: no + vars: + DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4' + LIB_DIR: '{{DATA_DIR}}/lib' + LIB_DIR_BIN: '{{LIB_DIR}}/bin' + state: 'latest' + install_packages: + bash: + packages: ['bash'] + # curl: ['curl'] + # wget: ['wget'] + # git: ['git'] + PACKAGE_BINPROVIDERS: {} + PACKAGE_BINARIES: {} + tasks: + # - package: update_cache=yes + # when: ansible_facts['os_family'] == "Debian" + + - name: Make sure lib folders exist + file: + path: '{{LIB_DIR_BIN}}' + state: directory + recurse: true + + - name: Get ansible binary abspath + command: 'which ansible' + register: ANSIBLE_INSTALLER_ABSPATH_FULL + changed_when: false + + - set_fact: + all_packages: "{{install_packages|dictsort|map(attribute='1')|map(attribute='packages')|flatten|unique}}" + + ################################################################################### + - name: "Install system packages: {{all_packages}}" + ansible.builtin.package: + name: "{{item}}" + state: "{{state}}" + loop: "{{all_packages}}" + + ################################################################################### + + + - set_fact: + PACKAGE_BINPROVIDERS: + ansible_package: + installer_abspath: "{{ANSIBLE_INSTALLER_ABSPATH_FULL.stdout}}" + installer_version: "{{ansible_version.full}}" + PATH: "{{ LIB_DIR_BIN }}" + + - set_fact: + BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(PACKAGE_BINPROVIDERS) }}" + + - debug: + msg: "{{ {'BINPROVIDERS': BINPROVIDERS} }}" + + +- import_playbook: load_binaries.yml + vars: + load_binaries: + bash: diff --git a/archivebox/playbooks/install_pip.yml b/archivebox/playbooks/install_pip.yml new file mode 100644 index 00000000..97a01e33 --- /dev/null +++ b/archivebox/playbooks/install_pip.yml @@ -0,0 +1,84 @@ +--- + +- import_playbook: install_package.yml + vars: + install_packages: + python: {packages: ['python3']} + state: 'latest' + +- import_playbook: load_binaries.yml + vars: + load_binaries: + python: {bin_name: 'python3', version_cmd: 'python3 --version'} + pip: {bin_name: 'pip3', version_cmd: 'pip3 --version'} + +- name: "Install python, pip, and pipx" + hosts: localhost + gather_facts: no + vars: + DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4' + LIB_DIR: '{{DATA_DIR}}/lib' + LIB_DIR_BIN: '{{LIB_DIR}}/bin' + LIB_DIR_PIP: '{{LIB_DIR}}/pip' + LIB_DIR_PIP_BIN: '{{LIB_DIR_PIP}}/venv/bin' + MIN_PYTHON_VERSION: '3.11.9' + MIN_PIP_VERSION: '24.2' + state: 'latest' + install_pip: + pip: + bin_name: 'pip' + packages: ['pip', 'setuptools'] + # ytdlp: + # bin_name: 'yt-dlp' + # packages: ['yt-dlp'] + tasks: + + - name: Make sure lib folders exist + file: + path: '{{item}}' + state: directory + recurse: true + loop: + - '{{LIB_DIR_PIP}}' + - '{{LIB_DIR_BIN}}' + + ################################################################################### + + - debug: + msg: + - "{{BINARIES.python}}" + - "{{BINARIES.pip}}" + + - assert: + that: + - BINARIES.python.version is version(MIN_PYTHON_VERSION, '<=') + - BINARIES.pip.version is version(MIN_PIP_VERSION, '<=') + quiet: true + + - name: "Install pip packages: {{install_pip}}" + ansible.builtin.pip: + name: '{{item}}' + state: "{{state}}" + virtualenv: '{{LIB_DIR_PIP}}/venv' + virtualenv_python: "{{BINARIES.python.abspath}}" + virtualenv_site_packages: yes + loop: "{{install_pip|dictsort|map(attribute='1')|map(attribute='packages')|flatten}}" + + + ################################################################################### + - set_fact: + PIP_BINPROVIDERS: + pip: + installer_abspath: "{{BINARIES.pip.abspath}}" + installer_version: "{{BINARIES.pip.version}}" + PATH: "{{LIB_DIR_PIP_BIN}}" + - set_fact: + BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(PIP_BINPROVIDERS) }}" + + - debug: + msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}" + +- import_playbook: load_binaries.yml + vars: + load_binaries: + bash: diff --git a/archivebox/playbooks/install_puppeteer.yml b/archivebox/playbooks/install_puppeteer.yml new file mode 100644 index 00000000..9618e4cd --- /dev/null +++ b/archivebox/playbooks/install_puppeteer.yml @@ -0,0 +1,91 @@ +--- +- import_playbook: install_npm.yml + vars: + packages: + - 'puppeteer' + - '@puppeteer/browsers' + state: 'latest' + +- name: "Install puppeteer, puppeteer/browsers, and chrome" + hosts: localhost + gather_facts: no + vars: + DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4' + LIB_DIR: '{{DATA_DIR}}/lib' + LIB_DIR_NPM: '{{LIB_DIR}}/npm' + LIB_DIR_NPM_BIN: '{{LIB_DIR_NPM}}/node_modules/.bin' + LIB_DIR_BROWSERS: '{{LIB_DIR}}/browsers' + LIB_DIR_BIN: '{{LIB_DIR}}/bin' + CHROME_RELEASE_CHANNEL: 'chrome@stable' + CHROME_VERSION_MIN: '128.0.6613.137' + environment: + PATH: "{{LIB_DIR_NPM_BIN}}:{{ ansible_env.PATH }}" + tasks: + - name: Make sure prerequisite folders exist + file: + path: '{{item}}' + state: directory + recurse: true + loop: + - '{{LIB_DIR_BROWSERS}}' + - '{{LIB_DIR_BIN}}' + + - name: Make sure Node and NPM are installed + assert: + that: + - BINARIES.node.version + - BINARIES.npm.version + quiet: true + + - name: Get installed puppeteer version + command: 'puppeteer --version' + register: PUPPETEER_VERSION_FULL + changed_when: False + + ################################################################################### + - name: Install Chrome browser using puppeteer/browsers + command: 'npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}} --path {{LIB_DIR_BROWSERS}}' + register: CHROME_VERSION_FULL + # -> 'chrome@128.0.6613.137 /data/lib/browsers/chrome/linux_arm-128.0.6613.138/chrome-linux-arm64/...' + changed_when: CHROME_VERSION_MIN not in CHROME_VERSION_FULL.stdout + + ################################################################################### + - set_fact: + PUPPETEER_ABSPATH: "{{LIB_DIR_NPM_BIN}}/puppeteer" + PUPPETEER_VERSION: "{{ PUPPETEER_VERSION_FULL.stdout_lines|first }}" + CHROME_ABSPATH: "{{ CHROME_VERSION_FULL.stdout_lines|last|split(' ', 1)|last }}" + CHROME_VERSION: "{{ CHROME_VERSION_FULL.stdout_lines|last|split('@', 1)|last|split(' ', 1)|first }}" + + - name: Check that installed Chrome matches expected version + assert: + that: CHROME_VERSION_MIN is version(CHROME_VERSION, '>=') + quiet: true + + - name: Create ./bin/chrome symlink to ./browsers/chrome/... binary + copy: + content: | + #!/bin/bash + exec '{{CHROME_ABSPATH}}' "$@" + dest: "{{LIB_DIR_BIN}}/chrome" + - file: + path: "{{LIB_DIR_BIN}}/chrome" + mode: u+rx,g-rx,o-rwx + state: 'file' + + ################################################################################### + - set_fact: + PUPPETEER_DEPENDENCIES: + node: "{{BINARIES.node}}" + npm: "{{BINARIES.npm}}" + PUPPETEER_BINARIES: + puppeteer: + abspath: "{{PUPPETEER_ABSPATH}}" + version: "{{PUPPETEER_VERSION}}" + chrome: + abspath: "{{CHROME_ABSPATH}}" + version: "{{CHROME_VERSION}}" + - set_fact: + BINARIES: "{{ BINARIES | combine(PUPPETEER_BINARIES) }}" + + - debug: + msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}" diff --git a/archivebox/playbooks/install_singlefile.yml b/archivebox/playbooks/install_singlefile.yml new file mode 100644 index 00000000..aa1c0503 --- /dev/null +++ b/archivebox/playbooks/install_singlefile.yml @@ -0,0 +1,35 @@ +--- +- name: Install node and single-file-cli + hosts: localhost + gather_facts: no + vars: + DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4' + NPM_BIN_DIR: '{{DATA_DIR}}/node_modules/.bin' + MIN_BIN_VERSION: '1.1.54' + tasks: + # - package: update_cache=yes + # when: ansible_facts['os_family'] == "Debian" + + - ansible.builtin.package: + name: node + state: present + + - community.general.npm: + name: 'single-file-cli' + version: '{{ MIN_BIN_VERSION }}' + path: '{{ NPM_BIN_DIR }}/../..' + + - command: '{{ NPM_BIN_DIR }}/single-file --version' + register: BIN_VERSION_FULL + changed_when: False + + - set_fact: + BIN_VERSION: "{{ BIN_VERSION_FULL.stdout_lines|first }}" + + - debug: + msg: + - "BIN_ABSPATH={{ NPM_BIN_DIR }}/single-file" + - "BIN_VERSION={{ BIN_VERSION }}" + + - assert: + that: BIN_VERSION is version(MIN_BIN_VERSION, '==', version_type='semver') diff --git a/archivebox/playbooks/install_ytdlp.yml b/archivebox/playbooks/install_ytdlp.yml new file mode 100644 index 00000000..2d1357ab --- /dev/null +++ b/archivebox/playbooks/install_ytdlp.yml @@ -0,0 +1,39 @@ +--- +- import_playbook: install_package.yml + vars: + install_package: + ffmpeg: {packages: ['ffmpeg']} + state: 'latest' + +- import_playbook: install_pip.yml + vars: + install_pip: + ytdlp: {packages: ['yt-dlp']} + state: 'latest' + +- import_playbook: load_binaries.yml + vars: + load_binaries: + ffmpeg: {bin_name: 'ffmpeg', version_cmd: 'ffmpeg -version'} + ytdlp: {bin_name: 'yt-dlp', version_cmd: 'yt-dlp --version'} + + +- name: "Install YT-DLP" + hosts: localhost + gather_facts: no + vars: + YTDLP_VERSION_MIN: '2024.8.6' + tasks: + - name: Make sure Python and Pip are installed + assert: + that: + - BINARIES.python.version + - BINARIES.pip.version + - BINARIES.ffmpeg.version + - BINARIES.ytdlp.version + quiet: true + + - name: Check that installed YT-DLP matches expected version + assert: + that: YTDLP_VERSION_MIN is version(BINARIES.ytdlp.version, '>=') + quiet: true diff --git a/archivebox/playbooks/load_binaries.yml b/archivebox/playbooks/load_binaries.yml new file mode 100644 index 00000000..7f0ef315 --- /dev/null +++ b/archivebox/playbooks/load_binaries.yml @@ -0,0 +1,71 @@ +--- +- name: Get binary abspaths and versions and add to BINARIES + hosts: localhost + gather_facts: false + vars: + DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4' + LIB_DIR: '{{DATA_DIR}}/lib' + LIB_DIR_BIN: '{{LIB_DIR}}/bin' + load_binaries: + bash: + bin_name: bash + version_cmd: 'bash --version' + # ffmpeg: + # version_cmd: 'ffmpeg -version' + # ytdlp: + # bin_name: ytdlp + # curl: + # wget: + # bin_name: wget + # version_cmd: 'wget --version' + tasks: + - name: Make sure lib folders exist + file: + path: '{{LIB_DIR_BIN}}' + state: directory + recurse: true + + ################################################################################### + - name: Get installed binary abspath + command: "which {{item.1.bin_name|default(item.0)}}" + register: ABSPATHS_FULL + changed_when: false + ignore_errors: true + loop: "{{load_binaries|dictsort}}" + + - name: Get installed binary version + command: "{{item.1.version_cmd|default((item.1.bin_name|default(item.0)) + ' --version')}}" + register: VERSIONS_FULL + changed_when: false + ignore_errors: true + loop: "{{load_binaries|dictsort}}" + + - name: Symlink installed binary into lib bin folder + file: + src: "{{ item }}" + dest: "{{ LIB_DIR_BIN }}/{{ item|basename }}" + state: link + loop: "{{ABSPATHS_FULL.results|map(attribute='stdout_lines')|map('first')}}" + + - set_fact: + BINARY_ABSPATHS: "{{ BINARY_ABSPATHS | default({}) | combine({item.0: item.1}) }}" + loop: "{{ keys | zip(values) }}" + vars: + keys: "{{load_binaries|dictsort|map(attribute='0')}}" + values: "{{ABSPATHS_FULL.results|map(attribute='stdout_lines')|map('first')}}" + + - set_fact: + BINARY_VERSIONS: "{{ BINARY_VERSIONS | default({}) | combine({item.0: item.1}) }}" + loop: "{{ keys | zip(values) }}" + vars: + keys: "{{load_binaries|dictsort|map(attribute='0')}}" + values: "{{VERSIONS_FULL.results|map(attribute='stdout_lines')|map('first')|map('regex_replace', '^.*?v?([\\d\\.]+).*$', '\\1')}}" + + - set_fact: + BINARIES: "{{ BINARIES | default({}) | combine({item: { 'abspath': BINARY_ABSPATHS[item], 'version': BINARY_VERSIONS[item] }}) }}" + loop: "{{load_binaries|dictsort|map(attribute='0')}}" + + ################################################################################### + + - debug: + msg: "{{ {'BINARIES': BINARIES} }}"