add ansible playbooks

archivebox/playbooks/install_npm.yml

@@ -0,0 +1,94 @@
+---
+- name: "Install node, npm, and npx"
+  hosts: localhost
+  gather_facts: no
+  vars:
+    DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4'
+    LIB_DIR: '{{DATA_DIR}}/lib'
+    LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+    LIB_DIR_NPM: '{{LIB_DIR}}/npm'
+    LIB_DIR_NPM_BIN: '{{LIB_DIR_NPM}}/node_modules/.bin'
+    MIN_NODE_VERSION: '22.7.0'
+    MIN_NPM_VERSION: '10.8.3'
+    state: 'latest'
+    packages: []
+  tasks:
+    # - package: update_cache=yes
+    #   when: ansible_facts['os_family'] == "Debian"
+
+    - name: Make sure lib folders exist
+      file:
+        path: '{{item}}'
+        state: directory
+        recurse: true
+      loop:
+        - '{{LIB_DIR_NPM_BIN}}'
+        - '{{LIB_DIR_BIN}}'
+
+    ###################################################################################
+    - name: "Install npm on system: [node, npm]"
+      ansible.builtin.package:
+        name: node
+        state: "{{state}}"
+
+    - name: "Install npm packages: {{packages}}"
+      community.general.npm:
+        name: '{{item}}'
+        state: "{{state}}"
+        path: '{{LIB_DIR_NPM}}'
+      loop: '{{packages}}'
+
+    ###################################################################################
+    - name: Get installed node abspath
+      command: 'which node'
+      register: NODE_ABSPATH_FULL
+      changed_when: False
+    
+    - name: Get installed node version
+      command: 'node --version'
+      register: NODE_VERSION_FULL
+      changed_when: False
+    
+    - name: Get installed npm abspath
+      command: 'which npm'
+      register: NPM_ABSPATH_FULL
+      changed_when: False
+
+    - name: Get installed npm version
+      command: 'npm --version'
+      register: NPM_VERSION_FULL
+      changed_when: False
+
+    - set_fact:
+        NODE_ABSPATH: "{{NODE_ABSPATH_FULL.stdout_lines|first}}"
+        NODE_VERSION: "{{NODE_VERSION_FULL.stdout_lines|first|regex_replace('^.*?v?([\\d\\.]+).*$', '\\1')}}"
+        NPM_ABSPATH: "{{NPM_ABSPATH_FULL.stdout_lines|first}}"
+        NPM_VERSION: "{{NPM_VERSION_FULL.stdout_lines|first|regex_replace('^.*?v?([\\d\\.]+).*$', '\\1')}}"
+
+    - name: Check that installed Chrome matches expected version
+      assert:
+        that:
+          - NODE_VERSION is version(MIN_NODE_VERSION, '<=')
+          - NPM_VERSION is version(MIN_NPM_VERSION, '<=')
+        quiet: true
+
+    ###################################################################################
+    - set_fact:
+        NODE_BINARIES:
+          node:
+            abspath: "{{NODE_ABSPATH}}"
+            version: "{{NODE_VERSION}}"
+          npm:
+            abspath: "{{NPM_ABSPATH}}"
+            version: "{{NPM_VERSION}}"
+        NODE_BINPROVIDERS:
+          npm:
+            installer_abspath: "{{NPM_ABSPATH}}"
+            installer_version: "{{NPM_VERSION}}"
+            PATH: "{{LIB_DIR_NPM_BIN}}"
+    - set_fact:
+        BINARIES: "{{ BINARIES | default({}) | combine(NODE_BINARIES) }}"
+        BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(NODE_BINPROVIDERS) }}"
+
+    - debug:
+        msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/playbooks/install_package.yml

@@ -0,0 +1,63 @@
+---
+- name: "Install using apt/brew/pkg/yum/etc."
+  hosts: localhost
+  gather_facts: no
+  vars:
+    DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4'
+    LIB_DIR: '{{DATA_DIR}}/lib'
+    LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+    state: 'latest'
+    install_packages:
+      bash:
+        packages: ['bash']
+      # curl: ['curl']
+      # wget: ['wget']
+      # git: ['git']
+    PACKAGE_BINPROVIDERS: {}
+    PACKAGE_BINARIES: {}
+  tasks:
+    # - package: update_cache=yes
+      # when: ansible_facts['os_family'] == "Debian"
+
+    - name: Make sure lib folders exist
+      file:
+        path: '{{LIB_DIR_BIN}}'
+        state: directory
+        recurse: true
+
+    - name: Get ansible binary abspath
+      command: 'which ansible'
+      register: ANSIBLE_INSTALLER_ABSPATH_FULL
+      changed_when: false
+
+    - set_fact:
+        all_packages: "{{install_packages|dictsort|map(attribute='1')|map(attribute='packages')|flatten|unique}}"
+
+    ###################################################################################
+    - name: "Install system packages: {{all_packages}}"
+      ansible.builtin.package:
+        name: "{{item}}"
+        state: "{{state}}"
+      loop: "{{all_packages}}"
+
+    ###################################################################################
+    
+
+    - set_fact:
+        PACKAGE_BINPROVIDERS:
+          ansible_package:
+            installer_abspath: "{{ANSIBLE_INSTALLER_ABSPATH_FULL.stdout}}"
+            installer_version: "{{ansible_version.full}}"
+            PATH: "{{ LIB_DIR_BIN }}"
+
+    - set_fact:
+        BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(PACKAGE_BINPROVIDERS) }}"
+
+    - debug:
+        msg: "{{ {'BINPROVIDERS': BINPROVIDERS} }}"
+
+
+- import_playbook: load_binaries.yml
+  vars:
+    load_binaries:
+      bash:

archivebox/playbooks/install_pip.yml

@@ -0,0 +1,84 @@
+---
+
+- import_playbook: install_package.yml
+  vars:
+    install_packages:
+      python: {packages: ['python3']}
+    state: 'latest'
+
+- import_playbook: load_binaries.yml
+  vars:
+    load_binaries:
+      python: {bin_name: 'python3', version_cmd: 'python3 --version'}
+      pip: {bin_name: 'pip3', version_cmd: 'pip3 --version'}
+
+- name: "Install python, pip, and pipx"
+  hosts: localhost
+  gather_facts: no
+  vars:
+    DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4'
+    LIB_DIR: '{{DATA_DIR}}/lib'
+    LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+    LIB_DIR_PIP: '{{LIB_DIR}}/pip'
+    LIB_DIR_PIP_BIN: '{{LIB_DIR_PIP}}/venv/bin'
+    MIN_PYTHON_VERSION: '3.11.9'
+    MIN_PIP_VERSION: '24.2'
+    state: 'latest'
+    install_pip:
+      pip:
+        bin_name: 'pip'
+        packages: ['pip', 'setuptools']
+      # ytdlp:
+      #   bin_name: 'yt-dlp'
+      #   packages: ['yt-dlp']
+  tasks:
+
+    - name: Make sure lib folders exist
+      file:
+        path: '{{item}}'
+        state: directory
+        recurse: true
+      loop:
+        - '{{LIB_DIR_PIP}}'
+        - '{{LIB_DIR_BIN}}'
+
+    ###################################################################################
+    
+    - debug:
+        msg:
+          - "{{BINARIES.python}}"
+          - "{{BINARIES.pip}}"
+
+    - assert:
+        that:
+          - BINARIES.python.version is version(MIN_PYTHON_VERSION, '<=')
+          - BINARIES.pip.version is version(MIN_PIP_VERSION, '<=')
+        quiet: true
+    
+    - name: "Install pip packages: {{install_pip}}"
+      ansible.builtin.pip:
+        name: '{{item}}'
+        state: "{{state}}"
+        virtualenv: '{{LIB_DIR_PIP}}/venv'
+        virtualenv_python: "{{BINARIES.python.abspath}}"
+        virtualenv_site_packages: yes
+      loop: "{{install_pip|dictsort|map(attribute='1')|map(attribute='packages')|flatten}}"
+
+
+    ###################################################################################
+    - set_fact:
+        PIP_BINPROVIDERS:
+          pip:
+            installer_abspath: "{{BINARIES.pip.abspath}}"
+            installer_version: "{{BINARIES.pip.version}}"
+            PATH: "{{LIB_DIR_PIP_BIN}}"
+    - set_fact:
+        BINPROVIDERS: "{{ BINPROVIDERS | default({}) | combine(PIP_BINPROVIDERS) }}"
+
+    - debug:
+        msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"
+
+- import_playbook: load_binaries.yml
+  vars:
+    load_binaries:
+      bash:

archivebox/playbooks/install_puppeteer.yml

@@ -0,0 +1,91 @@
+---
+- import_playbook: install_npm.yml
+  vars:
+    packages:
+      - 'puppeteer'
+      - '@puppeteer/browsers'
+    state: 'latest'
+
+- name: "Install puppeteer, puppeteer/browsers, and chrome"
+  hosts: localhost
+  gather_facts: no
+  vars:
+    DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4'
+    LIB_DIR: '{{DATA_DIR}}/lib'
+    LIB_DIR_NPM: '{{LIB_DIR}}/npm'
+    LIB_DIR_NPM_BIN: '{{LIB_DIR_NPM}}/node_modules/.bin'
+    LIB_DIR_BROWSERS: '{{LIB_DIR}}/browsers'
+    LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+    CHROME_RELEASE_CHANNEL: 'chrome@stable'
+    CHROME_VERSION_MIN: '128.0.6613.137'
+  environment:
+    PATH: "{{LIB_DIR_NPM_BIN}}:{{ ansible_env.PATH }}"
+  tasks:
+    - name: Make sure prerequisite folders exist
+      file:
+        path: '{{item}}'
+        state: directory
+        recurse: true
+      loop:
+        - '{{LIB_DIR_BROWSERS}}'
+        - '{{LIB_DIR_BIN}}'
+
+    - name: Make sure Node and NPM are installed
+      assert:
+        that:
+          - BINARIES.node.version
+          - BINARIES.npm.version
+        quiet: true
+
+    - name: Get installed puppeteer version
+      command: 'puppeteer --version'
+      register: PUPPETEER_VERSION_FULL
+      changed_when: False
+
+    ###################################################################################
+    - name: Install Chrome browser using puppeteer/browsers
+      command: 'npx @puppeteer/browsers install {{CHROME_RELEASE_CHANNEL}} --path {{LIB_DIR_BROWSERS}}'
+      register: CHROME_VERSION_FULL
+      #         -> 'chrome@128.0.6613.137 /data/lib/browsers/chrome/linux_arm-128.0.6613.138/chrome-linux-arm64/...'
+      changed_when: CHROME_VERSION_MIN not in CHROME_VERSION_FULL.stdout
+
+    ###################################################################################
+    - set_fact:
+        PUPPETEER_ABSPATH: "{{LIB_DIR_NPM_BIN}}/puppeteer"
+        PUPPETEER_VERSION: "{{ PUPPETEER_VERSION_FULL.stdout_lines|first }}"
+        CHROME_ABSPATH: "{{ CHROME_VERSION_FULL.stdout_lines|last|split(' ', 1)|last }}"
+        CHROME_VERSION: "{{ CHROME_VERSION_FULL.stdout_lines|last|split('@', 1)|last|split(' ', 1)|first }}"
+
+    - name: Check that installed Chrome matches expected version
+      assert:
+        that: CHROME_VERSION_MIN is version(CHROME_VERSION, '>=')
+        quiet: true
+
+    - name: Create ./bin/chrome symlink to ./browsers/chrome/... binary
+      copy:
+        content: |
+                 #!/bin/bash
+                 exec '{{CHROME_ABSPATH}}' "$@"
+        dest: "{{LIB_DIR_BIN}}/chrome"
+    - file:
+        path: "{{LIB_DIR_BIN}}/chrome"
+        mode: u+rx,g-rx,o-rwx
+        state: 'file'
+
+    ###################################################################################
+    - set_fact:
+        PUPPETEER_DEPENDENCIES:
+          node: "{{BINARIES.node}}"
+          npm: "{{BINARIES.npm}}"
+        PUPPETEER_BINARIES:
+          puppeteer:
+            abspath: "{{PUPPETEER_ABSPATH}}"
+            version: "{{PUPPETEER_VERSION}}"
+          chrome:
+            abspath: "{{CHROME_ABSPATH}}"
+            version: "{{CHROME_VERSION}}"
+    - set_fact:
+        BINARIES: "{{ BINARIES | combine(PUPPETEER_BINARIES) }}"
+
+    - debug:
+        msg: "{{ {'BINARIES': BINARIES, 'BINPROVIDERS': BINPROVIDERS} }}"

archivebox/playbooks/install_singlefile.yml

@@ -0,0 +1,35 @@
+---
+- name: Install node and single-file-cli
+  hosts: localhost
+  gather_facts: no
+  vars:
+    DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4'
+    NPM_BIN_DIR: '{{DATA_DIR}}/node_modules/.bin'
+    MIN_BIN_VERSION: '1.1.54'
+  tasks:
+    # - package: update_cache=yes
+    #   when: ansible_facts['os_family'] == "Debian"
+
+    - ansible.builtin.package:
+        name: node
+        state: present
+    
+    - community.general.npm:
+        name: 'single-file-cli'
+        version: '{{ MIN_BIN_VERSION }}'
+        path: '{{ NPM_BIN_DIR }}/../..'
+
+    - command: '{{ NPM_BIN_DIR }}/single-file --version'
+      register: BIN_VERSION_FULL
+      changed_when: False
+
+    - set_fact:
+        BIN_VERSION: "{{ BIN_VERSION_FULL.stdout_lines|first }}"
+
+    - debug:
+        msg:
+          - "BIN_ABSPATH={{ NPM_BIN_DIR }}/single-file"
+          - "BIN_VERSION={{ BIN_VERSION }}"
+
+    - assert:
+        that: BIN_VERSION is version(MIN_BIN_VERSION, '==', version_type='semver')

archivebox/playbooks/install_ytdlp.yml

@@ -0,0 +1,39 @@
+---
+- import_playbook: install_package.yml
+  vars:
+    install_package:
+      ffmpeg: {packages: ['ffmpeg']}
+    state: 'latest'
+
+- import_playbook: install_pip.yml
+  vars:
+    install_pip:
+      ytdlp: {packages: ['yt-dlp']}
+    state: 'latest'
+
+- import_playbook: load_binaries.yml
+  vars:
+    load_binaries:
+      ffmpeg: {bin_name: 'ffmpeg', version_cmd: 'ffmpeg -version'}
+      ytdlp: {bin_name: 'yt-dlp', version_cmd: 'yt-dlp --version'}
+
+
+- name: "Install YT-DLP"
+  hosts: localhost
+  gather_facts: no
+  vars:
+    YTDLP_VERSION_MIN: '2024.8.6'
+  tasks:
+    - name: Make sure Python and Pip are installed
+      assert:
+        that:
+          - BINARIES.python.version
+          - BINARIES.pip.version
+          - BINARIES.ffmpeg.version
+          - BINARIES.ytdlp.version
+        quiet: true
+
+    - name: Check that installed YT-DLP matches expected version
+      assert:
+        that: YTDLP_VERSION_MIN is version(BINARIES.ytdlp.version, '>=')
+        quiet: true

archivebox/playbooks/load_binaries.yml

@@ -0,0 +1,71 @@
+---
+- name: Get binary abspaths and versions and add to BINARIES
+  hosts: localhost
+  gather_facts: false
+  vars:
+    DATA_DIR: '/Volumes/NVME/Users/squash/Code/archiveboxes/archivebox7/data4'
+    LIB_DIR: '{{DATA_DIR}}/lib'
+    LIB_DIR_BIN: '{{LIB_DIR}}/bin'
+    load_binaries:
+      bash:
+        bin_name: bash
+        version_cmd: 'bash --version'
+      # ffmpeg:
+      #   version_cmd: 'ffmpeg -version'
+      # ytdlp:
+      #   bin_name: ytdlp
+      # curl:
+      # wget:
+      #   bin_name: wget
+      #   version_cmd: 'wget --version'
+  tasks:
+    - name: Make sure lib folders exist
+      file:
+        path: '{{LIB_DIR_BIN}}'
+        state: directory
+        recurse: true
+
+    ###################################################################################
+    - name: Get installed binary abspath
+      command: "which {{item.1.bin_name|default(item.0)}}"
+      register: ABSPATHS_FULL
+      changed_when: false
+      ignore_errors: true
+      loop: "{{load_binaries|dictsort}}"
+    
+    - name: Get installed binary version
+      command: "{{item.1.version_cmd|default((item.1.bin_name|default(item.0)) + ' --version')}}"
+      register: VERSIONS_FULL
+      changed_when: false
+      ignore_errors: true
+      loop: "{{load_binaries|dictsort}}"
+    
+    - name: Symlink installed binary into lib bin folder
+      file:
+        src: "{{ item }}"
+        dest: "{{ LIB_DIR_BIN }}/{{ item|basename }}"
+        state: link
+      loop: "{{ABSPATHS_FULL.results|map(attribute='stdout_lines')|map('first')}}"
+
+    - set_fact:
+        BINARY_ABSPATHS: "{{ BINARY_ABSPATHS | default({}) | combine({item.0: item.1}) }}"
+      loop: "{{ keys | zip(values) }}"
+      vars:
+        keys: "{{load_binaries|dictsort|map(attribute='0')}}"
+        values: "{{ABSPATHS_FULL.results|map(attribute='stdout_lines')|map('first')}}"
+
+    - set_fact:
+        BINARY_VERSIONS: "{{ BINARY_VERSIONS | default({}) | combine({item.0: item.1}) }}"
+      loop: "{{ keys | zip(values) }}"
+      vars:
+        keys: "{{load_binaries|dictsort|map(attribute='0')}}"
+        values: "{{VERSIONS_FULL.results|map(attribute='stdout_lines')|map('first')|map('regex_replace', '^.*?v?([\\d\\.]+).*$', '\\1')}}"        
+
+    - set_fact:
+        BINARIES: "{{ BINARIES | default({}) | combine({item: { 'abspath': BINARY_ABSPATHS[item], 'version': BINARY_VERSIONS[item] }}) }}"
+      loop: "{{load_binaries|dictsort|map(attribute='0')}}"
+
+    ###################################################################################
+
+    - debug:
+        msg: "{{ {'BINARIES': BINARIES} }}"