add change detection

dns.yml

@@ -35,6 +35,8 @@
             ip: "152.53.36.98"
           - name: habits
             ip: "152.53.36.98"
+          - name: watcher
+            ip: "152.53.36.98"
       - name: nerder.land
         dns_records:
           - name: "forms"

roles/docker/files/Caddyfile

@@ -32,6 +32,9 @@ loclog.thesatelliteoflove.com {
     reverse_proxy dawarich_app:3000
 }
 
+watcher.thesatelliteoflove.com {
+    reverse_proxy changedetection:5000
+}
 
 tasks.thesatelliteoflove.com {
     reverse_proxy authentik-server-1:9000
@@ -58,6 +61,8 @@ git.thesatelliteoflove.com {
 }
 
 thesatelliteoflove.com {
+    reverse_proxy /micropub/* micropub_server-micropub-1:5000
+    reverse_proxy /micropub micropub_server-micropub-1:5000
     root * /srv/tsol
     file_server
 }

roles/docker/tasks/changedetection.yml

@@ -0,0 +1,19 @@
+- name: make changedetection directories
+  ansible.builtin.file:
+    path: "{{ item}}"
+    state: directory
+  loop:
+    - /opt/stacks/changedetection
+
+- name: Template out the compose file
+  ansible.builtin.template:
+    src: changedetection-compose.yml.j2
+    dest: /opt/stacks/changedetection/compose.yml
+    owner: root
+    mode: 644
+
+- name: deploy changedetection stack
+  community.docker.docker_compose_v2:
+    project_src: /opt/stacks/changedetection
+    files:
+    - compose.yml

roles/docker/tasks/main.yml

@@ -146,6 +146,10 @@
   import_tasks: dawarich.yml
   tags: dawarich
 
-- name: Install beaver
+#- name: Install beaver
-  import_tasks: beaver.yml
+#  import_tasks: beaver.yml
-  tags: beaver
+#  tags: beaver
+
+- name: Install changedetection
+  import_tasks: changedetection.yml
+  tags: changedetection

roles/docker/templates/changedetection-compose.yml.j2

@@ -0,0 +1,122 @@
+version: '3.2'
+services:
+    changedetection:
+      image: ghcr.io/dgtlmoon/changedetection.io
+      container_name: changedetection
+      hostname: changedetection
+      volumes:
+        - changedetection-data:/datastore
+# Configurable proxy list support, see https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configuration#proxy-list-support
+#        - ./proxies.json:/datastore/proxies.json
+
+      environment:
+  #        Default listening port, can also be changed with the -p option
+  #      - PORT=5000
+  #
+  #        Log levels are in descending order. (TRACE is the most detailed one)
+  #        Log output levels: TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL
+  #      - LOGGER_LEVEL=TRACE
+  #
+  #       Alternative WebDriver/selenium URL, do not use "'s or 's!
+  #      - WEBDRIVER_URL=http://browser-chrome:4444/wd/hub
+  #
+  #       WebDriver proxy settings webdriver_proxyType, webdriver_ftpProxy, webdriver_noProxy,
+  #                                webdriver_proxyAutoconfigUrl, webdriver_autodetect,
+  #                                webdriver_socksProxy, webdriver_socksUsername, webdriver_socksVersion, webdriver_socksPassword
+  #
+  #             https://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.common.proxy
+  #
+  #       Alternative target "Chrome" Playwright URL, do not use "'s or 's!
+  #       "Playwright" is a driver/librarythat allows changedetection to talk to a Chrome or similar browser.
+        - PLAYWRIGHT_DRIVER_URL=ws://sockpuppetbrowser:3000
+  #
+  #       Playwright proxy settings playwright_proxy_server, playwright_proxy_bypass, playwright_proxy_username, playwright_proxy_password
+  #
+  #             https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch-option-proxy
+  #
+  #        Plain requests - proxy support example.
+  #      - HTTP_PROXY=socks5h://10.10.1.10:1080
+  #      - HTTPS_PROXY=socks5h://10.10.1.10:1080
+  #
+  #        An exclude list (useful for notification URLs above) can be specified by with
+  #      - NO_PROXY="localhost,192.168.0.0/24"
+  #
+  #        Base URL of your changedetection.io install (Added to the notification alert)
+        - BASE_URL=https://watcher.thesatelliteoflove.com
+  #        Respect proxy_pass type settings, `proxy_set_header Host "localhost";` and `proxy_set_header X-Forwarded-Prefix /app;`
+  #        More here https://github.com/dgtlmoon/changedetection.io/wiki/Running-changedetection.io-behind-a-reverse-proxy-sub-directory
+  #      - USE_X_SETTINGS=1
+  #
+  #        Hides the `Referer` header so that monitored websites can't see the changedetection.io hostname.
+  #      - HIDE_REFERER=true
+  #        
+  #        Default number of parallel/concurrent fetchers
+  #      - FETCH_WORKERS=10
+  #
+  #        Absolute minimum seconds to recheck, overrides any watch minimum, change to 0 to disable
+  #      - MINIMUM_SECONDS_RECHECK_TIME=3
+  #
+  #        If you want to watch local files file:///path/to/file.txt (careful! security implications!)
+  #      - ALLOW_FILE_URI=False
+  #
+  #        For complete privacy if you don't want to use the 'check version' / telemetry service
+  #      - DISABLE_VERSION_CHECK=true
+  #
+  #        A valid timezone name to run as (for scheduling watch checking) see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+        - TZ=America/Denver
+  
+      # Comment out ports: when using behind a reverse proxy , enable networks: etc.
+  #    ports:
+  #      - 5000:5000
+      restart: unless-stopped
+
+     # Used for fetching pages via WebDriver+Chrome where you need Javascript support.
+     # Now working on arm64 (needs testing on rPi - tested on Oracle ARM instance)
+     # replace image with seleniarm/standalone-chromium:4.0.0-20211213
+     
+     # If WEBDRIVER or PLAYWRIGHT are enabled, changedetection container depends on that
+     # and must wait before starting (substitute "browser-chrome" with "playwright-chrome" if last one is used)
+      depends_on:
+          sockpuppetbrowser:
+             condition: service_started
+
+
+     # Sockpuppetbrowser is basically chrome wrapped in an API for allowing fast fetching of web-pages.
+     # RECOMMENDED FOR FETCHING PAGES WITH CHROME
+    sockpuppetbrowser:
+        hostname: sockpuppetbrowser
+        image: dgtlmoon/sockpuppetbrowser:latest
+        cap_add:
+            - SYS_ADMIN
+## SYS_ADMIN might be too much, but it can be needed on your platform https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#running-puppeteer-on-gitlabci
+        restart: unless-stopped
+        environment:
+            - SCREEN_WIDTH=1920
+            - SCREEN_HEIGHT=1024
+            - SCREEN_DEPTH=16
+            - MAX_CONCURRENT_CHROME_PROCESSES=10
+
+     # Used for fetching pages via Playwright+Chrome where you need Javascript support.
+     # Note: Works well but is deprecated, does not fetch full page screenshots (doesnt work with Visual Selector)
+     #       Does not report status codes (200, 404, 403) and other issues
+#    browser-chrome:
+#        hostname: browser-chrome
+#        image: selenium/standalone-chrome:4
+#        environment:
+#            - VNC_NO_PASSWORD=1
+#            - SCREEN_WIDTH=1920
+#            - SCREEN_HEIGHT=1080
+#            - SCREEN_DEPTH=24
+#        volumes:
+#            # Workaround to avoid the browser crashing inside a docker container
+#            # See https://github.com/SeleniumHQ/docker-selenium#quick-start
+#            - /dev/shm:/dev/shm
+#        restart: unless-stopped
+
+volumes:
+  changedetection-data:
+
+networks:
+  default:
+    external: true
+    name: lava

roles/docker/templates/dawarich-compose.yml.j2

@@ -1,10 +1,10 @@
 services:
   dawarich_redis:
-    image: redis:7.0-alpine
+    image: redis:7.4-alpine
     container_name: dawarich_redis
     command: redis-server
     volumes:
-      - dawarich_shared:/data
+      - dawarich_redis_data:/var/shared/redis
     restart: always
     healthcheck:
       test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
@@ -13,55 +13,51 @@ services:
       start_period: 30s
       timeout: 10s
   dawarich_db:
-    image: postgres:14.2-alpine
+    image: postgres:17-alpine
     shm_size: 1G
     container_name: dawarich_db
     volumes:
       - dawarich_db_data:/var/lib/postgresql/data
-      - dawarich_shared:/var/shared
-      # - ./postgresql.conf:/etc/postgresql/postgresql.conf # Optional, uncomment if you want to use a custom config
     environment:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: {{ dawarich_db_password }}
+      POSTGRES_DB: dawarich_production
     restart: always
     healthcheck:
-      test: [ "CMD-SHELL", "pg_isready -U postgres -d dawarich_development" ]
+      test: [ "CMD", "pg_isready", "-U", "postgres" ]
       interval: 10s
       retries: 5
       start_period: 30s
       timeout: 10s
-    # command: postgres -c config_file=/etc/postgresql/postgresql.conf # Use custom config, uncomment if you want to use a custom config
   dawarich_app:
     image: freikin/dawarich:latest
     container_name: dawarich_app
     volumes:
-      - dawarich_gem_cache_app:/usr/local/bundle/gems
       - dawarich_public:/var/app/public
       - dawarich_watched:/var/app/tmp/imports/watched
     stdin_open: true
     tty: true
-    entrypoint: dev-entrypoint.sh
+    entrypoint: web-entrypoint.sh
-    command: ['bin/dev']
+    command: ['bin/rails', 'server', '-p', '3000', '-b', '::']
     restart: on-failure
     environment:
-      RAILS_ENV: development
+      RAILS_ENV: production
       REDIS_URL: redis://dawarich_redis:6379/0
       DATABASE_HOST: dawarich_db
+      DATABASE_PORT: 5432
       DATABASE_USERNAME: postgres
       DATABASE_PASSWORD: {{ dawarich_db_password }}
-      DATABASE_NAME: dawarich_development
+      DATABASE_NAME: dawarich_production
       MIN_MINUTES_SPENT_IN_CITY: 60
-      APPLICATION_HOST: "loclog.thesatelliteoflove.com"
+      APPLICATION_HOSTS: loclog.thesatelliteoflove.com,localhost,::1,127.0.0.1
-      APPLICATION_HOSTS: "loclog.thesatelliteoflove.com, localhost, 127.0.0.1"
       TIME_ZONE: America/Denver
       APPLICATION_PROTOCOL: http
       DISTANCE_UNIT: mi
-      PHOTON_API_HOST: photon.komoot.io
-      PHOTON_API_USE_HTTPS: true
       PROMETHEUS_EXPORTER_ENABLED: false
       PROMETHEUS_EXPORTER_HOST: 0.0.0.0
       PROMETHEUS_EXPORTER_PORT: 9394
-      ENABLE_TELEMETRY: false # More on telemetry: https://dawarich.app/docs/tutorials/telemetry
+      SECRET_KEY_BASE: 1234567890
+      RAILS_LOG_TO_STDOUT: "true"
     logging:
       driver: "json-file"
       options:
@@ -89,32 +85,30 @@ services:
     image: freikin/dawarich:latest
     container_name: dawarich_sidekiq
     volumes:
-      - dawarich_gem_cache_sidekiq:/usr/local/bundle/gems
       - dawarich_public:/var/app/public
       - dawarich_watched:/var/app/tmp/imports/watched
     stdin_open: true
     tty: true
-    entrypoint: dev-entrypoint.sh
+    entrypoint: sidekiq-entrypoint.sh
-    command: ['sidekiq']
+    command: ['bundle', 'exec', 'sidekiq']
     restart: on-failure
     environment:
-      RAILS_ENV: development
+      RAILS_ENV: production
       REDIS_URL: redis://dawarich_redis:6379/0
       DATABASE_HOST: dawarich_db
+      DATABASE_PORT: 5432
       DATABASE_USERNAME: postgres
       DATABASE_PASSWORD: {{ dawarich_db_password }}
-      DATABASE_NAME: dawarich_development
+      DATABASE_NAME: dawarich_production
-      APPLICATION_HOST: "loclog.thesatelliteoflove.com"
+      APPLICATION_HOSTS: loclog.thesatelliteoflove.com,localhost,::1,127.0.0.1
-      APPLICATION_HOSTS: "loclog.thesatelliteoflove.com, localhost, 127.0.0.1"
       BACKGROUND_PROCESSING_CONCURRENCY: 10
       APPLICATION_PROTOCOL: http
       DISTANCE_UNIT: mi
-      PHOTON_API_HOST: photon.komoot.io
-      PHOTON_API_USE_HTTPS: true
       PROMETHEUS_EXPORTER_ENABLED: false
       PROMETHEUS_EXPORTER_HOST: dawarich_app
       PROMETHEUS_EXPORTER_PORT: 9394
-      ENABLE_TELEMETRY: false # More on telemetry: https://dawarich.app/docs/tutorials/telemetry
+      SECRET_KEY_BASE: 1234567890
+      RAILS_LOG_TO_STDOUT: "true"
     logging:
       driver: "json-file"
       options:
@@ -144,12 +138,9 @@ services:
 
 volumes:
   dawarich_db_data:
-  dawarich_gem_cache_app:
+  dawarich_redis_data:
-  dawarich_gem_cache_sidekiq:
-  dawarich_shared:
   dawarich_public:
   dawarich_watched:
-
 networks:
   default:
     external: true

roles/docker/templates/gitea-compose.yml.j2

@@ -22,6 +22,7 @@ services:
       - 222:22
     extra_hosts:
       - 'auth.thesatelliteoflove.com:172.20.0.5'
+      - 'git.thesatelliteoflove.com:172.20.0.5'
   runner:
     image: gitea/act_runner:nightly
     depends_on:
@@ -32,6 +33,9 @@ services:
       - GITEA_RUNNER_REGISTRATION_TOKEN={{ gitea_runner_key }}
       - GITEA_RUNNER_NAME=runner_1
       - GITEA_RUNNER_LABELS=docker
+    extra_hosts:
+      - 'auth.thesatelliteoflove.com:172.20.0.5'
+      - 'git.thesatelliteoflove.com:172.20.0.5'
     volumes:
       - ./runner-config.yaml:/config.yaml
       - ./data:/data

roles/docker/templates/glance.yml.j2

@@ -24,9 +24,16 @@ pages:
               collapse-after: 3
               cache: 3h
               feeds:
-              - url: https://hiro.report/rss/
               - url: https://kill-the-newsletter.com/feeds/ij4twrnzhrwvyic13qcm.xml
-
+              - url: https://mrmoneymustache.ck.page/68f9e9826c
+            - type: rss
+              title: Gear
+              limit: 10
+              collapse-after: 3
+              cache: 3h
+              feeds:
+              - url: https://9to5toys.com/steals/feed
+              - url: https://hiro.report/rss/
           - type: videos
             channels:
               - UCR-DXc1voovS8nhAvccRZhg # Jeff Geerling
@@ -71,6 +78,8 @@ pages:
           feeds:
             - url: https://thesatelliteoflove.com/feeds/warhammer_rss_feed.xml
               title: Warhammer Community
+            - url: https://yenneferofexeter.wordpress.com/feed/
+              title: Yennefer of Exeter
   - name: Self Hosting
     columns:
     - size: small