Files
StarPunk/starpunk/syndication.py
T
PhilandClaude Opus 5.5 39c8259f3f fix: Crossposts include photos embedded in the post text (v2.6.6)
Some Micropub clients (OutPost) upload a photo to the media endpoint and
then embed it in content instead of sending it as `photo`. Those posts
have no attached media, so crossposts went out without the image.

Images in the text that are in this site's media library are now sent
like attached photos, with their alt text, and GoToSocial's text no
longer carries the image Markdown.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-02 18:11:27 -07:00

1394 lines
57 KiB
Python

"""
Crossposting (POSSE) for StarPunk (v2.1.0 ADR-065, v2.2.0 ADR-066, v2.6.0 ADR-069, v2.6.3 ADR-070)
Published notes are copied to syndication targets, and later edits and
deletes follow them. Three target kinds exist: a GoToSocial (or any
Mastodon-API) account, a Bluesky account and a Threads account. Bluesky and
Threads posts can't be edited, so there only deletes follow.
Each (note, target) pair has a row in `syndications`. Changing a note marks
its rows pending; a background worker then reconciles the remote post with
the note:
note published + target enabled, no remote post -> create
note published + target enabled, remote post -> update
otherwise, remote post exists -> delete
Rows carry a version so an edit made while a send is in flight is not lost.
"""
import fcntl
import hashlib
import io
import json
import logging
import os
import re
import threading
import time
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Dict, List, Optional
import httpx
from flask import current_app
# httpx logs request URLs at INFO, and Threads takes its token in the URL
logging.getLogger("httpx").setLevel(logging.WARNING)
from starpunk.database import get_db
MAX_ATTEMPTS = 8
CLAIM_MINUTES = 5
class SyndicationError(Exception):
"""A send failed. permanent=True means retrying won't help."""
def __init__(self, message: str, permanent: bool = False):
super().__init__(message)
self.permanent = permanent
class RemoteGone(SyndicationError):
"""The remote post no longer exists"""
class MastodonClient:
"""Minimal Mastodon-API client (works with GoToSocial)"""
def __init__(self, base_url: str, token: str, visibility: str = "public",
http: Optional[httpx.Client] = None):
self.base_url = base_url.rstrip("/")
self.visibility = visibility
self.http = http or httpx.Client(timeout=30.0)
self.headers = {"Authorization": f"Bearer {token}"}
def _call(self, method: str, path: str, **kwargs) -> httpx.Response:
headers = {**self.headers, **kwargs.pop("headers", {})}
try:
response = self.http.request(method, self.base_url + path, headers=headers, **kwargs)
except httpx.HTTPError as e:
raise SyndicationError(f"{method} {path}: {e}")
if response.status_code == 404:
raise RemoteGone(f"{method} {path}: not found", permanent=True)
if response.status_code >= 400:
# 4xx other than rate limiting won't fix itself
permanent = 400 <= response.status_code < 500 and response.status_code != 429
raise SyndicationError(
f"{method} {path}: HTTP {response.status_code} {response.text[:200]}",
permanent=permanent,
)
return response
def upload_media(self, path: Path, mime_type: str, description: str) -> str:
with open(path, "rb") as f:
response = self._call(
"POST", "/api/v2/media",
files={"file": (path.name, f, mime_type)},
data={"description": description or ""},
)
return response.json()["id"]
def _status_body(self, text: str, media_ids: List[str]) -> dict:
return {
"status": text,
"media_ids": media_ids,
"visibility": self.visibility,
"content_type": "text/markdown",
}
def create_status(self, text: str, media_ids: List[str], idempotency_key: str) -> dict:
response = self._call(
"POST", "/api/v1/statuses",
json=self._status_body(text, media_ids),
headers={"Idempotency-Key": idempotency_key},
)
data = response.json()
return {"id": data["id"], "url": data.get("url") or data.get("uri")}
def update_status(self, remote_id: str, text: str, media_ids: List[str]) -> dict:
response = self._call("PUT", f"/api/v1/statuses/{remote_id}",
json=self._status_body(text, media_ids))
data = response.json()
return {"id": data["id"], "url": data.get("url") or data.get("uri")}
def delete_status(self, remote_id: str) -> None:
try:
self._call("DELETE", f"/api/v1/statuses/{remote_id}")
except RemoteGone:
pass # already gone
# Interface used by the reconcile loop
can_edit = True
NATIVE = ("favourite", "reblog") # remote_id prefixes for likes/boosts (v2.5.0)
def find_status(self, url: str) -> Optional[str]:
"""Local id of a fediverse post, fetching it if needed; None if the URL isn't one"""
try:
response = self._call("GET", "/api/v2/search", params={
"q": url, "resolve": "true", "type": "statuses", "limit": 1,
})
except RemoteGone:
return None
statuses = response.json().get("statuses") or []
return statuses[0]["id"] if statuses else None
def _prepare(self, post: "Post") -> tuple:
if post.kind == "repost":
return compose_repost_text(post, post.note.content, post.max_chars), []
# Images in the text are sent as attachments instead (v2.6.6)
text = compose_text(post.note, post.permalink, post.tags, post.max_chars,
body=strip_images(post.note.content))
media_ids = [self.upload_media(f["path"], f["mime_type"], f["description"]) for f in post.media]
return text, media_ids
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
if post.kind in ("like", "repost") and post.target_url:
# Native favourite/boost when the target is a fediverse post; a
# repost with a comment is posted as a status so the comment stays
commented = bool(post.note.content.strip())
if post.kind == "like" or not commented:
status_id = self.find_status(post.target_url)
if status_id:
action = "favourite" if post.kind == "like" else "reblog"
self._call("POST", f"/api/v1/statuses/{status_id}/{action}")
return {"id": f"{action}:{status_id}", "url": None}
if post.kind == "like":
return {"id": None, "url": None} # nothing to like here
text, media_ids = self._prepare(post)
return self.create_status(text, media_ids, idempotency_key=key)
def update(self, remote_id: str, post: "Post") -> dict:
if remote_id.split(":", 1)[0] in self.NATIVE:
return {"id": remote_id, "url": None} # a favourite/boost has nothing to edit
text, media_ids = self._prepare(post)
return self.update_status(remote_id, text, media_ids)
def delete(self, remote_id: str) -> None:
action, _, status_id = remote_id.partition(":")
if action in self.NATIVE and status_id:
try:
self._call("POST", f"/api/v1/statuses/{status_id}/un{action}")
except RemoteGone:
pass
return
self.delete_status(remote_id)
BLUESKY_BLOB_LIMIT = 1_000_000 # app.bsky.embed.images maxSize
_TID_CHARS = "234567abcdefghijklmnopqrstuvwxyz"
class BlueskyClient:
"""
Minimal AT Protocol client for posting to Bluesky with an app password
Posts are plain text with link and hashtag facets. Images go in an
images embed; a note without images gets a link card instead.
"""
can_edit = False # Bluesky has no edits; only deletes follow the note
POST = "app.bsky.feed.post"
def __init__(self, service: str, identifier: str, password: str,
http: Optional[httpx.Client] = None):
self.service = service.rstrip("/")
self.identifier = identifier
self.password = password
self.http = http or httpx.Client(timeout=30.0)
self.pds = self.service
self.did: Optional[str] = None
self.handle = identifier
self._access: Optional[str] = None
self._refresh: Optional[str] = None
self._lock = threading.Lock()
# --- XRPC plumbing ---
def _raw(self, method: str, nsid: str, base: str, token: Optional[str], **kwargs) -> httpx.Response:
headers = kwargs.pop("headers", {})
if token:
headers["Authorization"] = f"Bearer {token}"
try:
return self.http.request(method, f"{base}/xrpc/{nsid}", headers=headers, **kwargs)
except httpx.HTTPError as e:
raise SyndicationError(f"{nsid}: {e}")
@staticmethod
def _error_name(response: httpx.Response) -> str:
try:
return response.json().get("error") or ""
except ValueError:
return ""
def _raise_for(self, nsid: str, response: httpx.Response) -> None:
if response.status_code < 400:
return
permanent = 400 <= response.status_code < 500 and response.status_code != 429
raise SyndicationError(
f"{nsid}: HTTP {response.status_code} {response.text[:200]}", permanent=permanent
)
def _login(self) -> None:
response = None
if self._refresh:
response = self._raw("POST", "com.atproto.server.refreshSession", self.service, self._refresh)
if response.status_code >= 400:
response = None
if response is None:
response = self._raw("POST", "com.atproto.server.createSession", self.service, None,
json={"identifier": self.identifier, "password": self.password})
self._raise_for("com.atproto.server.createSession", response)
data = response.json()
self._access, self._refresh = data["accessJwt"], data["refreshJwt"]
self.did = data["did"]
self.handle = data.get("handle") or self.handle
for service in (data.get("didDoc") or {}).get("service", []):
if service.get("id", "").endswith("#atproto_pds") and service.get("serviceEndpoint"):
self.pds = service["serviceEndpoint"].rstrip("/")
def _call(self, method: str, nsid: str, **kwargs) -> httpx.Response:
with self._lock:
if not self._access:
self._login()
response = self._raw(method, nsid, self.pds, self._access, **kwargs)
if response.status_code == 401 or self._error_name(response) == "ExpiredToken":
self._access = None
self._login()
response = self._raw(method, nsid, self.pds, self._access, **kwargs)
self._raise_for(nsid, response)
return response
# --- Posts ---
def upload_blob(self, data: bytes, mime_type: str) -> dict:
response = self._call("POST", "com.atproto.repo.uploadBlob", content=data,
headers={"Content-Type": mime_type})
return response.json()["blob"]
def _image(self, media: dict) -> dict:
data, mime_type, width, height = _fit_image(media["path"], media["mime_type"], BLUESKY_BLOB_LIMIT)
image = {"image": self.upload_blob(data, mime_type), "alt": media["description"] or ""}
if width and height:
image["aspectRatio"] = {"width": width, "height": height}
return image
@staticmethod
def _now() -> str:
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
def _link_card(self, url: str) -> dict:
"""External embed for a page, with its title, description and image when we can get them"""
card = page_card(self.http, url)
external = {"uri": url, "title": card["title"][:300], "description": card["description"][:1000]}
if card["image"]:
try:
data, mime_type, _ = _fetch_public(self.http, card["image"], CARD_IMAGE_LIMIT,
truncate=False)
data, mime_type, _, _ = _fit_image(data, mime_type.split(";")[0] or "image/jpeg",
BLUESKY_BLOB_LIMIT)
external["thumb"] = self.upload_blob(data, mime_type)
except Exception as e: # a card without a picture is fine
current_app.logger.info(f"No link card image for {url}: {e}")
return {"$type": "app.bsky.embed.external", "external": external}
def _record(self, post: "Post", text: Optional[str] = None, embed: Optional[dict] = None,
anchors: List[tuple] = ()) -> dict:
if text is None:
body, anchors = plain_text_links(post.note.content, links="anchor")
text = compose_text(post.note, post.permalink, post.tags, post.max_chars, body=body)
record = {"$type": self.POST, "text": text, "createdAt": self._now()}
facets = bluesky_facets(text, anchors)
if facets:
record["facets"] = facets
if embed is not None:
record["embed"] = embed
elif post.media:
record["embed"] = {"$type": "app.bsky.embed.images",
"images": [self._image(m) for m in post.media]}
elif note_links(post.note.content):
# The note links somewhere: the card shows that page, not the note
record["embed"] = self._link_card(note_links(post.note.content)[0])
else:
record["embed"] = {"$type": "app.bsky.embed.external", "external": {
"uri": post.permalink,
"title": post.note.title[:300],
"description": plain_text(post.note.content)[:300],
}}
return record
_POST_URL = re.compile(r"^https://bsky\.app/profile/([^/]+)/post/([^/?#]+)")
def resolve_post(self, url: str) -> Optional[dict]:
"""{'uri', 'cid'} of a Bluesky post URL; None if the URL isn't one"""
m = self._POST_URL.match(url or "")
if not m:
return None
actor, rkey = m.groups()
if not actor.startswith("did:"):
actor = self._call("GET", "com.atproto.identity.resolveHandle",
params={"handle": actor}).json()["did"]
uri = f"at://{actor}/{self.POST}/{rkey}"
posts = self._call("GET", "app.bsky.feed.getPosts", params={"uris": uri}).json().get("posts") or []
return {"uri": uri, "cid": posts[0]["cid"]} if posts else None
def _create(self, collection: str, record: dict) -> str:
response = self._call("POST", "com.atproto.repo.createRecord",
json={"repo": self.did or self.identifier, "collection": collection,
"record": record})
return response.json()["uri"]
def _react(self, post: "Post") -> dict:
"""Like, repost or quote a Bluesky post; fall back to a link post for reposts"""
comment, anchors = (plain_text_links(post.note.content, links="anchor")
if post.note.content.strip() else ("", []))
subject = self.resolve_post(post.target_url)
if subject and post.kind == "like":
uri = self._create("app.bsky.feed.like", {
"$type": "app.bsky.feed.like", "subject": subject, "createdAt": self._now()})
return {"id": uri, "url": None}
if subject and not comment:
uri = self._create("app.bsky.feed.repost", {
"$type": "app.bsky.feed.repost", "subject": subject, "createdAt": self._now()})
return {"id": uri, "url": None}
if subject: # repost with a comment -> quote post
text = comment if len(comment) <= post.max_chars else \
comment[:post.max_chars - 1].rsplit(" ", 1)[0].rstrip() + "…"
uri = self._create(self.POST, self._record(
post, text=text, embed={"$type": "app.bsky.embed.record", "record": subject},
anchors=anchors))
return {"id": uri, "url": self._web_url(uri)}
if post.kind == "like":
return {"id": None, "url": None} # not a Bluesky post: nothing to like
text = compose_repost_text(post, comment, post.max_chars)
embed = {"$type": "app.bsky.embed.external", "external": {
"uri": post.target_url, "title": (post.target_title or post.target_url)[:300],
"description": ""}}
uri = self._create(self.POST, self._record(post, text=text, embed=embed))
return {"id": uri, "url": self._web_url(uri)}
def _web_url(self, uri: str) -> str:
rkey = uri.rsplit("/", 1)[1]
return f"https://bsky.app/profile/{self.handle or self.did}/post/{rkey}"
def _find_existing(self, permalink: str) -> Optional[dict]:
"""A recent post that already links to this note (an earlier send whose reply was lost)"""
response = self._call("GET", "com.atproto.repo.listRecords",
params={"repo": self.did, "collection": self.POST, "limit": 25})
for item in response.json().get("records", []):
if permalink in (item.get("value") or {}).get("text", ""):
return {"id": item["uri"], "url": self._web_url(item["uri"])}
return None
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
if post.kind in ("like", "repost") and post.target_url:
if not self.did:
with self._lock:
self._login()
return self._react(post)
if retrying:
if not self.did:
with self._lock:
self._login()
existing = self._find_existing(post.permalink)
if existing:
return existing
record = self._record(post)
response = self._call("POST", "com.atproto.repo.createRecord",
json={"repo": self.did or self.identifier, "collection": self.POST,
"record": record})
uri = response.json()["uri"]
return {"id": uri, "url": self._web_url(uri)}
def update(self, remote_id: str, post: "Post") -> dict:
raise NotImplementedError("Bluesky posts can't be edited")
def delete(self, remote_id: str) -> None:
# at://<did>/app.bsky.feed.post/<rkey>; deleting a missing record succeeds
repo, collection, rkey = remote_id[len("at://"):].split("/", 2)
self._call("POST", "com.atproto.repo.deleteRecord",
json={"repo": repo, "collection": collection, "rkey": rkey})
def _fit_image(path, mime_type: str, limit: int) -> tuple:
"""
Image (a path, or bytes) no bigger than `limit`, re-encoded as JPEG if
needed. Returns (data, mime, w, h).
"""
from PIL import Image
data = path if isinstance(path, bytes) else Path(path).read_bytes()
name = "image" if isinstance(path, bytes) else Path(path).name
with Image.open(io.BytesIO(data)) as img:
width, height = img.size
if len(data) <= limit:
return data, mime_type, width, height
img = img.convert("RGB")
for max_side, quality in ((2000, 85), (2000, 75), (1600, 70), (1200, 65), (1000, 60)):
copy = img.copy()
copy.thumbnail((max_side, max_side))
buf = io.BytesIO()
copy.save(buf, format="JPEG", quality=quality, optimize=True)
if buf.tell() <= limit:
return buf.getvalue(), "image/jpeg", copy.size[0], copy.size[1]
raise SyndicationError(f"{name}: image too large for Bluesky", permanent=True)
# --- Link cards ---------------------------------------------------------------
CARD_PAGE_LIMIT = 512 * 1024 # the <head> is near the start; read no more
CARD_IMAGE_LIMIT = 5 * 1024 * 1024
def _fetch_public(http: httpx.Client, url: str, limit: int, truncate: bool) -> tuple:
"""
GET a public web URL, checking every redirect hop (no requests into the
LAN). Reads at most `limit` bytes: stops there if `truncate`, else fails.
Returns (data, content_type, final_url).
"""
from urllib.parse import urljoin, urlparse
from starpunk.micropub import _public_address
site = current_app.config.get("SITE_URL", "")
for _ in range(5):
parsed = urlparse(url)
if parsed.scheme not in ("http", "https") or not parsed.hostname:
raise SyndicationError(f"not a web URL: {url}", permanent=True)
if not current_app.config.get("TESTING") and not _public_address(parsed.hostname):
raise SyndicationError(f"not a public address: {url}", permanent=True)
try:
with http.stream("GET", url, follow_redirects=False, timeout=10.0,
headers={"User-Agent": f"StarPunk (+{site})"}) as response:
if response.is_redirect and "location" in response.headers:
url = urljoin(url, response.headers["location"])
continue
if response.status_code >= 400:
raise SyndicationError(f"{url}: HTTP {response.status_code}")
data = bytearray()
for chunk in response.iter_bytes():
data += chunk
if len(data) > limit:
if truncate:
break
raise SyndicationError(f"{url}: too large")
return bytes(data), response.headers.get("content-type", ""), url
except httpx.HTTPError as e:
raise SyndicationError(f"{url}: {type(e).__name__}")
raise SyndicationError(f"{url}: too many redirects")
def page_card(http: httpx.Client, url: str) -> dict:
"""
A page's title, description and image URL (Open Graph, Twitter or plain
HTML tags), for a link card. Missing parts come back empty; a page that
can't be fetched gives just the URL as the title.
"""
from urllib.parse import urljoin
from bs4 import BeautifulSoup
card = {"title": url, "description": "", "image": ""}
try:
data, content_type, final = _fetch_public(http, url, CARD_PAGE_LIMIT, truncate=True)
except SyndicationError as e:
current_app.logger.info(f"No link card details for {url}: {e}")
return card
if "html" not in content_type.lower():
return card
soup = BeautifulSoup(data, "html.parser")
def meta(*names):
for name in names:
tag = soup.find("meta", attrs={"property": name}) or soup.find("meta", attrs={"name": name})
if tag and (tag.get("content") or "").strip():
return " ".join(tag["content"].split())
return ""
title = meta("og:title", "twitter:title")
if not title and soup.title:
title = " ".join(soup.title.get_text(" ").split())
image = meta("og:image:secure_url", "og:image", "og:image:url", "twitter:image")
card.update(title=title or url,
description=meta("og:description", "twitter:description", "description"),
image=urljoin(final, image) if image else "")
return card
THREADS_API = "https://graph.threads.net/v1.0"
THREADS_IMAGE_TYPES = {"image/jpeg", "image/png"}
THREADS_IMAGE_LIMIT = 8_000_000
THREADS_MAX_LINKS = 5
THREADS_REFRESH_EVERY = 24 * 3600 # tokens last 60 days; Meta allows a refresh once one is a day old
# Graph API error codes that are temporary (rate limits, service hiccups)
_GRAPH_TRANSIENT = {1, 2, 4, 17, 32, 341, 613}
_GRAPH_AUTH = {10, 190, 200} # bad token or missing permission: retrying won't help
def threads_length(text: str) -> int:
"""Length as Threads counts it: emoji (and, to be safe, all non-ASCII) by UTF-8 bytes"""
return sum(1 if ord(ch) < 128 else len(ch.encode()) for ch in text)
def threads_topic(tags: List[str]) -> str:
"""The first tag usable as a Threads topic (one per post; no '.' or '&', 1-50 chars)"""
for tag in tags:
topic = re.sub(r"[.&]", "", tag).strip()[:50]
if topic:
return topic
return ""
class ThreadsClient:
"""
Minimal Threads API client (Meta Graph API)
Posting is two steps: create a container, wait for it to be ready, then
publish it. Meta fetches images from their public URL on this site.
Posts can't be edited through the API, so only deletes follow.
Long-lived tokens expire after 60 days unless refreshed. `maintain()`
refreshes the token once a day and keeps it in `token_file`, so it
survives restarts. A different token in the config replaces it.
"""
can_edit = False
def __init__(self, token: str, token_file: Optional[Path] = None, api: str = THREADS_API,
http: Optional[httpx.Client] = None, sleep=time.sleep):
self.config_token = token
self.token_file = Path(token_file) if token_file else None
self.api = api.rstrip("/")
self.http = http or httpx.Client(timeout=30.0)
self.sleep = sleep
self.user_id: Optional[str] = None
self.username: Optional[str] = None
self._lock = threading.Lock()
self._token = token
self._checked_at = 0.0
saved = self._read_file()
if saved:
self._token = saved["token"]
# --- Token ---
def _seed(self) -> str:
return hashlib.sha256(self.config_token.encode()).hexdigest()
def _read_file(self) -> Optional[dict]:
"""The saved token, if it descends from the configured one"""
if not self.token_file or not self.token_file.exists():
return None
try:
data = json.loads(self.token_file.read_text())
except (OSError, ValueError):
return None
return data if data.get("seed") == self._seed() and data.get("token") else None
def _write_file(self, token: str, refreshed_at: float) -> None:
tmp = self.token_file.with_suffix(".tmp")
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, "w") as f:
json.dump({"seed": self._seed(), "token": token, "refreshed_at": refreshed_at}, f)
os.replace(tmp, self.token_file)
def maintain(self) -> None:
"""Refresh the token if a day has passed since the last try (cheap to call often)"""
now = time.time()
if now - self._checked_at < THREADS_REFRESH_EVERY:
return
self._checked_at = now
if not self.token_file:
return
# Gunicorn workers share the file: one refreshes, the others pick up its result
self.token_file.parent.mkdir(parents=True, exist_ok=True)
with open(self.token_file.with_suffix(".lock"), "w") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
saved = self._read_file()
if saved:
self._token = saved["token"]
if now - saved.get("refreshed_at", 0) < THREADS_REFRESH_EVERY:
self._checked_at = saved["refreshed_at"]
return
base = self.api.rsplit("/", 1)[0] # refresh lives outside the versioned path
try:
response = self.http.get(f"{base}/refresh_access_token", params={
"grant_type": "th_refresh_token", "access_token": self._token})
except httpx.HTTPError as e:
current_app.logger.warning(f"Threads token refresh failed: {type(e).__name__}")
self._checked_at = now - THREADS_REFRESH_EVERY + 3600 # try again in an hour
return
if response.status_code < 400:
self._token = response.json()["access_token"]
current_app.logger.info("Threads access token refreshed")
else:
# e.g. a token less than a day old: keep it, try again tomorrow
current_app.logger.warning(
f"Threads token refresh failed: HTTP {response.status_code} {response.text[:200]}")
self._write_file(self._token, now)
# --- Graph plumbing ---
def _call(self, method: str, path: str, **params) -> dict:
params["access_token"] = self._token
kwargs = {"params": params} if method in ("GET", "DELETE") else {"data": params}
try:
response = self.http.request(method, f"{self.api}/{path}", **kwargs)
except httpx.HTTPError as e:
raise SyndicationError(f"{method} {path}: {type(e).__name__}") # never echo the URL/token
try:
body = response.json()
except ValueError:
body = {}
error = body.get("error") if isinstance(body, dict) else None
if response.status_code >= 400 or error:
error = error if isinstance(error, dict) else {}
code = error.get("code")
detail = " / ".join(str(error[k]) for k in ("error_user_title", "error_user_msg")
if error.get(k))
message = (f"{method} {path}: HTTP {response.status_code} "
f"{error.get('message') or response.text[:200]}"
f" (code {code}, subcode {error.get('error_subcode')}"
f"{', ' + detail if detail else ''}, fbtrace {error.get('fbtrace_id')})")
if response.status_code == 404 or (code == 100 and error.get("error_subcode") == 33):
raise RemoteGone(message, permanent=True)
permanent = (400 <= response.status_code < 500 and response.status_code != 429
and code not in _GRAPH_TRANSIENT)
exc = SyndicationError(message, permanent=permanent)
exc.code = code
raise exc
return body
def _me(self) -> str:
with self._lock:
if not self.user_id:
me = self._call("GET", "me", fields="id,username")
self.user_id, self.username = me["id"], me.get("username")
return self.user_id
def _wait(self, container_id: str) -> None:
"""Wait until Meta has processed a container (images take a few seconds)"""
delay = 1
self.sleep(2) # Meta can report a container ready a moment too early
for _ in range(8):
status = self._call("GET", container_id, fields="status,error_message")
state = status.get("status")
if state in ("FINISHED", None):
return
if state in ("ERROR", "EXPIRED"):
raise SyndicationError(
f"Threads container {state}: {status.get('error_message') or 'no detail'}",
permanent=True)
self.sleep(delay)
delay = min(delay * 2, 15)
raise SyndicationError("Threads container still processing")
# --- Posts ---
def _text(self, post: "Post") -> str:
# Links written "text (url)"; the first one gets Threads' preview card.
# Meta allows 5 different links per post and the permalink is one.
body = plain_text(post.note.content, links="inline", max_urls=THREADS_MAX_LINKS - 1)
limit = post.max_chars
while True:
text = compose_text(post.note, post.permalink, [], limit, body=body)
over = threads_length(text) - post.max_chars
if over <= 0:
return text
limit -= over
def _image_container(self, uid: str, media: dict, **extra) -> str:
params = {"media_type": "IMAGE", "image_url": media["url"], **extra}
if media["description"]:
params["alt_text"] = media["description"][:1000]
return self._call("POST", f"{uid}/threads", **params)["id"]
def _repost_text(self, post: "Post") -> str:
comment = plain_text(post.note.content, links="inline", max_urls=THREADS_MAX_LINKS - 1)
limit = post.max_chars
while True:
text = compose_repost_text(post, comment, limit)
over = threads_length(text) - post.max_chars
if over <= 0:
return text
limit -= over
def _container(self, uid: str, post: "Post") -> str:
if post.kind == "repost":
# No native repost of outside content: post "🔁 title url"
return self._call("POST", f"{uid}/threads", media_type="TEXT",
text=self._repost_text(post))["id"]
common = {"text": self._text(post)}
topic = threads_topic(post.tags)
if topic:
common["topic_tag"] = topic
# Threads takes JPEG/PNG up to 8 MB; others (GIF, WebP) are left off
images = [m for m in post.media if m["mime_type"] in THREADS_IMAGE_TYPES
and Path(m["path"]).stat().st_size <= THREADS_IMAGE_LIMIT]
if not images:
# No explicit link_attachment: Meta rejects one for a page it hasn't
# seen yet ("Invalid Link Attachment"), but previews the text's
# first link by itself without failing
return self._call("POST", f"{uid}/threads", media_type="TEXT", **common)["id"]
if len(images) == 1:
return self._image_container(uid, images[0], **common)
children = [self._image_container(uid, m, is_carousel_item="true") for m in images]
for child in children:
self._wait(child)
return self._call("POST", f"{uid}/threads", media_type="CAROUSEL",
children=",".join(children), **common)["id"]
def _profile_url(self) -> str:
return f"https://www.threads.com/@{self.username}" if self.username else "https://www.threads.com/"
def _find_existing(self, uid: str, link: str) -> Optional[dict]:
"""A recent post that already has this link (an earlier send whose reply was lost)"""
listing = self._call("GET", f"{uid}/threads", fields="id,text,permalink", limit=25)
for item in listing.get("data", []):
if link in (item.get("text") or ""):
return {"id": item["id"], "url": item.get("permalink") or self._profile_url()}
return None
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
if post.kind == "like" and post.target_url:
return {"id": None, "url": None} # the API can't like things; likes stay on the site
uid = self._me()
if retrying:
marker = post.target_url if post.kind == "repost" and post.target_url else post.permalink
existing = self._find_existing(uid, marker)
if existing:
return existing
container = self._container(uid, post)
self._wait(container)
try:
media_id = self._call("POST", f"{uid}/threads_publish", creation_id=container)["id"]
except SyndicationError as e:
if getattr(e, "code", None) in _GRAPH_AUTH:
raise
# Meta's publish step fails now and then ("Fatal") and works on a
# later try; the retry checks for an existing post first
raise SyndicationError(str(e))
try:
url = self._call("GET", media_id, fields="permalink").get("permalink")
except SyndicationError:
url = None # it's published; don't fail (and post twice) over the link
return {"id": media_id, "url": url or self._profile_url()}
def update(self, remote_id: str, post: "Post") -> dict:
raise NotImplementedError("Threads posts can't be edited through the API")
def delete(self, remote_id: str) -> None:
try:
self._call("DELETE", remote_id)
except RemoteGone:
pass # already gone
@dataclass
class Target:
id: str # stored in syndications.target
uid: str # Micropub syndicate-to uid
name: str # shown in the editor and Micropub clients
client: object # MastodonClient or BlueskyClient
max_chars: int = 5000
max_media: int = 4
@dataclass
class Post:
"""What a client needs to publish a note"""
note: object
permalink: str
tags: List[str]
max_chars: int
media: List[dict] = field(default_factory=list)
kind: str = "note" # note, like or repost (v2.5.0)
target_url: Optional[str] = None # what a like/repost points at
target_title: Optional[str] = None # its fetched title, if any
def compose_repost_text(post: "Post", comment: str, max_chars: int) -> str:
"""'comment' then '🔁 title' and the URL; the comment is shortened to fit"""
title = post.target_title or "Reposted"
tail = f"🔁 {title}\n{post.target_url}"
if len(tail) > max_chars:
tail = f"🔁 {post.target_url}"
comment = (comment or "").strip()
if not comment:
return tail
room = max_chars - len(tail) - 2
if len(comment) > room:
comment = comment[:max(room - 1, 0)].rsplit(" ", 1)[0].rstrip() + "…"
return f"{comment}\n\n{tail}" if room > 1 else tail
_client_cache: Dict[str, object] = {}
def targets_configured(config) -> bool:
"""Whether any crossposting target has credentials (used to start the worker)"""
return bool(
(config.get("GOTOSOCIAL_URL") and config.get("GOTOSOCIAL_ACCESS_TOKEN"))
or (config.get("BLUESKY_HANDLE") and config.get("BLUESKY_APP_PASSWORD"))
or config.get("THREADS_ACCESS_TOKEN")
)
def get_targets() -> Dict[str, Target]:
"""Configured syndication targets, by id"""
config = current_app.config
targets: Dict[str, Target] = {}
url = (config.get("GOTOSOCIAL_URL") or "").rstrip("/")
token = config.get("GOTOSOCIAL_ACCESS_TOKEN") or ""
if url and token:
client = _client_cache.get(url)
if client is None:
client = MastodonClient(url, token, config.get("GOTOSOCIAL_VISIBILITY", "public"))
_client_cache[url] = client
targets["gotosocial"] = Target(
id="gotosocial",
uid=url + "/",
name=config.get("GOTOSOCIAL_NAME", "GoToSocial"),
client=client,
max_chars=int(config.get("GOTOSOCIAL_MAX_CHARS", 5000)),
max_media=int(config.get("GOTOSOCIAL_MAX_MEDIA", 4)),
)
handle = (config.get("BLUESKY_HANDLE") or "").lstrip("@")
password = config.get("BLUESKY_APP_PASSWORD") or ""
if handle and password:
service = (config.get("BLUESKY_SERVICE") or "https://bsky.social").rstrip("/")
key = f"bluesky:{service}:{handle}"
client = _client_cache.get(key)
if client is None:
client = BlueskyClient(service, handle, password)
_client_cache[key] = client
targets["bluesky"] = Target(
id="bluesky",
uid=f"https://bsky.app/profile/{handle}",
name=config.get("BLUESKY_NAME", "Bluesky"),
client=client,
max_chars=int(config.get("BLUESKY_MAX_CHARS", 300)),
max_media=4,
)
token = config.get("THREADS_ACCESS_TOKEN") or ""
if token:
client = _client_cache.get("threads")
if client is None or client.config_token != token:
client = ThreadsClient(token, Path(config["DATA_PATH"]) / "threads_token.json",
api=config.get("THREADS_API") or THREADS_API)
_client_cache["threads"] = client
username = (config.get("THREADS_USERNAME") or "").lstrip("@")
targets["threads"] = Target(
id="threads",
uid=f"https://www.threads.com/@{username}" if username else "https://www.threads.com/",
name=config.get("THREADS_NAME", "Threads"),
client=client,
max_chars=int(config.get("THREADS_MAX_CHARS", 500)),
max_media=int(config.get("THREADS_MAX_MEDIA", 4)),
)
return targets
def resolve_targets(requested: List[str]) -> List[str]:
"""Map Micropub uids or target ids to configured target ids"""
targets = get_targets()
by_uid = {t.uid.rstrip("/"): t.id for t in targets.values()}
chosen = []
for value in requested:
value = (value or "").strip()
target_id = value if value in targets else by_uid.get(value.rstrip("/"))
if target_id and target_id not in chosen:
chosen.append(target_id)
return chosen
# --- Queue -----------------------------------------------------------------
def select_targets(note_id: int, target_ids: List[str]) -> None:
"""
Set which targets a note goes to, and queue the reconcile
Targets not listed are disabled (their remote post, if any, is deleted).
Nothing is recorded for unlisted targets the note never went to.
"""
db = get_db(current_app)
for target_id in get_targets():
enabled = 1 if target_id in target_ids else 0
existing = db.execute(
"SELECT id FROM syndications WHERE note_id = ? AND target = ?",
(note_id, target_id),
).fetchone()
if existing:
db.execute(
"""
UPDATE syndications
SET enabled = ?, state = 'pending', version = version + 1, attempts = 0,
last_error = NULL, next_attempt_at = datetime('now'), updated_at = datetime('now')
WHERE id = ?
""",
(enabled, existing["id"]),
)
elif enabled:
db.execute(
"INSERT INTO syndications (note_id, target, enabled) VALUES (?, ?, 1)",
(note_id, target_id),
)
db.commit()
def note_changed(note_id: int) -> None:
"""Queue a reconcile for every target the note was sent to"""
db = get_db(current_app)
db.execute(
"""
UPDATE syndications
SET state = 'pending', version = version + 1, attempts = 0, last_error = NULL,
next_attempt_at = datetime('now'), updated_at = datetime('now')
WHERE note_id = ?
""",
(note_id,),
)
db.commit()
def retry(syndication_id: int) -> None:
"""Retry a failed or pending row now"""
db = get_db(current_app)
db.execute(
"""
UPDATE syndications
SET state = 'pending', attempts = 0, last_error = NULL, next_attempt_at = datetime('now')
WHERE id = ?
""",
(syndication_id,),
)
db.commit()
def get_note_syndications(note_id: int) -> List[dict]:
"""Rows for a note, with the target's display name"""
targets = get_targets()
rows = get_db(current_app).execute(
"SELECT * FROM syndications WHERE note_id = ? ORDER BY target", (note_id,)
).fetchall()
result = []
for row in rows:
item = dict(row)
item["name"] = targets[row["target"]].name if row["target"] in targets else row["target"]
result.append(item)
return result
# --- Composing --------------------------------------------------------------
def _hashtag(display_name: str) -> str:
word = re.sub(r"[^\w]", "", display_name)
return f"#{word}" if word else ""
def compose_text(note, permalink: str, tags: List[str], max_chars: int,
body: Optional[str] = None) -> str:
"""
Full note (Markdown unless `body` is given), then the permalink and
hashtags; shortened at a word boundary if too long. On short targets the
hashtags may use at most a third of the space.
"""
hashtags = ""
for h in (_hashtag(t) for t in tags):
if h and len(hashtags) + len(h) + 1 <= max(max_chars // 3, 1):
hashtags = f"{hashtags} {h}".strip()
tail = f"\n\n{permalink}" + (f"\n\n{hashtags}" if hashtags else "")
body = (note.content if body is None else body).strip()
if len(body) + len(tail) > max_chars:
room = max_chars - len(tail) - 1
body = body[:room].rsplit(" ", 1)[0].rstrip() + "…"
# A photo post may have no text: start with the link (v2.6.5)
return (body + tail).lstrip()
_BLOCKS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "pre", "blockquote", "ul", "ol", "hr", "table"]
def _note_soup(markdown_text: str):
import markdown
from bs4 import BeautifulSoup
return BeautifulSoup(markdown.markdown(markdown_text, extensions=["fenced_code"]), "html.parser")
def _clean_url(url: str) -> str:
return url.rstrip(".,;:!?)\"'…")
def _is_web(url: str) -> bool:
return url.startswith(("http://", "https://"))
def note_links(markdown_text: str) -> List[str]:
"""Web links in a note, in reading order: Markdown links, <autolinks> and bare URLs"""
from bs4 import NavigableString
links = []
for node in _note_soup(markdown_text).descendants:
if getattr(node, "name", None) == "a" and _is_web(node.get("href", "")):
links.append(node["href"])
elif isinstance(node, NavigableString) and not node.find_parent("a"):
links.extend(_clean_url(m.group(0)) for m in _URL_RE.finditer(str(node)))
return list(dict.fromkeys(links))
def plain_text(markdown_text: str, links: str = "text", max_urls: Optional[int] = None) -> str:
"""Markdown as readable plain text (for targets without Markdown). See `plain_text_links`."""
return plain_text_links(markdown_text, links, max_urls)[0]
def plain_text_links(markdown_text: str, links: str = "text",
max_urls: Optional[int] = None) -> tuple:
"""
Markdown as plain text, and the links whose text stands in for a URL.
A Markdown link `[text](url)` becomes:
links="text" just its text
links="inline" "text (url)", for targets with no clickable link text;
after `max_urls` different URLs, just the text
links="anchor" its text, listed in the returned anchors as (text, url)
so the target can make that text clickable
A link whose text is the URL itself stays a bare URL.
"""
from bs4 import NavigableString
soup = _note_soup(markdown_text)
for img in soup.find_all("img"):
img.decompose()
anchors = []
seen = set()
for node in list(soup.descendants): # bare URLs count towards max_urls first
if isinstance(node, NavigableString) and not node.find_parent("a"):
seen.update(_clean_url(m.group(0)) for m in _URL_RE.finditer(str(node)))
for a in soup.find_all("a"):
href, label = a.get("href", ""), a.get_text()
if not _is_web(href):
continue
bare = re.sub(r"^https?://", "", href).rstrip("/")
if label.strip().rstrip("/") in (href.rstrip("/"), bare):
a.replace_with(href)
seen.add(href)
elif links == "inline" and (href in seen or max_urls is None or len(seen) < max_urls):
a.replace_with(f"{label} ({href})")
seen.add(href)
elif links == "anchor":
anchors.append((label, href))
for node in soup.find_all(string=True):
if not node.strip() and "\n" in node and node.parent.name != "pre":
node.extract() # layout newlines between tags; we add our own
for br in soup.find_all("br"):
br.replace_with("\n")
for li in soup.find_all("li"):
li.insert(0, "• ")
li.append("\n")
for block in soup.find_all(_BLOCKS):
block.append("\n\n")
text = soup.get_text()
text = re.sub(r"[ \t]+\n", "\n", text)
return re.sub(r"\n{3,}", "\n\n", text).strip(), anchors
_URL_RE = re.compile(r"https?://[^\s<>\"]+")
_TAG_RE = re.compile(r"(?<![\w#])#(\w*[^\W\d]\w*)")
def bluesky_facets(text: str, anchors: List[tuple] = ()) -> List[dict]:
"""
Link and hashtag facets (UTF-8 byte ranges) for a Bluesky post.
`anchors` are (text, url) pairs: that text, where it appears, links to url.
"""
def span(start: int, end: int) -> dict:
return {"byteStart": len(text[:start].encode()), "byteEnd": len(text[:end].encode())}
facets = []
for m in _URL_RE.finditer(text):
url = _clean_url(m.group(0))
facets.append({"index": span(m.start(), m.start() + len(url)),
"features": [{"$type": "app.bsky.richtext.facet#link", "uri": url}]})
taken = [(m.start(), m.end()) for m in _URL_RE.finditer(text)]
pos = 0
for label, url in anchors:
start = text.find(label, pos) if label.strip() else -1
if start < 0:
continue # shortened away
end = start + len(label)
if any(a < end and start < b for a, b in taken):
continue
taken.append((start, end))
pos = end
facets.append({"index": span(start, end),
"features": [{"$type": "app.bsky.richtext.facet#link", "uri": url}]})
urls = [(f["index"]["byteStart"], f["index"]["byteEnd"]) for f in facets]
facets.sort(key=lambda f: f["index"]["byteStart"])
for m in _TAG_RE.finditer(text):
index = span(m.start(), m.end())
if any(a <= index["byteStart"] < b for a, b in urls):
continue # a #fragment inside a URL
facets.append({"index": index,
"features": [{"$type": "app.bsky.richtext.facet#tag", "tag": m.group(1)}]})
return facets
_MD_IMAGE_RE = re.compile(r"!\[[^\]]*\]\([^)]*\)")
_HTML_IMAGE_RE = re.compile(r"<img\b[^>]*>", re.IGNORECASE)
def inline_images(markdown_text: str) -> List[tuple]:
"""(url, alt) for each image embedded in a note's text, in reading order"""
images = []
for img in _note_soup(markdown_text).find_all("img"):
if img.get("src"):
images.append((img["src"], img.get("alt") or ""))
return images
def strip_images(markdown_text: str) -> str:
"""A note's Markdown without its embedded images"""
text = _HTML_IMAGE_RE.sub("", _MD_IMAGE_RE.sub("", markdown_text))
return re.sub(r"\n{3,}", "\n\n", text).strip()
def _own_media_path(url: str) -> Optional[str]:
"""Path in the media library for one of this site's /media/ URLs"""
site = current_app.config["SITE_URL"].rstrip("/")
if url.startswith(site):
url = url[len(site):]
return url[len("/media/"):] if url.startswith("/media/") else None
def _media_files(note_id: int, limit: int, content: str = "") -> List[dict]:
"""
Image files for a note: the 'large' variant when there is one.
Attached photos come first, then images embedded in the text that are in
this site's media library. Some Micropub clients (OutPost, for one)
upload a photo and put it in the content rather than in `photo` (v2.6.6).
"""
from starpunk.media import get_note_media
media_dir = Path(current_app.config["DATA_PATH"]) / "media"
site = current_app.config["SITE_URL"].rstrip("/")
items = [(m, m.get("caption") or "") for m in get_note_media(note_id)]
seen = {m["path"] for m, _ in items}
for url, alt in inline_images(content) if content else []:
path = _own_media_path(url)
if not path or path in seen:
continue
media = _media_by_path(path)
if media:
seen.add(path)
items.append((media, alt or media.get("caption") or ""))
files = []
for media, description in items[:limit]:
variant = (media.get("variants") or {}).get("large")
rel = variant["path"] if variant else media["path"]
files.append({
"path": media_dir / rel,
"url": f"{site}/media/{rel}", # for targets that fetch images themselves
"mime_type": media.get("mime_type") or "image/jpeg",
"description": description,
})
return files
def _media_by_path(path: str) -> Optional[dict]:
"""A media row with its variants, found by its path in the library"""
db = get_db(current_app)
row = db.execute("SELECT id, path, mime_type FROM media WHERE path = ?", (path,)).fetchone()
if not row:
return None
variants = {
v["variant_type"]: {"path": v["path"]}
for v in db.execute(
"SELECT variant_type, path FROM media_variants WHERE media_id = ?", (row["id"],)
)
}
return {"path": row["path"], "mime_type": row["mime_type"], "variants": variants, "caption": ""}
# --- Worker -----------------------------------------------------------------
def _claim(db, row_id: int) -> bool:
cursor = db.execute(
f"""
UPDATE syndications SET claimed_until = datetime('now', '+{CLAIM_MINUTES} minutes')
WHERE id = ? AND state = 'pending'
AND (claimed_until IS NULL OR claimed_until < datetime('now'))
""",
(row_id,),
)
db.commit()
return cursor.rowcount == 1
def _finish(db, row, remote_id: Optional[str], remote_url: Optional[str]) -> None:
# Only mark synced if nothing changed while we were sending
db.execute(
"""
UPDATE syndications
SET remote_id = ?, remote_url = ?, claimed_until = NULL, attempts = 0, last_error = NULL,
state = CASE WHEN version = ? THEN 'synced' ELSE 'pending' END,
updated_at = datetime('now')
WHERE id = ?
""",
(remote_id, remote_url, row["version"], row["id"]),
)
db.commit()
def _fail(db, row, error: SyndicationError) -> None:
attempts = row["attempts"] + 1
failed = error.permanent or attempts >= MAX_ATTEMPTS
delay = min(30 * (2 ** attempts), 6 * 3600)
db.execute(
f"""
UPDATE syndications
SET attempts = ?, last_error = ?, claimed_until = NULL,
state = CASE WHEN version = ? THEN ? ELSE 'pending' END,
next_attempt_at = datetime('now', '+{delay} seconds'), updated_at = datetime('now')
WHERE id = ?
""",
(attempts, str(error)[:500], row["version"], "failed" if failed else "pending", row["id"]),
)
db.commit()
current_app.logger.warning(
f"Syndication {row['target']} for note {row['note_id']} failed "
f"(attempt {attempts}{', giving up' if failed else ''}): {error}"
)
def _target_title(url: Optional[str]) -> Optional[str]:
"""Fetched title of a liked/reposted post, if we have one"""
if not url:
return None
try:
from starpunk.webmention_queries import get_reply_context
context = get_reply_context(url) or {}
except Exception:
return None
title = context.get("title") or ""
if not title and context.get("author_name"):
title = f"Post by {context['author_name']}"
return " ".join(title.split())[:200] or None
def _reconcile(row, target: Target) -> tuple:
"""Bring the remote post in line with the note. Returns (remote_id, remote_url)."""
from starpunk.notes import get_note
from starpunk.tags import get_note_tags
note = get_note(id=row["note_id"], load_content=True)
wanted = bool(row["enabled"]) and note is not None and note.published
if not wanted:
if row["remote_id"]:
target.client.delete(row["remote_id"])
return None, None
if row["remote_id"] and not target.client.can_edit:
return row["remote_id"], row["remote_url"] # posted already; edits don't follow
site = current_app.config["SITE_URL"].rstrip("/")
post = Post(
note=note,
permalink=f"{site}/note/{note.slug}",
tags=[t["display_name"] for t in get_note_tags(note.id, entered_order=True)],
max_chars=target.max_chars,
media=_media_files(note.id, target.max_media, note.content)
if note.kind == "note" else [],
kind=note.kind,
target_url=note.reacts_to,
target_title=_target_title(note.reacts_to),
)
if row["remote_id"]:
try:
result = target.client.update(row["remote_id"], post)
return result["id"], result["url"]
except RemoteGone:
pass # deleted on the remote side; post it again
result = target.client.create(
post, key=f"starpunk-{row['id']}-{row['version']}", retrying=row["attempts"] > 0
)
return result["id"], result["url"]
def process_due(limit: int = 5) -> int:
"""Reconcile up to `limit` pending rows. Returns how many were handled."""
targets = get_targets()
if not targets:
return 0
for target in targets.values():
maintain = getattr(target.client, "maintain", None)
if maintain:
try:
maintain()
except Exception as e:
current_app.logger.error(f"Syndication {target.id} upkeep failed: {e}")
db = get_db(current_app)
rows = db.execute(
"""
SELECT * FROM syndications
WHERE state = 'pending' AND next_attempt_at <= datetime('now')
AND (claimed_until IS NULL OR claimed_until < datetime('now'))
ORDER BY next_attempt_at LIMIT ?
""",
(limit,),
).fetchall()
handled = 0
for row in rows:
if not _claim(db, row["id"]):
continue # another worker took it
handled += 1
target = targets.get(row["target"])
if target is None:
_fail(db, row, SyndicationError(f"target {row['target']} is not configured", permanent=True))
continue
try:
remote_id, remote_url = _reconcile(row, target)
except SyndicationError as e:
_fail(db, row, e)
continue
except Exception as e: # never let one bad note stop the worker
_fail(db, row, SyndicationError(f"unexpected error: {e}"))
continue
_finish(db, row, remote_id, remote_url)
if remote_id:
current_app.logger.info(f"Syndicated note {row['note_id']} to {row['target']}: {remote_url}")
return handled
class SyndicationWorker(threading.Thread):
"""Daemon thread that drains the syndication queue"""
def __init__(self, app, poll_interval: int = 10):
super().__init__(daemon=True, name="syndication-worker")
self.app = app
self.poll_interval = poll_interval
self._stop_event = threading.Event()
def run(self):
while not self._stop_event.is_set():
try:
with self.app.app_context():
handled = process_due()
except Exception as e:
self.app.logger.error(f"Syndication worker error: {e}", exc_info=True)
handled = 0
if handled == 0:
self._stop_event.wait(self.poll_interval)
def stop(self):
self._stop_event.set()