BlueskyClient behind the ADR-065 reconcile loop: plain text with link and hashtag facets, images under 1 MB with alt text, link cards, app-password sessions with refresh. Deletes follow; edits don't (ADR-066). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
717 lines
27 KiB
Python
717 lines
27 KiB
Python
"""
|
|
Crossposting (POSSE) for StarPunk (v2.1.0 ADR-065, v2.2.0 ADR-066)
|
|
|
|
Published notes are copied to syndication targets, and later edits and
|
|
deletes follow them. Two target kinds exist: a GoToSocial (or any
|
|
Mastodon-API) account, and a Bluesky account. Bluesky posts can't be edited,
|
|
so there only deletes follow.
|
|
|
|
Each (note, target) pair has a row in `syndications`. Changing a note marks
|
|
its rows pending; a background worker then reconciles the remote post with
|
|
the note:
|
|
|
|
note published + target enabled, no remote post -> create
|
|
note published + target enabled, remote post -> update
|
|
otherwise, remote post exists -> delete
|
|
|
|
Rows carry a version so an edit made while a send is in flight is not lost.
|
|
"""
|
|
|
|
import io
|
|
import re
|
|
import threading
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional
|
|
|
|
import httpx
|
|
from flask import current_app
|
|
|
|
from starpunk.database import get_db
|
|
|
|
MAX_ATTEMPTS = 8
|
|
CLAIM_MINUTES = 5
|
|
|
|
|
|
class SyndicationError(Exception):
|
|
"""A send failed. permanent=True means retrying won't help."""
|
|
|
|
def __init__(self, message: str, permanent: bool = False):
|
|
super().__init__(message)
|
|
self.permanent = permanent
|
|
|
|
|
|
class RemoteGone(SyndicationError):
|
|
"""The remote post no longer exists"""
|
|
|
|
|
|
class MastodonClient:
|
|
"""Minimal Mastodon-API client (works with GoToSocial)"""
|
|
|
|
def __init__(self, base_url: str, token: str, visibility: str = "public",
|
|
http: Optional[httpx.Client] = None):
|
|
self.base_url = base_url.rstrip("/")
|
|
self.visibility = visibility
|
|
self.http = http or httpx.Client(timeout=30.0)
|
|
self.headers = {"Authorization": f"Bearer {token}"}
|
|
|
|
def _call(self, method: str, path: str, **kwargs) -> httpx.Response:
|
|
headers = {**self.headers, **kwargs.pop("headers", {})}
|
|
try:
|
|
response = self.http.request(method, self.base_url + path, headers=headers, **kwargs)
|
|
except httpx.HTTPError as e:
|
|
raise SyndicationError(f"{method} {path}: {e}")
|
|
if response.status_code == 404:
|
|
raise RemoteGone(f"{method} {path}: not found", permanent=True)
|
|
if response.status_code >= 400:
|
|
# 4xx other than rate limiting won't fix itself
|
|
permanent = 400 <= response.status_code < 500 and response.status_code != 429
|
|
raise SyndicationError(
|
|
f"{method} {path}: HTTP {response.status_code} {response.text[:200]}",
|
|
permanent=permanent,
|
|
)
|
|
return response
|
|
|
|
def upload_media(self, path: Path, mime_type: str, description: str) -> str:
|
|
with open(path, "rb") as f:
|
|
response = self._call(
|
|
"POST", "/api/v2/media",
|
|
files={"file": (path.name, f, mime_type)},
|
|
data={"description": description or ""},
|
|
)
|
|
return response.json()["id"]
|
|
|
|
def _status_body(self, text: str, media_ids: List[str]) -> dict:
|
|
return {
|
|
"status": text,
|
|
"media_ids": media_ids,
|
|
"visibility": self.visibility,
|
|
"content_type": "text/markdown",
|
|
}
|
|
|
|
def create_status(self, text: str, media_ids: List[str], idempotency_key: str) -> dict:
|
|
response = self._call(
|
|
"POST", "/api/v1/statuses",
|
|
json=self._status_body(text, media_ids),
|
|
headers={"Idempotency-Key": idempotency_key},
|
|
)
|
|
data = response.json()
|
|
return {"id": data["id"], "url": data.get("url") or data.get("uri")}
|
|
|
|
def update_status(self, remote_id: str, text: str, media_ids: List[str]) -> dict:
|
|
response = self._call("PUT", f"/api/v1/statuses/{remote_id}",
|
|
json=self._status_body(text, media_ids))
|
|
data = response.json()
|
|
return {"id": data["id"], "url": data.get("url") or data.get("uri")}
|
|
|
|
def delete_status(self, remote_id: str) -> None:
|
|
try:
|
|
self._call("DELETE", f"/api/v1/statuses/{remote_id}")
|
|
except RemoteGone:
|
|
pass # already gone
|
|
|
|
# Interface used by the reconcile loop
|
|
can_edit = True
|
|
|
|
def _prepare(self, post: "Post") -> tuple:
|
|
text = compose_text(post.note, post.permalink, post.tags, post.max_chars)
|
|
media_ids = [self.upload_media(f["path"], f["mime_type"], f["description"]) for f in post.media]
|
|
return text, media_ids
|
|
|
|
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
|
|
text, media_ids = self._prepare(post)
|
|
return self.create_status(text, media_ids, idempotency_key=key)
|
|
|
|
def update(self, remote_id: str, post: "Post") -> dict:
|
|
text, media_ids = self._prepare(post)
|
|
return self.update_status(remote_id, text, media_ids)
|
|
|
|
def delete(self, remote_id: str) -> None:
|
|
self.delete_status(remote_id)
|
|
|
|
|
|
BLUESKY_BLOB_LIMIT = 1_000_000 # app.bsky.embed.images maxSize
|
|
_TID_CHARS = "234567abcdefghijklmnopqrstuvwxyz"
|
|
|
|
|
|
class BlueskyClient:
|
|
"""
|
|
Minimal AT Protocol client for posting to Bluesky with an app password
|
|
|
|
Posts are plain text with link and hashtag facets. Images go in an
|
|
images embed; a note without images gets a link card instead.
|
|
"""
|
|
|
|
can_edit = False # Bluesky has no edits; only deletes follow the note
|
|
POST = "app.bsky.feed.post"
|
|
|
|
def __init__(self, service: str, identifier: str, password: str,
|
|
http: Optional[httpx.Client] = None):
|
|
self.service = service.rstrip("/")
|
|
self.identifier = identifier
|
|
self.password = password
|
|
self.http = http or httpx.Client(timeout=30.0)
|
|
self.pds = self.service
|
|
self.did: Optional[str] = None
|
|
self.handle = identifier
|
|
self._access: Optional[str] = None
|
|
self._refresh: Optional[str] = None
|
|
self._lock = threading.Lock()
|
|
|
|
# --- XRPC plumbing ---
|
|
|
|
def _raw(self, method: str, nsid: str, base: str, token: Optional[str], **kwargs) -> httpx.Response:
|
|
headers = kwargs.pop("headers", {})
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
try:
|
|
return self.http.request(method, f"{base}/xrpc/{nsid}", headers=headers, **kwargs)
|
|
except httpx.HTTPError as e:
|
|
raise SyndicationError(f"{nsid}: {e}")
|
|
|
|
@staticmethod
|
|
def _error_name(response: httpx.Response) -> str:
|
|
try:
|
|
return response.json().get("error") or ""
|
|
except ValueError:
|
|
return ""
|
|
|
|
def _raise_for(self, nsid: str, response: httpx.Response) -> None:
|
|
if response.status_code < 400:
|
|
return
|
|
permanent = 400 <= response.status_code < 500 and response.status_code != 429
|
|
raise SyndicationError(
|
|
f"{nsid}: HTTP {response.status_code} {response.text[:200]}", permanent=permanent
|
|
)
|
|
|
|
def _login(self) -> None:
|
|
response = None
|
|
if self._refresh:
|
|
response = self._raw("POST", "com.atproto.server.refreshSession", self.service, self._refresh)
|
|
if response.status_code >= 400:
|
|
response = None
|
|
if response is None:
|
|
response = self._raw("POST", "com.atproto.server.createSession", self.service, None,
|
|
json={"identifier": self.identifier, "password": self.password})
|
|
self._raise_for("com.atproto.server.createSession", response)
|
|
data = response.json()
|
|
self._access, self._refresh = data["accessJwt"], data["refreshJwt"]
|
|
self.did = data["did"]
|
|
self.handle = data.get("handle") or self.handle
|
|
for service in (data.get("didDoc") or {}).get("service", []):
|
|
if service.get("id", "").endswith("#atproto_pds") and service.get("serviceEndpoint"):
|
|
self.pds = service["serviceEndpoint"].rstrip("/")
|
|
|
|
def _call(self, method: str, nsid: str, **kwargs) -> httpx.Response:
|
|
with self._lock:
|
|
if not self._access:
|
|
self._login()
|
|
response = self._raw(method, nsid, self.pds, self._access, **kwargs)
|
|
if response.status_code == 401 or self._error_name(response) == "ExpiredToken":
|
|
self._access = None
|
|
self._login()
|
|
response = self._raw(method, nsid, self.pds, self._access, **kwargs)
|
|
self._raise_for(nsid, response)
|
|
return response
|
|
|
|
# --- Posts ---
|
|
|
|
def upload_blob(self, data: bytes, mime_type: str) -> dict:
|
|
response = self._call("POST", "com.atproto.repo.uploadBlob", content=data,
|
|
headers={"Content-Type": mime_type})
|
|
return response.json()["blob"]
|
|
|
|
def _image(self, media: dict) -> dict:
|
|
data, mime_type, width, height = _fit_image(media["path"], media["mime_type"], BLUESKY_BLOB_LIMIT)
|
|
image = {"image": self.upload_blob(data, mime_type), "alt": media["description"] or ""}
|
|
if width and height:
|
|
image["aspectRatio"] = {"width": width, "height": height}
|
|
return image
|
|
|
|
def _record(self, post: "Post") -> dict:
|
|
text = compose_text(post.note, post.permalink, post.tags, post.max_chars,
|
|
body=plain_text(post.note.content))
|
|
record = {
|
|
"$type": self.POST,
|
|
"text": text,
|
|
"createdAt": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
|
|
}
|
|
facets = bluesky_facets(text)
|
|
if facets:
|
|
record["facets"] = facets
|
|
if post.media:
|
|
record["embed"] = {"$type": "app.bsky.embed.images",
|
|
"images": [self._image(m) for m in post.media]}
|
|
else:
|
|
record["embed"] = {"$type": "app.bsky.embed.external", "external": {
|
|
"uri": post.permalink,
|
|
"title": post.note.title[:300],
|
|
"description": plain_text(post.note.content)[:300],
|
|
}}
|
|
return record
|
|
|
|
def _web_url(self, uri: str) -> str:
|
|
rkey = uri.rsplit("/", 1)[1]
|
|
return f"https://bsky.app/profile/{self.handle or self.did}/post/{rkey}"
|
|
|
|
def _find_existing(self, permalink: str) -> Optional[dict]:
|
|
"""A recent post that already links to this note (an earlier send whose reply was lost)"""
|
|
response = self._call("GET", "com.atproto.repo.listRecords",
|
|
params={"repo": self.did, "collection": self.POST, "limit": 25})
|
|
for item in response.json().get("records", []):
|
|
if permalink in (item.get("value") or {}).get("text", ""):
|
|
return {"id": item["uri"], "url": self._web_url(item["uri"])}
|
|
return None
|
|
|
|
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
|
|
if retrying:
|
|
if not self.did:
|
|
with self._lock:
|
|
self._login()
|
|
existing = self._find_existing(post.permalink)
|
|
if existing:
|
|
return existing
|
|
record = self._record(post)
|
|
response = self._call("POST", "com.atproto.repo.createRecord",
|
|
json={"repo": self.did or self.identifier, "collection": self.POST,
|
|
"record": record})
|
|
uri = response.json()["uri"]
|
|
return {"id": uri, "url": self._web_url(uri)}
|
|
|
|
def update(self, remote_id: str, post: "Post") -> dict:
|
|
raise NotImplementedError("Bluesky posts can't be edited")
|
|
|
|
def delete(self, remote_id: str) -> None:
|
|
# at://<did>/app.bsky.feed.post/<rkey>; deleting a missing record succeeds
|
|
repo, collection, rkey = remote_id[len("at://"):].split("/", 2)
|
|
self._call("POST", "com.atproto.repo.deleteRecord",
|
|
json={"repo": repo, "collection": collection, "rkey": rkey})
|
|
|
|
|
|
def _fit_image(path: Path, mime_type: str, limit: int) -> tuple:
|
|
"""Image bytes no bigger than `limit`, re-encoded as JPEG if needed. Returns (data, mime, w, h)."""
|
|
from PIL import Image
|
|
|
|
data = Path(path).read_bytes()
|
|
with Image.open(io.BytesIO(data)) as img:
|
|
width, height = img.size
|
|
if len(data) <= limit:
|
|
return data, mime_type, width, height
|
|
img = img.convert("RGB")
|
|
for max_side, quality in ((2000, 85), (2000, 75), (1600, 70), (1200, 65), (1000, 60)):
|
|
copy = img.copy()
|
|
copy.thumbnail((max_side, max_side))
|
|
buf = io.BytesIO()
|
|
copy.save(buf, format="JPEG", quality=quality, optimize=True)
|
|
if buf.tell() <= limit:
|
|
return buf.getvalue(), "image/jpeg", copy.size[0], copy.size[1]
|
|
raise SyndicationError(f"{Path(path).name}: image too large for Bluesky", permanent=True)
|
|
|
|
|
|
@dataclass
|
|
class Target:
|
|
id: str # stored in syndications.target
|
|
uid: str # Micropub syndicate-to uid
|
|
name: str # shown in the editor and Micropub clients
|
|
client: object # MastodonClient or BlueskyClient
|
|
max_chars: int = 5000
|
|
max_media: int = 4
|
|
|
|
|
|
@dataclass
|
|
class Post:
|
|
"""What a client needs to publish a note"""
|
|
note: object
|
|
permalink: str
|
|
tags: List[str]
|
|
max_chars: int
|
|
media: List[dict] = field(default_factory=list)
|
|
|
|
|
|
_client_cache: Dict[str, object] = {}
|
|
|
|
|
|
def targets_configured(config) -> bool:
|
|
"""Whether any crossposting target has credentials (used to start the worker)"""
|
|
return bool(
|
|
(config.get("GOTOSOCIAL_URL") and config.get("GOTOSOCIAL_ACCESS_TOKEN"))
|
|
or (config.get("BLUESKY_HANDLE") and config.get("BLUESKY_APP_PASSWORD"))
|
|
)
|
|
|
|
|
|
def get_targets() -> Dict[str, Target]:
|
|
"""Configured syndication targets, by id"""
|
|
config = current_app.config
|
|
targets: Dict[str, Target] = {}
|
|
|
|
url = (config.get("GOTOSOCIAL_URL") or "").rstrip("/")
|
|
token = config.get("GOTOSOCIAL_ACCESS_TOKEN") or ""
|
|
if url and token:
|
|
client = _client_cache.get(url)
|
|
if client is None:
|
|
client = MastodonClient(url, token, config.get("GOTOSOCIAL_VISIBILITY", "public"))
|
|
_client_cache[url] = client
|
|
targets["gotosocial"] = Target(
|
|
id="gotosocial",
|
|
uid=url + "/",
|
|
name=config.get("GOTOSOCIAL_NAME", "GoToSocial"),
|
|
client=client,
|
|
max_chars=int(config.get("GOTOSOCIAL_MAX_CHARS", 5000)),
|
|
max_media=int(config.get("GOTOSOCIAL_MAX_MEDIA", 4)),
|
|
)
|
|
|
|
handle = (config.get("BLUESKY_HANDLE") or "").lstrip("@")
|
|
password = config.get("BLUESKY_APP_PASSWORD") or ""
|
|
if handle and password:
|
|
service = (config.get("BLUESKY_SERVICE") or "https://bsky.social").rstrip("/")
|
|
key = f"bluesky:{service}:{handle}"
|
|
client = _client_cache.get(key)
|
|
if client is None:
|
|
client = BlueskyClient(service, handle, password)
|
|
_client_cache[key] = client
|
|
targets["bluesky"] = Target(
|
|
id="bluesky",
|
|
uid=f"https://bsky.app/profile/{handle}",
|
|
name=config.get("BLUESKY_NAME", "Bluesky"),
|
|
client=client,
|
|
max_chars=int(config.get("BLUESKY_MAX_CHARS", 300)),
|
|
max_media=4,
|
|
)
|
|
return targets
|
|
|
|
|
|
def resolve_targets(requested: List[str]) -> List[str]:
|
|
"""Map Micropub uids or target ids to configured target ids"""
|
|
targets = get_targets()
|
|
by_uid = {t.uid.rstrip("/"): t.id for t in targets.values()}
|
|
chosen = []
|
|
for value in requested:
|
|
value = (value or "").strip()
|
|
target_id = value if value in targets else by_uid.get(value.rstrip("/"))
|
|
if target_id and target_id not in chosen:
|
|
chosen.append(target_id)
|
|
return chosen
|
|
|
|
|
|
# --- Queue -----------------------------------------------------------------
|
|
|
|
def select_targets(note_id: int, target_ids: List[str]) -> None:
|
|
"""
|
|
Set which targets a note goes to, and queue the reconcile
|
|
|
|
Targets not listed are disabled (their remote post, if any, is deleted).
|
|
Nothing is recorded for unlisted targets the note never went to.
|
|
"""
|
|
db = get_db(current_app)
|
|
for target_id in get_targets():
|
|
enabled = 1 if target_id in target_ids else 0
|
|
existing = db.execute(
|
|
"SELECT id FROM syndications WHERE note_id = ? AND target = ?",
|
|
(note_id, target_id),
|
|
).fetchone()
|
|
if existing:
|
|
db.execute(
|
|
"""
|
|
UPDATE syndications
|
|
SET enabled = ?, state = 'pending', version = version + 1, attempts = 0,
|
|
last_error = NULL, next_attempt_at = datetime('now'), updated_at = datetime('now')
|
|
WHERE id = ?
|
|
""",
|
|
(enabled, existing["id"]),
|
|
)
|
|
elif enabled:
|
|
db.execute(
|
|
"INSERT INTO syndications (note_id, target, enabled) VALUES (?, ?, 1)",
|
|
(note_id, target_id),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def note_changed(note_id: int) -> None:
|
|
"""Queue a reconcile for every target the note was sent to"""
|
|
db = get_db(current_app)
|
|
db.execute(
|
|
"""
|
|
UPDATE syndications
|
|
SET state = 'pending', version = version + 1, attempts = 0, last_error = NULL,
|
|
next_attempt_at = datetime('now'), updated_at = datetime('now')
|
|
WHERE note_id = ?
|
|
""",
|
|
(note_id,),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def retry(syndication_id: int) -> None:
|
|
"""Retry a failed or pending row now"""
|
|
db = get_db(current_app)
|
|
db.execute(
|
|
"""
|
|
UPDATE syndications
|
|
SET state = 'pending', attempts = 0, last_error = NULL, next_attempt_at = datetime('now')
|
|
WHERE id = ?
|
|
""",
|
|
(syndication_id,),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def get_note_syndications(note_id: int) -> List[dict]:
|
|
"""Rows for a note, with the target's display name"""
|
|
targets = get_targets()
|
|
rows = get_db(current_app).execute(
|
|
"SELECT * FROM syndications WHERE note_id = ? ORDER BY target", (note_id,)
|
|
).fetchall()
|
|
result = []
|
|
for row in rows:
|
|
item = dict(row)
|
|
item["name"] = targets[row["target"]].name if row["target"] in targets else row["target"]
|
|
result.append(item)
|
|
return result
|
|
|
|
|
|
# --- Composing --------------------------------------------------------------
|
|
|
|
def _hashtag(display_name: str) -> str:
|
|
word = re.sub(r"[^\w]", "", display_name)
|
|
return f"#{word}" if word else ""
|
|
|
|
|
|
def compose_text(note, permalink: str, tags: List[str], max_chars: int,
|
|
body: Optional[str] = None) -> str:
|
|
"""
|
|
Full note (Markdown unless `body` is given), then the permalink and
|
|
hashtags; shortened at a word boundary if too long. On short targets the
|
|
hashtags may use at most a third of the space.
|
|
"""
|
|
hashtags = ""
|
|
for h in (_hashtag(t) for t in tags):
|
|
if h and len(hashtags) + len(h) + 1 <= max(max_chars // 3, 1):
|
|
hashtags = f"{hashtags} {h}".strip()
|
|
tail = f"\n\n{permalink}" + (f"\n\n{hashtags}" if hashtags else "")
|
|
body = (note.content if body is None else body).strip()
|
|
if len(body) + len(tail) > max_chars:
|
|
room = max_chars - len(tail) - 1
|
|
body = body[:room].rsplit(" ", 1)[0].rstrip() + "…"
|
|
return body + tail
|
|
|
|
|
|
_BLOCKS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "pre", "blockquote", "ul", "ol", "hr", "table"]
|
|
|
|
|
|
def plain_text(markdown_text: str) -> str:
|
|
"""Markdown as readable plain text (for targets without Markdown)"""
|
|
import markdown
|
|
from bs4 import BeautifulSoup
|
|
|
|
soup = BeautifulSoup(markdown.markdown(markdown_text, extensions=["fenced_code"]), "html.parser")
|
|
for img in soup.find_all("img"):
|
|
img.decompose()
|
|
for node in soup.find_all(string=True):
|
|
if not node.strip() and "\n" in node and node.parent.name != "pre":
|
|
node.extract() # layout newlines between tags; we add our own
|
|
for br in soup.find_all("br"):
|
|
br.replace_with("\n")
|
|
for li in soup.find_all("li"):
|
|
li.insert(0, "• ")
|
|
li.append("\n")
|
|
for block in soup.find_all(_BLOCKS):
|
|
block.append("\n\n")
|
|
text = soup.get_text()
|
|
text = re.sub(r"[ \t]+\n", "\n", text)
|
|
return re.sub(r"\n{3,}", "\n\n", text).strip()
|
|
|
|
|
|
_URL_RE = re.compile(r"https?://[^\s<>\"]+")
|
|
_TAG_RE = re.compile(r"(?<![\w#])#(\w*[^\W\d]\w*)")
|
|
|
|
|
|
def bluesky_facets(text: str) -> List[dict]:
|
|
"""Link and hashtag facets (UTF-8 byte ranges) for a Bluesky post"""
|
|
def span(start: int, end: int) -> dict:
|
|
return {"byteStart": len(text[:start].encode()), "byteEnd": len(text[:end].encode())}
|
|
|
|
facets = []
|
|
for m in _URL_RE.finditer(text):
|
|
url = m.group(0).rstrip(".,;:!?)\"'…")
|
|
facets.append({"index": span(m.start(), m.start() + len(url)),
|
|
"features": [{"$type": "app.bsky.richtext.facet#link", "uri": url}]})
|
|
urls = [(f["index"]["byteStart"], f["index"]["byteEnd"]) for f in facets]
|
|
for m in _TAG_RE.finditer(text):
|
|
index = span(m.start(), m.end())
|
|
if any(a <= index["byteStart"] < b for a, b in urls):
|
|
continue # a #fragment inside a URL
|
|
facets.append({"index": index,
|
|
"features": [{"$type": "app.bsky.richtext.facet#tag", "tag": m.group(1)}]})
|
|
return facets
|
|
|
|
|
|
def _media_files(note_id: int, limit: int) -> List[dict]:
|
|
"""Image files for a note: the 'large' variant when there is one"""
|
|
from starpunk.media import get_note_media
|
|
|
|
media_dir = Path(current_app.config["DATA_PATH"]) / "media"
|
|
files = []
|
|
for media in get_note_media(note_id)[:limit]:
|
|
variant = (media.get("variants") or {}).get("large")
|
|
rel = variant["path"] if variant else media["path"]
|
|
files.append({
|
|
"path": media_dir / rel,
|
|
"mime_type": media.get("mime_type") or "image/jpeg",
|
|
"description": media.get("caption") or "",
|
|
})
|
|
return files
|
|
|
|
|
|
# --- Worker -----------------------------------------------------------------
|
|
|
|
def _claim(db, row_id: int) -> bool:
|
|
cursor = db.execute(
|
|
f"""
|
|
UPDATE syndications SET claimed_until = datetime('now', '+{CLAIM_MINUTES} minutes')
|
|
WHERE id = ? AND state = 'pending'
|
|
AND (claimed_until IS NULL OR claimed_until < datetime('now'))
|
|
""",
|
|
(row_id,),
|
|
)
|
|
db.commit()
|
|
return cursor.rowcount == 1
|
|
|
|
|
|
def _finish(db, row, remote_id: Optional[str], remote_url: Optional[str]) -> None:
|
|
# Only mark synced if nothing changed while we were sending
|
|
db.execute(
|
|
"""
|
|
UPDATE syndications
|
|
SET remote_id = ?, remote_url = ?, claimed_until = NULL, attempts = 0, last_error = NULL,
|
|
state = CASE WHEN version = ? THEN 'synced' ELSE 'pending' END,
|
|
updated_at = datetime('now')
|
|
WHERE id = ?
|
|
""",
|
|
(remote_id, remote_url, row["version"], row["id"]),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def _fail(db, row, error: SyndicationError) -> None:
|
|
attempts = row["attempts"] + 1
|
|
failed = error.permanent or attempts >= MAX_ATTEMPTS
|
|
delay = min(30 * (2 ** attempts), 6 * 3600)
|
|
db.execute(
|
|
f"""
|
|
UPDATE syndications
|
|
SET attempts = ?, last_error = ?, claimed_until = NULL,
|
|
state = CASE WHEN version = ? THEN ? ELSE 'pending' END,
|
|
next_attempt_at = datetime('now', '+{delay} seconds'), updated_at = datetime('now')
|
|
WHERE id = ?
|
|
""",
|
|
(attempts, str(error)[:500], row["version"], "failed" if failed else "pending", row["id"]),
|
|
)
|
|
db.commit()
|
|
current_app.logger.warning(
|
|
f"Syndication {row['target']} for note {row['note_id']} failed "
|
|
f"(attempt {attempts}{', giving up' if failed else ''}): {error}"
|
|
)
|
|
|
|
|
|
def _reconcile(row, target: Target) -> tuple:
|
|
"""Bring the remote post in line with the note. Returns (remote_id, remote_url)."""
|
|
from starpunk.notes import get_note
|
|
from starpunk.tags import get_note_tags
|
|
|
|
note = get_note(id=row["note_id"], load_content=True)
|
|
wanted = bool(row["enabled"]) and note is not None and note.published
|
|
|
|
if not wanted:
|
|
if row["remote_id"]:
|
|
target.client.delete(row["remote_id"])
|
|
return None, None
|
|
|
|
if row["remote_id"] and not target.client.can_edit:
|
|
return row["remote_id"], row["remote_url"] # posted already; edits don't follow
|
|
|
|
site = current_app.config["SITE_URL"].rstrip("/")
|
|
post = Post(
|
|
note=note,
|
|
permalink=f"{site}/note/{note.slug}",
|
|
tags=[t["display_name"] for t in get_note_tags(note.id)],
|
|
max_chars=target.max_chars,
|
|
media=_media_files(note.id, target.max_media),
|
|
)
|
|
|
|
if row["remote_id"]:
|
|
try:
|
|
result = target.client.update(row["remote_id"], post)
|
|
return result["id"], result["url"]
|
|
except RemoteGone:
|
|
pass # deleted on the remote side; post it again
|
|
|
|
result = target.client.create(
|
|
post, key=f"starpunk-{row['id']}-{row['version']}", retrying=row["attempts"] > 0
|
|
)
|
|
return result["id"], result["url"]
|
|
|
|
|
|
def process_due(limit: int = 5) -> int:
|
|
"""Reconcile up to `limit` pending rows. Returns how many were handled."""
|
|
targets = get_targets()
|
|
if not targets:
|
|
return 0
|
|
db = get_db(current_app)
|
|
rows = db.execute(
|
|
"""
|
|
SELECT * FROM syndications
|
|
WHERE state = 'pending' AND next_attempt_at <= datetime('now')
|
|
AND (claimed_until IS NULL OR claimed_until < datetime('now'))
|
|
ORDER BY next_attempt_at LIMIT ?
|
|
""",
|
|
(limit,),
|
|
).fetchall()
|
|
|
|
handled = 0
|
|
for row in rows:
|
|
if not _claim(db, row["id"]):
|
|
continue # another worker took it
|
|
handled += 1
|
|
target = targets.get(row["target"])
|
|
if target is None:
|
|
_fail(db, row, SyndicationError(f"target {row['target']} is not configured", permanent=True))
|
|
continue
|
|
try:
|
|
remote_id, remote_url = _reconcile(row, target)
|
|
except SyndicationError as e:
|
|
_fail(db, row, e)
|
|
continue
|
|
except Exception as e: # never let one bad note stop the worker
|
|
_fail(db, row, SyndicationError(f"unexpected error: {e}"))
|
|
continue
|
|
_finish(db, row, remote_id, remote_url)
|
|
if remote_id:
|
|
current_app.logger.info(f"Syndicated note {row['note_id']} to {row['target']}: {remote_url}")
|
|
return handled
|
|
|
|
|
|
class SyndicationWorker(threading.Thread):
|
|
"""Daemon thread that drains the syndication queue"""
|
|
|
|
def __init__(self, app, poll_interval: int = 10):
|
|
super().__init__(daemon=True, name="syndication-worker")
|
|
self.app = app
|
|
self.poll_interval = poll_interval
|
|
self._stop_event = threading.Event()
|
|
|
|
def run(self):
|
|
while not self._stop_event.is_set():
|
|
try:
|
|
with self.app.app_context():
|
|
handled = process_due()
|
|
except Exception as e:
|
|
self.app.logger.error(f"Syndication worker error: {e}", exc_info=True)
|
|
handled = 0
|
|
if handled == 0:
|
|
self._stop_event.wait(self.poll_interval)
|
|
|
|
def stop(self):
|
|
self._stop_event.set()
|