Files
StarPunk/starpunk/syndication.py
T
PhilandClaude Opus 5.5 e92c27a84c feat: Crosspost notes to Bluesky (v2.2.0)
BlueskyClient behind the ADR-065 reconcile loop: plain text with link and
hashtag facets, images under 1 MB with alt text, link cards, app-password
sessions with refresh. Deletes follow; edits don't (ADR-066).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-31 14:19:17 -06:00

717 lines
27 KiB
Python

"""
Crossposting (POSSE) for StarPunk (v2.1.0 ADR-065, v2.2.0 ADR-066)
Published notes are copied to syndication targets, and later edits and
deletes follow them. Two target kinds exist: a GoToSocial (or any
Mastodon-API) account, and a Bluesky account. Bluesky posts can't be edited,
so there only deletes follow.
Each (note, target) pair has a row in `syndications`. Changing a note marks
its rows pending; a background worker then reconciles the remote post with
the note:
note published + target enabled, no remote post -> create
note published + target enabled, remote post -> update
otherwise, remote post exists -> delete
Rows carry a version so an edit made while a send is in flight is not lost.
"""
import io
import re
import threading
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Dict, List, Optional
import httpx
from flask import current_app
from starpunk.database import get_db
MAX_ATTEMPTS = 8
CLAIM_MINUTES = 5
class SyndicationError(Exception):
"""A send failed. permanent=True means retrying won't help."""
def __init__(self, message: str, permanent: bool = False):
super().__init__(message)
self.permanent = permanent
class RemoteGone(SyndicationError):
"""The remote post no longer exists"""
class MastodonClient:
"""Minimal Mastodon-API client (works with GoToSocial)"""
def __init__(self, base_url: str, token: str, visibility: str = "public",
http: Optional[httpx.Client] = None):
self.base_url = base_url.rstrip("/")
self.visibility = visibility
self.http = http or httpx.Client(timeout=30.0)
self.headers = {"Authorization": f"Bearer {token}"}
def _call(self, method: str, path: str, **kwargs) -> httpx.Response:
headers = {**self.headers, **kwargs.pop("headers", {})}
try:
response = self.http.request(method, self.base_url + path, headers=headers, **kwargs)
except httpx.HTTPError as e:
raise SyndicationError(f"{method} {path}: {e}")
if response.status_code == 404:
raise RemoteGone(f"{method} {path}: not found", permanent=True)
if response.status_code >= 400:
# 4xx other than rate limiting won't fix itself
permanent = 400 <= response.status_code < 500 and response.status_code != 429
raise SyndicationError(
f"{method} {path}: HTTP {response.status_code} {response.text[:200]}",
permanent=permanent,
)
return response
def upload_media(self, path: Path, mime_type: str, description: str) -> str:
with open(path, "rb") as f:
response = self._call(
"POST", "/api/v2/media",
files={"file": (path.name, f, mime_type)},
data={"description": description or ""},
)
return response.json()["id"]
def _status_body(self, text: str, media_ids: List[str]) -> dict:
return {
"status": text,
"media_ids": media_ids,
"visibility": self.visibility,
"content_type": "text/markdown",
}
def create_status(self, text: str, media_ids: List[str], idempotency_key: str) -> dict:
response = self._call(
"POST", "/api/v1/statuses",
json=self._status_body(text, media_ids),
headers={"Idempotency-Key": idempotency_key},
)
data = response.json()
return {"id": data["id"], "url": data.get("url") or data.get("uri")}
def update_status(self, remote_id: str, text: str, media_ids: List[str]) -> dict:
response = self._call("PUT", f"/api/v1/statuses/{remote_id}",
json=self._status_body(text, media_ids))
data = response.json()
return {"id": data["id"], "url": data.get("url") or data.get("uri")}
def delete_status(self, remote_id: str) -> None:
try:
self._call("DELETE", f"/api/v1/statuses/{remote_id}")
except RemoteGone:
pass # already gone
# Interface used by the reconcile loop
can_edit = True
def _prepare(self, post: "Post") -> tuple:
text = compose_text(post.note, post.permalink, post.tags, post.max_chars)
media_ids = [self.upload_media(f["path"], f["mime_type"], f["description"]) for f in post.media]
return text, media_ids
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
text, media_ids = self._prepare(post)
return self.create_status(text, media_ids, idempotency_key=key)
def update(self, remote_id: str, post: "Post") -> dict:
text, media_ids = self._prepare(post)
return self.update_status(remote_id, text, media_ids)
def delete(self, remote_id: str) -> None:
self.delete_status(remote_id)
BLUESKY_BLOB_LIMIT = 1_000_000 # app.bsky.embed.images maxSize
_TID_CHARS = "234567abcdefghijklmnopqrstuvwxyz"
class BlueskyClient:
"""
Minimal AT Protocol client for posting to Bluesky with an app password
Posts are plain text with link and hashtag facets. Images go in an
images embed; a note without images gets a link card instead.
"""
can_edit = False # Bluesky has no edits; only deletes follow the note
POST = "app.bsky.feed.post"
def __init__(self, service: str, identifier: str, password: str,
http: Optional[httpx.Client] = None):
self.service = service.rstrip("/")
self.identifier = identifier
self.password = password
self.http = http or httpx.Client(timeout=30.0)
self.pds = self.service
self.did: Optional[str] = None
self.handle = identifier
self._access: Optional[str] = None
self._refresh: Optional[str] = None
self._lock = threading.Lock()
# --- XRPC plumbing ---
def _raw(self, method: str, nsid: str, base: str, token: Optional[str], **kwargs) -> httpx.Response:
headers = kwargs.pop("headers", {})
if token:
headers["Authorization"] = f"Bearer {token}"
try:
return self.http.request(method, f"{base}/xrpc/{nsid}", headers=headers, **kwargs)
except httpx.HTTPError as e:
raise SyndicationError(f"{nsid}: {e}")
@staticmethod
def _error_name(response: httpx.Response) -> str:
try:
return response.json().get("error") or ""
except ValueError:
return ""
def _raise_for(self, nsid: str, response: httpx.Response) -> None:
if response.status_code < 400:
return
permanent = 400 <= response.status_code < 500 and response.status_code != 429
raise SyndicationError(
f"{nsid}: HTTP {response.status_code} {response.text[:200]}", permanent=permanent
)
def _login(self) -> None:
response = None
if self._refresh:
response = self._raw("POST", "com.atproto.server.refreshSession", self.service, self._refresh)
if response.status_code >= 400:
response = None
if response is None:
response = self._raw("POST", "com.atproto.server.createSession", self.service, None,
json={"identifier": self.identifier, "password": self.password})
self._raise_for("com.atproto.server.createSession", response)
data = response.json()
self._access, self._refresh = data["accessJwt"], data["refreshJwt"]
self.did = data["did"]
self.handle = data.get("handle") or self.handle
for service in (data.get("didDoc") or {}).get("service", []):
if service.get("id", "").endswith("#atproto_pds") and service.get("serviceEndpoint"):
self.pds = service["serviceEndpoint"].rstrip("/")
def _call(self, method: str, nsid: str, **kwargs) -> httpx.Response:
with self._lock:
if not self._access:
self._login()
response = self._raw(method, nsid, self.pds, self._access, **kwargs)
if response.status_code == 401 or self._error_name(response) == "ExpiredToken":
self._access = None
self._login()
response = self._raw(method, nsid, self.pds, self._access, **kwargs)
self._raise_for(nsid, response)
return response
# --- Posts ---
def upload_blob(self, data: bytes, mime_type: str) -> dict:
response = self._call("POST", "com.atproto.repo.uploadBlob", content=data,
headers={"Content-Type": mime_type})
return response.json()["blob"]
def _image(self, media: dict) -> dict:
data, mime_type, width, height = _fit_image(media["path"], media["mime_type"], BLUESKY_BLOB_LIMIT)
image = {"image": self.upload_blob(data, mime_type), "alt": media["description"] or ""}
if width and height:
image["aspectRatio"] = {"width": width, "height": height}
return image
def _record(self, post: "Post") -> dict:
text = compose_text(post.note, post.permalink, post.tags, post.max_chars,
body=plain_text(post.note.content))
record = {
"$type": self.POST,
"text": text,
"createdAt": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
}
facets = bluesky_facets(text)
if facets:
record["facets"] = facets
if post.media:
record["embed"] = {"$type": "app.bsky.embed.images",
"images": [self._image(m) for m in post.media]}
else:
record["embed"] = {"$type": "app.bsky.embed.external", "external": {
"uri": post.permalink,
"title": post.note.title[:300],
"description": plain_text(post.note.content)[:300],
}}
return record
def _web_url(self, uri: str) -> str:
rkey = uri.rsplit("/", 1)[1]
return f"https://bsky.app/profile/{self.handle or self.did}/post/{rkey}"
def _find_existing(self, permalink: str) -> Optional[dict]:
"""A recent post that already links to this note (an earlier send whose reply was lost)"""
response = self._call("GET", "com.atproto.repo.listRecords",
params={"repo": self.did, "collection": self.POST, "limit": 25})
for item in response.json().get("records", []):
if permalink in (item.get("value") or {}).get("text", ""):
return {"id": item["uri"], "url": self._web_url(item["uri"])}
return None
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
if retrying:
if not self.did:
with self._lock:
self._login()
existing = self._find_existing(post.permalink)
if existing:
return existing
record = self._record(post)
response = self._call("POST", "com.atproto.repo.createRecord",
json={"repo": self.did or self.identifier, "collection": self.POST,
"record": record})
uri = response.json()["uri"]
return {"id": uri, "url": self._web_url(uri)}
def update(self, remote_id: str, post: "Post") -> dict:
raise NotImplementedError("Bluesky posts can't be edited")
def delete(self, remote_id: str) -> None:
# at://<did>/app.bsky.feed.post/<rkey>; deleting a missing record succeeds
repo, collection, rkey = remote_id[len("at://"):].split("/", 2)
self._call("POST", "com.atproto.repo.deleteRecord",
json={"repo": repo, "collection": collection, "rkey": rkey})
def _fit_image(path: Path, mime_type: str, limit: int) -> tuple:
"""Image bytes no bigger than `limit`, re-encoded as JPEG if needed. Returns (data, mime, w, h)."""
from PIL import Image
data = Path(path).read_bytes()
with Image.open(io.BytesIO(data)) as img:
width, height = img.size
if len(data) <= limit:
return data, mime_type, width, height
img = img.convert("RGB")
for max_side, quality in ((2000, 85), (2000, 75), (1600, 70), (1200, 65), (1000, 60)):
copy = img.copy()
copy.thumbnail((max_side, max_side))
buf = io.BytesIO()
copy.save(buf, format="JPEG", quality=quality, optimize=True)
if buf.tell() <= limit:
return buf.getvalue(), "image/jpeg", copy.size[0], copy.size[1]
raise SyndicationError(f"{Path(path).name}: image too large for Bluesky", permanent=True)
@dataclass
class Target:
id: str # stored in syndications.target
uid: str # Micropub syndicate-to uid
name: str # shown in the editor and Micropub clients
client: object # MastodonClient or BlueskyClient
max_chars: int = 5000
max_media: int = 4
@dataclass
class Post:
"""What a client needs to publish a note"""
note: object
permalink: str
tags: List[str]
max_chars: int
media: List[dict] = field(default_factory=list)
_client_cache: Dict[str, object] = {}
def targets_configured(config) -> bool:
"""Whether any crossposting target has credentials (used to start the worker)"""
return bool(
(config.get("GOTOSOCIAL_URL") and config.get("GOTOSOCIAL_ACCESS_TOKEN"))
or (config.get("BLUESKY_HANDLE") and config.get("BLUESKY_APP_PASSWORD"))
)
def get_targets() -> Dict[str, Target]:
"""Configured syndication targets, by id"""
config = current_app.config
targets: Dict[str, Target] = {}
url = (config.get("GOTOSOCIAL_URL") or "").rstrip("/")
token = config.get("GOTOSOCIAL_ACCESS_TOKEN") or ""
if url and token:
client = _client_cache.get(url)
if client is None:
client = MastodonClient(url, token, config.get("GOTOSOCIAL_VISIBILITY", "public"))
_client_cache[url] = client
targets["gotosocial"] = Target(
id="gotosocial",
uid=url + "/",
name=config.get("GOTOSOCIAL_NAME", "GoToSocial"),
client=client,
max_chars=int(config.get("GOTOSOCIAL_MAX_CHARS", 5000)),
max_media=int(config.get("GOTOSOCIAL_MAX_MEDIA", 4)),
)
handle = (config.get("BLUESKY_HANDLE") or "").lstrip("@")
password = config.get("BLUESKY_APP_PASSWORD") or ""
if handle and password:
service = (config.get("BLUESKY_SERVICE") or "https://bsky.social").rstrip("/")
key = f"bluesky:{service}:{handle}"
client = _client_cache.get(key)
if client is None:
client = BlueskyClient(service, handle, password)
_client_cache[key] = client
targets["bluesky"] = Target(
id="bluesky",
uid=f"https://bsky.app/profile/{handle}",
name=config.get("BLUESKY_NAME", "Bluesky"),
client=client,
max_chars=int(config.get("BLUESKY_MAX_CHARS", 300)),
max_media=4,
)
return targets
def resolve_targets(requested: List[str]) -> List[str]:
"""Map Micropub uids or target ids to configured target ids"""
targets = get_targets()
by_uid = {t.uid.rstrip("/"): t.id for t in targets.values()}
chosen = []
for value in requested:
value = (value or "").strip()
target_id = value if value in targets else by_uid.get(value.rstrip("/"))
if target_id and target_id not in chosen:
chosen.append(target_id)
return chosen
# --- Queue -----------------------------------------------------------------
def select_targets(note_id: int, target_ids: List[str]) -> None:
"""
Set which targets a note goes to, and queue the reconcile
Targets not listed are disabled (their remote post, if any, is deleted).
Nothing is recorded for unlisted targets the note never went to.
"""
db = get_db(current_app)
for target_id in get_targets():
enabled = 1 if target_id in target_ids else 0
existing = db.execute(
"SELECT id FROM syndications WHERE note_id = ? AND target = ?",
(note_id, target_id),
).fetchone()
if existing:
db.execute(
"""
UPDATE syndications
SET enabled = ?, state = 'pending', version = version + 1, attempts = 0,
last_error = NULL, next_attempt_at = datetime('now'), updated_at = datetime('now')
WHERE id = ?
""",
(enabled, existing["id"]),
)
elif enabled:
db.execute(
"INSERT INTO syndications (note_id, target, enabled) VALUES (?, ?, 1)",
(note_id, target_id),
)
db.commit()
def note_changed(note_id: int) -> None:
"""Queue a reconcile for every target the note was sent to"""
db = get_db(current_app)
db.execute(
"""
UPDATE syndications
SET state = 'pending', version = version + 1, attempts = 0, last_error = NULL,
next_attempt_at = datetime('now'), updated_at = datetime('now')
WHERE note_id = ?
""",
(note_id,),
)
db.commit()
def retry(syndication_id: int) -> None:
"""Retry a failed or pending row now"""
db = get_db(current_app)
db.execute(
"""
UPDATE syndications
SET state = 'pending', attempts = 0, last_error = NULL, next_attempt_at = datetime('now')
WHERE id = ?
""",
(syndication_id,),
)
db.commit()
def get_note_syndications(note_id: int) -> List[dict]:
"""Rows for a note, with the target's display name"""
targets = get_targets()
rows = get_db(current_app).execute(
"SELECT * FROM syndications WHERE note_id = ? ORDER BY target", (note_id,)
).fetchall()
result = []
for row in rows:
item = dict(row)
item["name"] = targets[row["target"]].name if row["target"] in targets else row["target"]
result.append(item)
return result
# --- Composing --------------------------------------------------------------
def _hashtag(display_name: str) -> str:
word = re.sub(r"[^\w]", "", display_name)
return f"#{word}" if word else ""
def compose_text(note, permalink: str, tags: List[str], max_chars: int,
body: Optional[str] = None) -> str:
"""
Full note (Markdown unless `body` is given), then the permalink and
hashtags; shortened at a word boundary if too long. On short targets the
hashtags may use at most a third of the space.
"""
hashtags = ""
for h in (_hashtag(t) for t in tags):
if h and len(hashtags) + len(h) + 1 <= max(max_chars // 3, 1):
hashtags = f"{hashtags} {h}".strip()
tail = f"\n\n{permalink}" + (f"\n\n{hashtags}" if hashtags else "")
body = (note.content if body is None else body).strip()
if len(body) + len(tail) > max_chars:
room = max_chars - len(tail) - 1
body = body[:room].rsplit(" ", 1)[0].rstrip() + "…"
return body + tail
_BLOCKS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "pre", "blockquote", "ul", "ol", "hr", "table"]
def plain_text(markdown_text: str) -> str:
"""Markdown as readable plain text (for targets without Markdown)"""
import markdown
from bs4 import BeautifulSoup
soup = BeautifulSoup(markdown.markdown(markdown_text, extensions=["fenced_code"]), "html.parser")
for img in soup.find_all("img"):
img.decompose()
for node in soup.find_all(string=True):
if not node.strip() and "\n" in node and node.parent.name != "pre":
node.extract() # layout newlines between tags; we add our own
for br in soup.find_all("br"):
br.replace_with("\n")
for li in soup.find_all("li"):
li.insert(0, "• ")
li.append("\n")
for block in soup.find_all(_BLOCKS):
block.append("\n\n")
text = soup.get_text()
text = re.sub(r"[ \t]+\n", "\n", text)
return re.sub(r"\n{3,}", "\n\n", text).strip()
_URL_RE = re.compile(r"https?://[^\s<>\"]+")
_TAG_RE = re.compile(r"(?<![\w#])#(\w*[^\W\d]\w*)")
def bluesky_facets(text: str) -> List[dict]:
"""Link and hashtag facets (UTF-8 byte ranges) for a Bluesky post"""
def span(start: int, end: int) -> dict:
return {"byteStart": len(text[:start].encode()), "byteEnd": len(text[:end].encode())}
facets = []
for m in _URL_RE.finditer(text):
url = m.group(0).rstrip(".,;:!?)\"'…")
facets.append({"index": span(m.start(), m.start() + len(url)),
"features": [{"$type": "app.bsky.richtext.facet#link", "uri": url}]})
urls = [(f["index"]["byteStart"], f["index"]["byteEnd"]) for f in facets]
for m in _TAG_RE.finditer(text):
index = span(m.start(), m.end())
if any(a <= index["byteStart"] < b for a, b in urls):
continue # a #fragment inside a URL
facets.append({"index": index,
"features": [{"$type": "app.bsky.richtext.facet#tag", "tag": m.group(1)}]})
return facets
def _media_files(note_id: int, limit: int) -> List[dict]:
"""Image files for a note: the 'large' variant when there is one"""
from starpunk.media import get_note_media
media_dir = Path(current_app.config["DATA_PATH"]) / "media"
files = []
for media in get_note_media(note_id)[:limit]:
variant = (media.get("variants") or {}).get("large")
rel = variant["path"] if variant else media["path"]
files.append({
"path": media_dir / rel,
"mime_type": media.get("mime_type") or "image/jpeg",
"description": media.get("caption") or "",
})
return files
# --- Worker -----------------------------------------------------------------
def _claim(db, row_id: int) -> bool:
cursor = db.execute(
f"""
UPDATE syndications SET claimed_until = datetime('now', '+{CLAIM_MINUTES} minutes')
WHERE id = ? AND state = 'pending'
AND (claimed_until IS NULL OR claimed_until < datetime('now'))
""",
(row_id,),
)
db.commit()
return cursor.rowcount == 1
def _finish(db, row, remote_id: Optional[str], remote_url: Optional[str]) -> None:
# Only mark synced if nothing changed while we were sending
db.execute(
"""
UPDATE syndications
SET remote_id = ?, remote_url = ?, claimed_until = NULL, attempts = 0, last_error = NULL,
state = CASE WHEN version = ? THEN 'synced' ELSE 'pending' END,
updated_at = datetime('now')
WHERE id = ?
""",
(remote_id, remote_url, row["version"], row["id"]),
)
db.commit()
def _fail(db, row, error: SyndicationError) -> None:
attempts = row["attempts"] + 1
failed = error.permanent or attempts >= MAX_ATTEMPTS
delay = min(30 * (2 ** attempts), 6 * 3600)
db.execute(
f"""
UPDATE syndications
SET attempts = ?, last_error = ?, claimed_until = NULL,
state = CASE WHEN version = ? THEN ? ELSE 'pending' END,
next_attempt_at = datetime('now', '+{delay} seconds'), updated_at = datetime('now')
WHERE id = ?
""",
(attempts, str(error)[:500], row["version"], "failed" if failed else "pending", row["id"]),
)
db.commit()
current_app.logger.warning(
f"Syndication {row['target']} for note {row['note_id']} failed "
f"(attempt {attempts}{', giving up' if failed else ''}): {error}"
)
def _reconcile(row, target: Target) -> tuple:
"""Bring the remote post in line with the note. Returns (remote_id, remote_url)."""
from starpunk.notes import get_note
from starpunk.tags import get_note_tags
note = get_note(id=row["note_id"], load_content=True)
wanted = bool(row["enabled"]) and note is not None and note.published
if not wanted:
if row["remote_id"]:
target.client.delete(row["remote_id"])
return None, None
if row["remote_id"] and not target.client.can_edit:
return row["remote_id"], row["remote_url"] # posted already; edits don't follow
site = current_app.config["SITE_URL"].rstrip("/")
post = Post(
note=note,
permalink=f"{site}/note/{note.slug}",
tags=[t["display_name"] for t in get_note_tags(note.id)],
max_chars=target.max_chars,
media=_media_files(note.id, target.max_media),
)
if row["remote_id"]:
try:
result = target.client.update(row["remote_id"], post)
return result["id"], result["url"]
except RemoteGone:
pass # deleted on the remote side; post it again
result = target.client.create(
post, key=f"starpunk-{row['id']}-{row['version']}", retrying=row["attempts"] > 0
)
return result["id"], result["url"]
def process_due(limit: int = 5) -> int:
"""Reconcile up to `limit` pending rows. Returns how many were handled."""
targets = get_targets()
if not targets:
return 0
db = get_db(current_app)
rows = db.execute(
"""
SELECT * FROM syndications
WHERE state = 'pending' AND next_attempt_at <= datetime('now')
AND (claimed_until IS NULL OR claimed_until < datetime('now'))
ORDER BY next_attempt_at LIMIT ?
""",
(limit,),
).fetchall()
handled = 0
for row in rows:
if not _claim(db, row["id"]):
continue # another worker took it
handled += 1
target = targets.get(row["target"])
if target is None:
_fail(db, row, SyndicationError(f"target {row['target']} is not configured", permanent=True))
continue
try:
remote_id, remote_url = _reconcile(row, target)
except SyndicationError as e:
_fail(db, row, e)
continue
except Exception as e: # never let one bad note stop the worker
_fail(db, row, SyndicationError(f"unexpected error: {e}"))
continue
_finish(db, row, remote_id, remote_url)
if remote_id:
current_app.logger.info(f"Syndicated note {row['note_id']} to {row['target']}: {remote_url}")
return handled
class SyndicationWorker(threading.Thread):
"""Daemon thread that drains the syndication queue"""
def __init__(self, app, poll_interval: int = 10):
super().__init__(daemon=True, name="syndication-worker")
self.app = app
self.poll_interval = poll_interval
self._stop_event = threading.Event()
def run(self):
while not self._stop_event.is_set():
try:
with self.app.app_context():
handled = process_due()
except Exception as e:
self.app.logger.error(f"Syndication worker error: {e}", exc_info=True)
handled = 0
if handled == 0:
self._stop_event.wait(self.poll_interval)
def stop(self):
self._stop_event.set()