- All dependencies to current releases (97 known vulnerabilities -> 0), Python 3.14, uv pinned, requirements-test.txt, lxml/dateutil pinned - CSRF tokens on admin forms, logout and IndieAuth consent - IndieAuth client and crosspost link-card fetches via safe_get - FLASK_DEBUG/FLASK_ENV default off/production; image sets them - Foreign keys enforced on pooled connections; migration 018 removes orphans - Rate limits shared across processes (rate_limits, migration 019) - datetime.utcnow() and sqlite3 default adapters replaced - compose.yaml fixed; duplicate dev_auth.py and test.ini removed Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1395 lines
57 KiB
Python
1395 lines
57 KiB
Python
"""
|
|
Crossposting (POSSE) for StarPunk (v2.1.0 ADR-065, v2.2.0 ADR-066, v2.6.0 ADR-069, v2.6.3 ADR-070)
|
|
|
|
Published notes are copied to syndication targets, and later edits and
|
|
deletes follow them. Three target kinds exist: a GoToSocial (or any
|
|
Mastodon-API) account, a Bluesky account and a Threads account. Bluesky and
|
|
Threads posts can't be edited, so there only deletes follow.
|
|
|
|
Each (note, target) pair has a row in `syndications`. Changing a note marks
|
|
its rows pending; a background worker then reconciles the remote post with
|
|
the note:
|
|
|
|
note published + target enabled, no remote post -> create
|
|
note published + target enabled, remote post -> update
|
|
otherwise, remote post exists -> delete
|
|
|
|
Rows carry a version so an edit made while a send is in flight is not lost.
|
|
"""
|
|
|
|
import fcntl
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import threading
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional
|
|
|
|
import httpx
|
|
from flask import current_app
|
|
|
|
# httpx logs request URLs at INFO, and Threads takes its token in the URL
|
|
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
|
|
from starpunk.database import get_db
|
|
|
|
MAX_ATTEMPTS = 8
|
|
CLAIM_MINUTES = 5
|
|
|
|
|
|
class SyndicationError(Exception):
|
|
"""A send failed. permanent=True means retrying won't help."""
|
|
|
|
def __init__(self, message: str, permanent: bool = False):
|
|
super().__init__(message)
|
|
self.permanent = permanent
|
|
|
|
|
|
class RemoteGone(SyndicationError):
|
|
"""The remote post no longer exists"""
|
|
|
|
|
|
class MastodonClient:
|
|
"""Minimal Mastodon-API client (works with GoToSocial)"""
|
|
|
|
def __init__(self, base_url: str, token: str, visibility: str = "public",
|
|
http: Optional[httpx.Client] = None):
|
|
self.base_url = base_url.rstrip("/")
|
|
self.visibility = visibility
|
|
self.http = http or httpx.Client(timeout=30.0)
|
|
self.headers = {"Authorization": f"Bearer {token}"}
|
|
|
|
def _call(self, method: str, path: str, **kwargs) -> httpx.Response:
|
|
headers = {**self.headers, **kwargs.pop("headers", {})}
|
|
try:
|
|
response = self.http.request(method, self.base_url + path, headers=headers, **kwargs)
|
|
except httpx.HTTPError as e:
|
|
raise SyndicationError(f"{method} {path}: {e}")
|
|
if response.status_code == 404:
|
|
raise RemoteGone(f"{method} {path}: not found", permanent=True)
|
|
if response.status_code >= 400:
|
|
# 4xx other than rate limiting won't fix itself
|
|
permanent = 400 <= response.status_code < 500 and response.status_code != 429
|
|
raise SyndicationError(
|
|
f"{method} {path}: HTTP {response.status_code} {response.text[:200]}",
|
|
permanent=permanent,
|
|
)
|
|
return response
|
|
|
|
def upload_media(self, path: Path, mime_type: str, description: str) -> str:
|
|
with open(path, "rb") as f:
|
|
response = self._call(
|
|
"POST", "/api/v2/media",
|
|
files={"file": (path.name, f, mime_type)},
|
|
data={"description": description or ""},
|
|
)
|
|
return response.json()["id"]
|
|
|
|
def _status_body(self, text: str, media_ids: List[str]) -> dict:
|
|
return {
|
|
"status": text,
|
|
"media_ids": media_ids,
|
|
"visibility": self.visibility,
|
|
"content_type": "text/markdown",
|
|
}
|
|
|
|
def create_status(self, text: str, media_ids: List[str], idempotency_key: str) -> dict:
|
|
response = self._call(
|
|
"POST", "/api/v1/statuses",
|
|
json=self._status_body(text, media_ids),
|
|
headers={"Idempotency-Key": idempotency_key},
|
|
)
|
|
data = response.json()
|
|
return {"id": data["id"], "url": data.get("url") or data.get("uri")}
|
|
|
|
def update_status(self, remote_id: str, text: str, media_ids: List[str]) -> dict:
|
|
response = self._call("PUT", f"/api/v1/statuses/{remote_id}",
|
|
json=self._status_body(text, media_ids))
|
|
data = response.json()
|
|
return {"id": data["id"], "url": data.get("url") or data.get("uri")}
|
|
|
|
def delete_status(self, remote_id: str) -> None:
|
|
try:
|
|
self._call("DELETE", f"/api/v1/statuses/{remote_id}")
|
|
except RemoteGone:
|
|
pass # already gone
|
|
|
|
# Interface used by the reconcile loop
|
|
can_edit = True
|
|
NATIVE = ("favourite", "reblog") # remote_id prefixes for likes/boosts (v2.5.0)
|
|
|
|
def find_status(self, url: str) -> Optional[str]:
|
|
"""Local id of a fediverse post, fetching it if needed; None if the URL isn't one"""
|
|
try:
|
|
response = self._call("GET", "/api/v2/search", params={
|
|
"q": url, "resolve": "true", "type": "statuses", "limit": 1,
|
|
})
|
|
except RemoteGone:
|
|
return None
|
|
statuses = response.json().get("statuses") or []
|
|
return statuses[0]["id"] if statuses else None
|
|
|
|
def _prepare(self, post: "Post") -> tuple:
|
|
if post.kind == "repost":
|
|
return compose_repost_text(post, post.note.content, post.max_chars), []
|
|
# Images in the text are sent as attachments instead (v2.6.6)
|
|
text = compose_text(post.note, post.permalink, post.tags, post.max_chars,
|
|
body=strip_images(post.note.content))
|
|
media_ids = [self.upload_media(f["path"], f["mime_type"], f["description"]) for f in post.media]
|
|
return text, media_ids
|
|
|
|
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
|
|
if post.kind in ("like", "repost") and post.target_url:
|
|
# Native favourite/boost when the target is a fediverse post; a
|
|
# repost with a comment is posted as a status so the comment stays
|
|
commented = bool(post.note.content.strip())
|
|
if post.kind == "like" or not commented:
|
|
status_id = self.find_status(post.target_url)
|
|
if status_id:
|
|
action = "favourite" if post.kind == "like" else "reblog"
|
|
self._call("POST", f"/api/v1/statuses/{status_id}/{action}")
|
|
return {"id": f"{action}:{status_id}", "url": None}
|
|
if post.kind == "like":
|
|
return {"id": None, "url": None} # nothing to like here
|
|
text, media_ids = self._prepare(post)
|
|
return self.create_status(text, media_ids, idempotency_key=key)
|
|
|
|
def update(self, remote_id: str, post: "Post") -> dict:
|
|
if remote_id.split(":", 1)[0] in self.NATIVE:
|
|
return {"id": remote_id, "url": None} # a favourite/boost has nothing to edit
|
|
text, media_ids = self._prepare(post)
|
|
return self.update_status(remote_id, text, media_ids)
|
|
|
|
def delete(self, remote_id: str) -> None:
|
|
action, _, status_id = remote_id.partition(":")
|
|
if action in self.NATIVE and status_id:
|
|
try:
|
|
self._call("POST", f"/api/v1/statuses/{status_id}/un{action}")
|
|
except RemoteGone:
|
|
pass
|
|
return
|
|
self.delete_status(remote_id)
|
|
|
|
|
|
BLUESKY_BLOB_LIMIT = 1_000_000 # app.bsky.embed.images maxSize
|
|
_TID_CHARS = "234567abcdefghijklmnopqrstuvwxyz"
|
|
|
|
|
|
class BlueskyClient:
|
|
"""
|
|
Minimal AT Protocol client for posting to Bluesky with an app password
|
|
|
|
Posts are plain text with link and hashtag facets. Images go in an
|
|
images embed; a note without images gets a link card instead.
|
|
"""
|
|
|
|
can_edit = False # Bluesky has no edits; only deletes follow the note
|
|
POST = "app.bsky.feed.post"
|
|
|
|
def __init__(self, service: str, identifier: str, password: str,
|
|
http: Optional[httpx.Client] = None):
|
|
self.service = service.rstrip("/")
|
|
self.identifier = identifier
|
|
self.password = password
|
|
self.http = http or httpx.Client(timeout=30.0)
|
|
self.pds = self.service
|
|
self.did: Optional[str] = None
|
|
self.handle = identifier
|
|
self._access: Optional[str] = None
|
|
self._refresh: Optional[str] = None
|
|
self._lock = threading.Lock()
|
|
|
|
# --- XRPC plumbing ---
|
|
|
|
def _raw(self, method: str, nsid: str, base: str, token: Optional[str], **kwargs) -> httpx.Response:
|
|
headers = kwargs.pop("headers", {})
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
try:
|
|
return self.http.request(method, f"{base}/xrpc/{nsid}", headers=headers, **kwargs)
|
|
except httpx.HTTPError as e:
|
|
raise SyndicationError(f"{nsid}: {e}")
|
|
|
|
@staticmethod
|
|
def _error_name(response: httpx.Response) -> str:
|
|
try:
|
|
return response.json().get("error") or ""
|
|
except ValueError:
|
|
return ""
|
|
|
|
def _raise_for(self, nsid: str, response: httpx.Response) -> None:
|
|
if response.status_code < 400:
|
|
return
|
|
permanent = 400 <= response.status_code < 500 and response.status_code != 429
|
|
raise SyndicationError(
|
|
f"{nsid}: HTTP {response.status_code} {response.text[:200]}", permanent=permanent
|
|
)
|
|
|
|
def _login(self) -> None:
|
|
response = None
|
|
if self._refresh:
|
|
response = self._raw("POST", "com.atproto.server.refreshSession", self.service, self._refresh)
|
|
if response.status_code >= 400:
|
|
response = None
|
|
if response is None:
|
|
response = self._raw("POST", "com.atproto.server.createSession", self.service, None,
|
|
json={"identifier": self.identifier, "password": self.password})
|
|
self._raise_for("com.atproto.server.createSession", response)
|
|
data = response.json()
|
|
self._access, self._refresh = data["accessJwt"], data["refreshJwt"]
|
|
self.did = data["did"]
|
|
self.handle = data.get("handle") or self.handle
|
|
for service in (data.get("didDoc") or {}).get("service", []):
|
|
if service.get("id", "").endswith("#atproto_pds") and service.get("serviceEndpoint"):
|
|
self.pds = service["serviceEndpoint"].rstrip("/")
|
|
|
|
def _call(self, method: str, nsid: str, **kwargs) -> httpx.Response:
|
|
with self._lock:
|
|
if not self._access:
|
|
self._login()
|
|
response = self._raw(method, nsid, self.pds, self._access, **kwargs)
|
|
if response.status_code == 401 or self._error_name(response) == "ExpiredToken":
|
|
self._access = None
|
|
self._login()
|
|
response = self._raw(method, nsid, self.pds, self._access, **kwargs)
|
|
self._raise_for(nsid, response)
|
|
return response
|
|
|
|
# --- Posts ---
|
|
|
|
def upload_blob(self, data: bytes, mime_type: str) -> dict:
|
|
response = self._call("POST", "com.atproto.repo.uploadBlob", content=data,
|
|
headers={"Content-Type": mime_type})
|
|
return response.json()["blob"]
|
|
|
|
def _image(self, media: dict) -> dict:
|
|
data, mime_type, width, height = _fit_image(media["path"], media["mime_type"], BLUESKY_BLOB_LIMIT)
|
|
image = {"image": self.upload_blob(data, mime_type), "alt": media["description"] or ""}
|
|
if width and height:
|
|
image["aspectRatio"] = {"width": width, "height": height}
|
|
return image
|
|
|
|
@staticmethod
|
|
def _now() -> str:
|
|
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
|
|
|
def _link_card(self, url: str) -> dict:
|
|
"""External embed for a page, with its title, description and image when we can get them"""
|
|
card = page_card(self.http, url)
|
|
external = {"uri": url, "title": card["title"][:300], "description": card["description"][:1000]}
|
|
if card["image"]:
|
|
try:
|
|
data, mime_type, _ = _fetch_public(self.http, card["image"], CARD_IMAGE_LIMIT,
|
|
truncate=False)
|
|
data, mime_type, _, _ = _fit_image(data, mime_type.split(";")[0] or "image/jpeg",
|
|
BLUESKY_BLOB_LIMIT)
|
|
external["thumb"] = self.upload_blob(data, mime_type)
|
|
except Exception as e: # a card without a picture is fine
|
|
current_app.logger.info(f"No link card image for {url}: {e}")
|
|
return {"$type": "app.bsky.embed.external", "external": external}
|
|
|
|
def _record(self, post: "Post", text: Optional[str] = None, embed: Optional[dict] = None,
|
|
anchors: List[tuple] = ()) -> dict:
|
|
if text is None:
|
|
body, anchors = plain_text_links(post.note.content, links="anchor")
|
|
text = compose_text(post.note, post.permalink, post.tags, post.max_chars, body=body)
|
|
record = {"$type": self.POST, "text": text, "createdAt": self._now()}
|
|
facets = bluesky_facets(text, anchors)
|
|
if facets:
|
|
record["facets"] = facets
|
|
if embed is not None:
|
|
record["embed"] = embed
|
|
elif post.media:
|
|
record["embed"] = {"$type": "app.bsky.embed.images",
|
|
"images": [self._image(m) for m in post.media]}
|
|
elif note_links(post.note.content):
|
|
# The note links somewhere: the card shows that page, not the note
|
|
record["embed"] = self._link_card(note_links(post.note.content)[0])
|
|
else:
|
|
record["embed"] = {"$type": "app.bsky.embed.external", "external": {
|
|
"uri": post.permalink,
|
|
"title": post.note.title[:300],
|
|
"description": plain_text(post.note.content)[:300],
|
|
}}
|
|
return record
|
|
|
|
_POST_URL = re.compile(r"^https://bsky\.app/profile/([^/]+)/post/([^/?#]+)")
|
|
|
|
def resolve_post(self, url: str) -> Optional[dict]:
|
|
"""{'uri', 'cid'} of a Bluesky post URL; None if the URL isn't one"""
|
|
m = self._POST_URL.match(url or "")
|
|
if not m:
|
|
return None
|
|
actor, rkey = m.groups()
|
|
if not actor.startswith("did:"):
|
|
actor = self._call("GET", "com.atproto.identity.resolveHandle",
|
|
params={"handle": actor}).json()["did"]
|
|
uri = f"at://{actor}/{self.POST}/{rkey}"
|
|
posts = self._call("GET", "app.bsky.feed.getPosts", params={"uris": uri}).json().get("posts") or []
|
|
return {"uri": uri, "cid": posts[0]["cid"]} if posts else None
|
|
|
|
def _create(self, collection: str, record: dict) -> str:
|
|
response = self._call("POST", "com.atproto.repo.createRecord",
|
|
json={"repo": self.did or self.identifier, "collection": collection,
|
|
"record": record})
|
|
return response.json()["uri"]
|
|
|
|
def _react(self, post: "Post") -> dict:
|
|
"""Like, repost or quote a Bluesky post; fall back to a link post for reposts"""
|
|
comment, anchors = (plain_text_links(post.note.content, links="anchor")
|
|
if post.note.content.strip() else ("", []))
|
|
subject = self.resolve_post(post.target_url)
|
|
if subject and post.kind == "like":
|
|
uri = self._create("app.bsky.feed.like", {
|
|
"$type": "app.bsky.feed.like", "subject": subject, "createdAt": self._now()})
|
|
return {"id": uri, "url": None}
|
|
if subject and not comment:
|
|
uri = self._create("app.bsky.feed.repost", {
|
|
"$type": "app.bsky.feed.repost", "subject": subject, "createdAt": self._now()})
|
|
return {"id": uri, "url": None}
|
|
if subject: # repost with a comment -> quote post
|
|
text = comment if len(comment) <= post.max_chars else \
|
|
comment[:post.max_chars - 1].rsplit(" ", 1)[0].rstrip() + "…"
|
|
uri = self._create(self.POST, self._record(
|
|
post, text=text, embed={"$type": "app.bsky.embed.record", "record": subject},
|
|
anchors=anchors))
|
|
return {"id": uri, "url": self._web_url(uri)}
|
|
if post.kind == "like":
|
|
return {"id": None, "url": None} # not a Bluesky post: nothing to like
|
|
text = compose_repost_text(post, comment, post.max_chars)
|
|
embed = {"$type": "app.bsky.embed.external", "external": {
|
|
"uri": post.target_url, "title": (post.target_title or post.target_url)[:300],
|
|
"description": ""}}
|
|
uri = self._create(self.POST, self._record(post, text=text, embed=embed))
|
|
return {"id": uri, "url": self._web_url(uri)}
|
|
|
|
def _web_url(self, uri: str) -> str:
|
|
rkey = uri.rsplit("/", 1)[1]
|
|
return f"https://bsky.app/profile/{self.handle or self.did}/post/{rkey}"
|
|
|
|
def _find_existing(self, permalink: str) -> Optional[dict]:
|
|
"""A recent post that already links to this note (an earlier send whose reply was lost)"""
|
|
response = self._call("GET", "com.atproto.repo.listRecords",
|
|
params={"repo": self.did, "collection": self.POST, "limit": 25})
|
|
for item in response.json().get("records", []):
|
|
if permalink in (item.get("value") or {}).get("text", ""):
|
|
return {"id": item["uri"], "url": self._web_url(item["uri"])}
|
|
return None
|
|
|
|
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
|
|
if post.kind in ("like", "repost") and post.target_url:
|
|
if not self.did:
|
|
with self._lock:
|
|
self._login()
|
|
return self._react(post)
|
|
if retrying:
|
|
if not self.did:
|
|
with self._lock:
|
|
self._login()
|
|
existing = self._find_existing(post.permalink)
|
|
if existing:
|
|
return existing
|
|
record = self._record(post)
|
|
response = self._call("POST", "com.atproto.repo.createRecord",
|
|
json={"repo": self.did or self.identifier, "collection": self.POST,
|
|
"record": record})
|
|
uri = response.json()["uri"]
|
|
return {"id": uri, "url": self._web_url(uri)}
|
|
|
|
def update(self, remote_id: str, post: "Post") -> dict:
|
|
raise NotImplementedError("Bluesky posts can't be edited")
|
|
|
|
def delete(self, remote_id: str) -> None:
|
|
# at://<did>/app.bsky.feed.post/<rkey>; deleting a missing record succeeds
|
|
repo, collection, rkey = remote_id[len("at://"):].split("/", 2)
|
|
self._call("POST", "com.atproto.repo.deleteRecord",
|
|
json={"repo": repo, "collection": collection, "rkey": rkey})
|
|
|
|
|
|
def _fit_image(path, mime_type: str, limit: int) -> tuple:
|
|
"""
|
|
Image (a path, or bytes) no bigger than `limit`, re-encoded as JPEG if
|
|
needed. Returns (data, mime, w, h).
|
|
"""
|
|
from PIL import Image
|
|
|
|
data = path if isinstance(path, bytes) else Path(path).read_bytes()
|
|
name = "image" if isinstance(path, bytes) else Path(path).name
|
|
with Image.open(io.BytesIO(data)) as img:
|
|
width, height = img.size
|
|
if len(data) <= limit:
|
|
return data, mime_type, width, height
|
|
img = img.convert("RGB")
|
|
for max_side, quality in ((2000, 85), (2000, 75), (1600, 70), (1200, 65), (1000, 60)):
|
|
copy = img.copy()
|
|
copy.thumbnail((max_side, max_side))
|
|
buf = io.BytesIO()
|
|
copy.save(buf, format="JPEG", quality=quality, optimize=True)
|
|
if buf.tell() <= limit:
|
|
return buf.getvalue(), "image/jpeg", copy.size[0], copy.size[1]
|
|
raise SyndicationError(f"{name}: image too large for Bluesky", permanent=True)
|
|
|
|
|
|
# --- Link cards ---------------------------------------------------------------
|
|
|
|
CARD_PAGE_LIMIT = 512 * 1024 # the <head> is near the start; read no more
|
|
CARD_IMAGE_LIMIT = 5 * 1024 * 1024
|
|
|
|
|
|
def _fetch_public(http: httpx.Client, url: str, limit: int, truncate: bool) -> tuple:
|
|
"""
|
|
GET a public web URL (link card pages, images to re-upload).
|
|
|
|
Uses the shared SSRF-safe fetcher (v2.11.0): every redirect hop checked,
|
|
the connection pinned to the checked address, a hard 10 s limit. Reads at
|
|
most `limit` bytes: stops there if `truncate`, else fails.
|
|
Returns (data, content_type, final_url). `http` is no longer used.
|
|
"""
|
|
from urllib.parse import urlparse
|
|
from starpunk.webmention_ssrf import (
|
|
FetchTooLargeError, ResolutionError, SSRFBlockedError, safe_get, user_agent,
|
|
)
|
|
|
|
parsed = urlparse(url)
|
|
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
|
raise SyndicationError(f"not a web URL: {url}", permanent=True)
|
|
try:
|
|
result = safe_get(url, headers={"User-Agent": user_agent(current_app.config.get("SITE_URL", ""))},
|
|
max_bytes=limit, max_redirects=5, timeout=10.0, truncate=truncate)
|
|
except ResolutionError as e:
|
|
raise SyndicationError(f"{url}: {e}")
|
|
except (SSRFBlockedError, ValueError) as e:
|
|
raise SyndicationError(f"not a public address: {url} ({e})", permanent=True)
|
|
except FetchTooLargeError:
|
|
raise SyndicationError(f"{url}: too large")
|
|
except httpx.TooManyRedirects:
|
|
raise SyndicationError(f"{url}: too many redirects")
|
|
except httpx.HTTPError as e:
|
|
raise SyndicationError(f"{url}: {type(e).__name__}")
|
|
if result.status_code >= 400:
|
|
raise SyndicationError(f"{url}: HTTP {result.status_code}")
|
|
return result.content, result.headers.get("content-type", ""), result.url
|
|
|
|
|
|
def page_card(http: httpx.Client, url: str) -> dict:
|
|
"""
|
|
A page's title, description and image URL (Open Graph, Twitter or plain
|
|
HTML tags), for a link card. Missing parts come back empty; a page that
|
|
can't be fetched gives just the URL as the title.
|
|
"""
|
|
from urllib.parse import urljoin
|
|
from bs4 import BeautifulSoup
|
|
|
|
card = {"title": url, "description": "", "image": ""}
|
|
try:
|
|
data, content_type, final = _fetch_public(http, url, CARD_PAGE_LIMIT, truncate=True)
|
|
except SyndicationError as e:
|
|
current_app.logger.info(f"No link card details for {url}: {e}")
|
|
return card
|
|
if "html" not in content_type.lower():
|
|
return card
|
|
soup = BeautifulSoup(data, "html.parser")
|
|
|
|
def meta(*names):
|
|
for name in names:
|
|
tag = soup.find("meta", attrs={"property": name}) or soup.find("meta", attrs={"name": name})
|
|
if tag and (tag.get("content") or "").strip():
|
|
return " ".join(tag["content"].split())
|
|
return ""
|
|
|
|
title = meta("og:title", "twitter:title")
|
|
if not title and soup.title:
|
|
title = " ".join(soup.title.get_text(" ").split())
|
|
image = meta("og:image:secure_url", "og:image", "og:image:url", "twitter:image")
|
|
card.update(title=title or url,
|
|
description=meta("og:description", "twitter:description", "description"),
|
|
image=urljoin(final, image) if image else "")
|
|
return card
|
|
|
|
|
|
THREADS_API = "https://graph.threads.net/v1.0"
|
|
THREADS_IMAGE_TYPES = {"image/jpeg", "image/png"}
|
|
THREADS_IMAGE_LIMIT = 8_000_000
|
|
THREADS_MAX_LINKS = 5
|
|
THREADS_REFRESH_EVERY = 24 * 3600 # tokens last 60 days; Meta allows a refresh once one is a day old
|
|
# Graph API error codes that are temporary (rate limits, service hiccups)
|
|
_GRAPH_TRANSIENT = {1, 2, 4, 17, 32, 341, 613}
|
|
_GRAPH_AUTH = {10, 190, 200} # bad token or missing permission: retrying won't help
|
|
|
|
|
|
def threads_length(text: str) -> int:
|
|
"""Length as Threads counts it: emoji (and, to be safe, all non-ASCII) by UTF-8 bytes"""
|
|
return sum(1 if ord(ch) < 128 else len(ch.encode()) for ch in text)
|
|
|
|
|
|
def threads_topic(tags: List[str]) -> str:
|
|
"""The first tag usable as a Threads topic (one per post; no '.' or '&', 1-50 chars)"""
|
|
for tag in tags:
|
|
topic = re.sub(r"[.&]", "", tag).strip()[:50]
|
|
if topic:
|
|
return topic
|
|
return ""
|
|
|
|
|
|
class ThreadsClient:
|
|
"""
|
|
Minimal Threads API client (Meta Graph API)
|
|
|
|
Posting is two steps: create a container, wait for it to be ready, then
|
|
publish it. Meta fetches images from their public URL on this site.
|
|
Posts can't be edited through the API, so only deletes follow.
|
|
|
|
Long-lived tokens expire after 60 days unless refreshed. `maintain()`
|
|
refreshes the token once a day and keeps it in `token_file`, so it
|
|
survives restarts. A different token in the config replaces it.
|
|
"""
|
|
|
|
can_edit = False
|
|
|
|
def __init__(self, token: str, token_file: Optional[Path] = None, api: str = THREADS_API,
|
|
http: Optional[httpx.Client] = None, sleep=time.sleep):
|
|
self.config_token = token
|
|
self.token_file = Path(token_file) if token_file else None
|
|
self.api = api.rstrip("/")
|
|
self.http = http or httpx.Client(timeout=30.0)
|
|
self.sleep = sleep
|
|
self.user_id: Optional[str] = None
|
|
self.username: Optional[str] = None
|
|
self._lock = threading.Lock()
|
|
self._token = token
|
|
self._checked_at = 0.0
|
|
saved = self._read_file()
|
|
if saved:
|
|
self._token = saved["token"]
|
|
|
|
# --- Token ---
|
|
|
|
def _seed(self) -> str:
|
|
return hashlib.sha256(self.config_token.encode()).hexdigest()
|
|
|
|
def _read_file(self) -> Optional[dict]:
|
|
"""The saved token, if it descends from the configured one"""
|
|
if not self.token_file or not self.token_file.exists():
|
|
return None
|
|
try:
|
|
data = json.loads(self.token_file.read_text())
|
|
except (OSError, ValueError):
|
|
return None
|
|
return data if data.get("seed") == self._seed() and data.get("token") else None
|
|
|
|
def _write_file(self, token: str, refreshed_at: float) -> None:
|
|
tmp = self.token_file.with_suffix(".tmp")
|
|
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
with os.fdopen(fd, "w") as f:
|
|
json.dump({"seed": self._seed(), "token": token, "refreshed_at": refreshed_at}, f)
|
|
os.replace(tmp, self.token_file)
|
|
|
|
def maintain(self) -> None:
|
|
"""Refresh the token if a day has passed since the last try (cheap to call often)"""
|
|
now = time.time()
|
|
if now - self._checked_at < THREADS_REFRESH_EVERY:
|
|
return
|
|
self._checked_at = now
|
|
if not self.token_file:
|
|
return
|
|
# Gunicorn workers share the file: one refreshes, the others pick up its result
|
|
self.token_file.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(self.token_file.with_suffix(".lock"), "w") as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX)
|
|
saved = self._read_file()
|
|
if saved:
|
|
self._token = saved["token"]
|
|
if now - saved.get("refreshed_at", 0) < THREADS_REFRESH_EVERY:
|
|
self._checked_at = saved["refreshed_at"]
|
|
return
|
|
base = self.api.rsplit("/", 1)[0] # refresh lives outside the versioned path
|
|
try:
|
|
response = self.http.get(f"{base}/refresh_access_token", params={
|
|
"grant_type": "th_refresh_token", "access_token": self._token})
|
|
except httpx.HTTPError as e:
|
|
current_app.logger.warning(f"Threads token refresh failed: {type(e).__name__}")
|
|
self._checked_at = now - THREADS_REFRESH_EVERY + 3600 # try again in an hour
|
|
return
|
|
if response.status_code < 400:
|
|
self._token = response.json()["access_token"]
|
|
current_app.logger.info("Threads access token refreshed")
|
|
else:
|
|
# e.g. a token less than a day old: keep it, try again tomorrow
|
|
current_app.logger.warning(
|
|
f"Threads token refresh failed: HTTP {response.status_code} {response.text[:200]}")
|
|
self._write_file(self._token, now)
|
|
|
|
# --- Graph plumbing ---
|
|
|
|
def _call(self, method: str, path: str, **params) -> dict:
|
|
params["access_token"] = self._token
|
|
kwargs = {"params": params} if method in ("GET", "DELETE") else {"data": params}
|
|
try:
|
|
response = self.http.request(method, f"{self.api}/{path}", **kwargs)
|
|
except httpx.HTTPError as e:
|
|
raise SyndicationError(f"{method} {path}: {type(e).__name__}") # never echo the URL/token
|
|
try:
|
|
body = response.json()
|
|
except ValueError:
|
|
body = {}
|
|
error = body.get("error") if isinstance(body, dict) else None
|
|
if response.status_code >= 400 or error:
|
|
error = error if isinstance(error, dict) else {}
|
|
code = error.get("code")
|
|
detail = " / ".join(str(error[k]) for k in ("error_user_title", "error_user_msg")
|
|
if error.get(k))
|
|
message = (f"{method} {path}: HTTP {response.status_code} "
|
|
f"{error.get('message') or response.text[:200]}"
|
|
f" (code {code}, subcode {error.get('error_subcode')}"
|
|
f"{', ' + detail if detail else ''}, fbtrace {error.get('fbtrace_id')})")
|
|
if response.status_code == 404 or (code == 100 and error.get("error_subcode") == 33):
|
|
raise RemoteGone(message, permanent=True)
|
|
permanent = (400 <= response.status_code < 500 and response.status_code != 429
|
|
and code not in _GRAPH_TRANSIENT)
|
|
exc = SyndicationError(message, permanent=permanent)
|
|
exc.code = code
|
|
raise exc
|
|
return body
|
|
|
|
def _me(self) -> str:
|
|
with self._lock:
|
|
if not self.user_id:
|
|
me = self._call("GET", "me", fields="id,username")
|
|
self.user_id, self.username = me["id"], me.get("username")
|
|
return self.user_id
|
|
|
|
def _wait(self, container_id: str) -> None:
|
|
"""Wait until Meta has processed a container (images take a few seconds)"""
|
|
delay = 1
|
|
self.sleep(2) # Meta can report a container ready a moment too early
|
|
for _ in range(8):
|
|
status = self._call("GET", container_id, fields="status,error_message")
|
|
state = status.get("status")
|
|
if state in ("FINISHED", None):
|
|
return
|
|
if state in ("ERROR", "EXPIRED"):
|
|
raise SyndicationError(
|
|
f"Threads container {state}: {status.get('error_message') or 'no detail'}",
|
|
permanent=True)
|
|
self.sleep(delay)
|
|
delay = min(delay * 2, 15)
|
|
raise SyndicationError("Threads container still processing")
|
|
|
|
# --- Posts ---
|
|
|
|
def _text(self, post: "Post") -> str:
|
|
# Links written "text (url)"; the first one gets Threads' preview card.
|
|
# Meta allows 5 different links per post and the permalink is one.
|
|
body = plain_text(post.note.content, links="inline", max_urls=THREADS_MAX_LINKS - 1)
|
|
limit = post.max_chars
|
|
while True:
|
|
text = compose_text(post.note, post.permalink, [], limit, body=body)
|
|
over = threads_length(text) - post.max_chars
|
|
if over <= 0:
|
|
return text
|
|
limit -= over
|
|
|
|
def _image_container(self, uid: str, media: dict, **extra) -> str:
|
|
params = {"media_type": "IMAGE", "image_url": media["url"], **extra}
|
|
if media["description"]:
|
|
params["alt_text"] = media["description"][:1000]
|
|
return self._call("POST", f"{uid}/threads", **params)["id"]
|
|
|
|
def _repost_text(self, post: "Post") -> str:
|
|
comment = plain_text(post.note.content, links="inline", max_urls=THREADS_MAX_LINKS - 1)
|
|
limit = post.max_chars
|
|
while True:
|
|
text = compose_repost_text(post, comment, limit)
|
|
over = threads_length(text) - post.max_chars
|
|
if over <= 0:
|
|
return text
|
|
limit -= over
|
|
|
|
def _container(self, uid: str, post: "Post") -> str:
|
|
if post.kind == "repost":
|
|
# No native repost of outside content: post "🔁 title url"
|
|
return self._call("POST", f"{uid}/threads", media_type="TEXT",
|
|
text=self._repost_text(post))["id"]
|
|
common = {"text": self._text(post)}
|
|
topic = threads_topic(post.tags)
|
|
if topic:
|
|
common["topic_tag"] = topic
|
|
# Threads takes JPEG/PNG up to 8 MB; others (GIF, WebP) are left off
|
|
images = [m for m in post.media if m["mime_type"] in THREADS_IMAGE_TYPES
|
|
and Path(m["path"]).stat().st_size <= THREADS_IMAGE_LIMIT]
|
|
if not images:
|
|
# No explicit link_attachment: Meta rejects one for a page it hasn't
|
|
# seen yet ("Invalid Link Attachment"), but previews the text's
|
|
# first link by itself without failing
|
|
return self._call("POST", f"{uid}/threads", media_type="TEXT", **common)["id"]
|
|
if len(images) == 1:
|
|
return self._image_container(uid, images[0], **common)
|
|
children = [self._image_container(uid, m, is_carousel_item="true") for m in images]
|
|
for child in children:
|
|
self._wait(child)
|
|
return self._call("POST", f"{uid}/threads", media_type="CAROUSEL",
|
|
children=",".join(children), **common)["id"]
|
|
|
|
def _profile_url(self) -> str:
|
|
return f"https://www.threads.com/@{self.username}" if self.username else "https://www.threads.com/"
|
|
|
|
def _find_existing(self, uid: str, link: str) -> Optional[dict]:
|
|
"""A recent post that already has this link (an earlier send whose reply was lost)"""
|
|
listing = self._call("GET", f"{uid}/threads", fields="id,text,permalink", limit=25)
|
|
for item in listing.get("data", []):
|
|
if link in (item.get("text") or ""):
|
|
return {"id": item["id"], "url": item.get("permalink") or self._profile_url()}
|
|
return None
|
|
|
|
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
|
|
if post.kind == "like" and post.target_url:
|
|
return {"id": None, "url": None} # the API can't like things; likes stay on the site
|
|
uid = self._me()
|
|
if retrying:
|
|
marker = post.target_url if post.kind == "repost" and post.target_url else post.permalink
|
|
existing = self._find_existing(uid, marker)
|
|
if existing:
|
|
return existing
|
|
container = self._container(uid, post)
|
|
self._wait(container)
|
|
try:
|
|
media_id = self._call("POST", f"{uid}/threads_publish", creation_id=container)["id"]
|
|
except SyndicationError as e:
|
|
if getattr(e, "code", None) in _GRAPH_AUTH:
|
|
raise
|
|
# Meta's publish step fails now and then ("Fatal") and works on a
|
|
# later try; the retry checks for an existing post first
|
|
raise SyndicationError(str(e))
|
|
try:
|
|
url = self._call("GET", media_id, fields="permalink").get("permalink")
|
|
except SyndicationError:
|
|
url = None # it's published; don't fail (and post twice) over the link
|
|
return {"id": media_id, "url": url or self._profile_url()}
|
|
|
|
def update(self, remote_id: str, post: "Post") -> dict:
|
|
raise NotImplementedError("Threads posts can't be edited through the API")
|
|
|
|
def delete(self, remote_id: str) -> None:
|
|
try:
|
|
self._call("DELETE", remote_id)
|
|
except RemoteGone:
|
|
pass # already gone
|
|
|
|
|
|
@dataclass
|
|
class Target:
|
|
id: str # stored in syndications.target
|
|
uid: str # Micropub syndicate-to uid
|
|
name: str # shown in the editor and Micropub clients
|
|
client: object # MastodonClient or BlueskyClient
|
|
max_chars: int = 5000
|
|
max_media: int = 4
|
|
|
|
|
|
@dataclass
|
|
class Post:
|
|
"""What a client needs to publish a note"""
|
|
note: object
|
|
permalink: str
|
|
tags: List[str]
|
|
max_chars: int
|
|
media: List[dict] = field(default_factory=list)
|
|
kind: str = "note" # note, like or repost (v2.5.0)
|
|
target_url: Optional[str] = None # what a like/repost points at
|
|
target_title: Optional[str] = None # its fetched title, if any
|
|
|
|
|
|
def compose_repost_text(post: "Post", comment: str, max_chars: int) -> str:
|
|
"""'comment' then '🔁 title' and the URL; the comment is shortened to fit"""
|
|
title = post.target_title or "Reposted"
|
|
tail = f"🔁 {title}\n{post.target_url}"
|
|
if len(tail) > max_chars:
|
|
tail = f"🔁 {post.target_url}"
|
|
comment = (comment or "").strip()
|
|
if not comment:
|
|
return tail
|
|
room = max_chars - len(tail) - 2
|
|
if len(comment) > room:
|
|
comment = comment[:max(room - 1, 0)].rsplit(" ", 1)[0].rstrip() + "…"
|
|
return f"{comment}\n\n{tail}" if room > 1 else tail
|
|
|
|
|
|
_client_cache: Dict[str, object] = {}
|
|
|
|
|
|
def targets_configured(config) -> bool:
|
|
"""Whether any crossposting target has credentials (used to start the worker)"""
|
|
return bool(
|
|
(config.get("GOTOSOCIAL_URL") and config.get("GOTOSOCIAL_ACCESS_TOKEN"))
|
|
or (config.get("BLUESKY_HANDLE") and config.get("BLUESKY_APP_PASSWORD"))
|
|
or config.get("THREADS_ACCESS_TOKEN")
|
|
)
|
|
|
|
|
|
def get_targets() -> Dict[str, Target]:
|
|
"""Configured syndication targets, by id"""
|
|
config = current_app.config
|
|
targets: Dict[str, Target] = {}
|
|
|
|
url = (config.get("GOTOSOCIAL_URL") or "").rstrip("/")
|
|
token = config.get("GOTOSOCIAL_ACCESS_TOKEN") or ""
|
|
if url and token:
|
|
client = _client_cache.get(url)
|
|
if client is None:
|
|
client = MastodonClient(url, token, config.get("GOTOSOCIAL_VISIBILITY", "public"))
|
|
_client_cache[url] = client
|
|
targets["gotosocial"] = Target(
|
|
id="gotosocial",
|
|
uid=url + "/",
|
|
name=config.get("GOTOSOCIAL_NAME", "GoToSocial"),
|
|
client=client,
|
|
max_chars=int(config.get("GOTOSOCIAL_MAX_CHARS", 5000)),
|
|
max_media=int(config.get("GOTOSOCIAL_MAX_MEDIA", 4)),
|
|
)
|
|
|
|
handle = (config.get("BLUESKY_HANDLE") or "").lstrip("@")
|
|
password = config.get("BLUESKY_APP_PASSWORD") or ""
|
|
if handle and password:
|
|
service = (config.get("BLUESKY_SERVICE") or "https://bsky.social").rstrip("/")
|
|
key = f"bluesky:{service}:{handle}"
|
|
client = _client_cache.get(key)
|
|
if client is None:
|
|
client = BlueskyClient(service, handle, password)
|
|
_client_cache[key] = client
|
|
targets["bluesky"] = Target(
|
|
id="bluesky",
|
|
uid=f"https://bsky.app/profile/{handle}",
|
|
name=config.get("BLUESKY_NAME", "Bluesky"),
|
|
client=client,
|
|
max_chars=int(config.get("BLUESKY_MAX_CHARS", 300)),
|
|
max_media=4,
|
|
)
|
|
|
|
token = config.get("THREADS_ACCESS_TOKEN") or ""
|
|
if token:
|
|
client = _client_cache.get("threads")
|
|
if client is None or client.config_token != token:
|
|
client = ThreadsClient(token, Path(config["DATA_PATH"]) / "threads_token.json",
|
|
api=config.get("THREADS_API") or THREADS_API)
|
|
_client_cache["threads"] = client
|
|
username = (config.get("THREADS_USERNAME") or "").lstrip("@")
|
|
targets["threads"] = Target(
|
|
id="threads",
|
|
uid=f"https://www.threads.com/@{username}" if username else "https://www.threads.com/",
|
|
name=config.get("THREADS_NAME", "Threads"),
|
|
client=client,
|
|
max_chars=int(config.get("THREADS_MAX_CHARS", 500)),
|
|
max_media=int(config.get("THREADS_MAX_MEDIA", 4)),
|
|
)
|
|
return targets
|
|
|
|
|
|
def resolve_targets(requested: List[str]) -> List[str]:
|
|
"""Map Micropub uids or target ids to configured target ids"""
|
|
targets = get_targets()
|
|
by_uid = {t.uid.rstrip("/"): t.id for t in targets.values()}
|
|
chosen = []
|
|
for value in requested:
|
|
value = (value or "").strip()
|
|
target_id = value if value in targets else by_uid.get(value.rstrip("/"))
|
|
if target_id and target_id not in chosen:
|
|
chosen.append(target_id)
|
|
return chosen
|
|
|
|
|
|
# --- Queue -----------------------------------------------------------------
|
|
|
|
def select_targets(note_id: int, target_ids: List[str]) -> None:
|
|
"""
|
|
Set which targets a note goes to, and queue the reconcile
|
|
|
|
Targets not listed are disabled (their remote post, if any, is deleted).
|
|
Nothing is recorded for unlisted targets the note never went to.
|
|
"""
|
|
db = get_db(current_app)
|
|
for target_id in get_targets():
|
|
enabled = 1 if target_id in target_ids else 0
|
|
existing = db.execute(
|
|
"SELECT id FROM syndications WHERE note_id = ? AND target = ?",
|
|
(note_id, target_id),
|
|
).fetchone()
|
|
if existing:
|
|
db.execute(
|
|
"""
|
|
UPDATE syndications
|
|
SET enabled = ?, state = 'pending', version = version + 1, attempts = 0,
|
|
last_error = NULL, next_attempt_at = datetime('now'), updated_at = datetime('now')
|
|
WHERE id = ?
|
|
""",
|
|
(enabled, existing["id"]),
|
|
)
|
|
elif enabled:
|
|
db.execute(
|
|
"INSERT INTO syndications (note_id, target, enabled) VALUES (?, ?, 1)",
|
|
(note_id, target_id),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def note_changed(note_id: int) -> None:
|
|
"""Queue a reconcile for every target the note was sent to"""
|
|
db = get_db(current_app)
|
|
db.execute(
|
|
"""
|
|
UPDATE syndications
|
|
SET state = 'pending', version = version + 1, attempts = 0, last_error = NULL,
|
|
next_attempt_at = datetime('now'), updated_at = datetime('now')
|
|
WHERE note_id = ?
|
|
""",
|
|
(note_id,),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def retry(syndication_id: int) -> None:
|
|
"""Retry a failed or pending row now"""
|
|
db = get_db(current_app)
|
|
db.execute(
|
|
"""
|
|
UPDATE syndications
|
|
SET state = 'pending', attempts = 0, last_error = NULL, next_attempt_at = datetime('now')
|
|
WHERE id = ?
|
|
""",
|
|
(syndication_id,),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def get_note_syndications(note_id: int) -> List[dict]:
|
|
"""Rows for a note, with the target's display name"""
|
|
targets = get_targets()
|
|
rows = get_db(current_app).execute(
|
|
"SELECT * FROM syndications WHERE note_id = ? ORDER BY target", (note_id,)
|
|
).fetchall()
|
|
result = []
|
|
for row in rows:
|
|
item = dict(row)
|
|
item["name"] = targets[row["target"]].name if row["target"] in targets else row["target"]
|
|
result.append(item)
|
|
return result
|
|
|
|
|
|
# --- Composing --------------------------------------------------------------
|
|
|
|
def _hashtag(display_name: str) -> str:
|
|
word = re.sub(r"[^\w]", "", display_name)
|
|
return f"#{word}" if word else ""
|
|
|
|
|
|
def compose_text(note, permalink: str, tags: List[str], max_chars: int,
|
|
body: Optional[str] = None) -> str:
|
|
"""
|
|
Full note (Markdown unless `body` is given), then the permalink and
|
|
hashtags; shortened at a word boundary if too long. On short targets the
|
|
hashtags may use at most a third of the space.
|
|
"""
|
|
hashtags = ""
|
|
for h in (_hashtag(t) for t in tags):
|
|
if h and len(hashtags) + len(h) + 1 <= max(max_chars // 3, 1):
|
|
hashtags = f"{hashtags} {h}".strip()
|
|
tail = f"\n\n{permalink}" + (f"\n\n{hashtags}" if hashtags else "")
|
|
body = (note.content if body is None else body).strip()
|
|
if getattr(note, "bookmark_of", None):
|
|
# A bookmark leads with what it bookmarks (v2.9.0)
|
|
body = f"🔖 {note.bookmark_of}\n\n{body}".strip()
|
|
if len(body) + len(tail) > max_chars:
|
|
room = max_chars - len(tail) - 1
|
|
body = body[:room].rsplit(" ", 1)[0].rstrip() + "…"
|
|
# A photo post may have no text: start with the link (v2.6.5)
|
|
return (body + tail).lstrip()
|
|
|
|
|
|
_BLOCKS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "pre", "blockquote", "ul", "ol", "hr", "table"]
|
|
|
|
|
|
def _note_soup(markdown_text: str):
|
|
import markdown
|
|
from bs4 import BeautifulSoup
|
|
|
|
return BeautifulSoup(markdown.markdown(markdown_text, extensions=["fenced_code"]), "html.parser")
|
|
|
|
|
|
def _clean_url(url: str) -> str:
|
|
return url.rstrip(".,;:!?)\"'…")
|
|
|
|
|
|
def _is_web(url: str) -> bool:
|
|
return url.startswith(("http://", "https://"))
|
|
|
|
|
|
def note_links(markdown_text: str) -> List[str]:
|
|
"""Web links in a note, in reading order: Markdown links, <autolinks> and bare URLs"""
|
|
from bs4 import NavigableString
|
|
|
|
links = []
|
|
for node in _note_soup(markdown_text).descendants:
|
|
if getattr(node, "name", None) == "a" and _is_web(node.get("href", "")):
|
|
links.append(node["href"])
|
|
elif isinstance(node, NavigableString) and not node.find_parent("a"):
|
|
links.extend(_clean_url(m.group(0)) for m in _URL_RE.finditer(str(node)))
|
|
return list(dict.fromkeys(links))
|
|
|
|
|
|
def plain_text(markdown_text: str, links: str = "text", max_urls: Optional[int] = None) -> str:
|
|
"""Markdown as readable plain text (for targets without Markdown). See `plain_text_links`."""
|
|
return plain_text_links(markdown_text, links, max_urls)[0]
|
|
|
|
|
|
def plain_text_links(markdown_text: str, links: str = "text",
|
|
max_urls: Optional[int] = None) -> tuple:
|
|
"""
|
|
Markdown as plain text, and the links whose text stands in for a URL.
|
|
|
|
A Markdown link `[text](url)` becomes:
|
|
links="text" just its text
|
|
links="inline" "text (url)", for targets with no clickable link text;
|
|
after `max_urls` different URLs, just the text
|
|
links="anchor" its text, listed in the returned anchors as (text, url)
|
|
so the target can make that text clickable
|
|
A link whose text is the URL itself stays a bare URL.
|
|
"""
|
|
from bs4 import NavigableString
|
|
|
|
soup = _note_soup(markdown_text)
|
|
for img in soup.find_all("img"):
|
|
img.decompose()
|
|
anchors = []
|
|
seen = set()
|
|
for node in list(soup.descendants): # bare URLs count towards max_urls first
|
|
if isinstance(node, NavigableString) and not node.find_parent("a"):
|
|
seen.update(_clean_url(m.group(0)) for m in _URL_RE.finditer(str(node)))
|
|
for a in soup.find_all("a"):
|
|
href, label = a.get("href", ""), a.get_text()
|
|
if not _is_web(href):
|
|
continue
|
|
bare = re.sub(r"^https?://", "", href).rstrip("/")
|
|
if label.strip().rstrip("/") in (href.rstrip("/"), bare):
|
|
a.replace_with(href)
|
|
seen.add(href)
|
|
elif links == "inline" and (href in seen or max_urls is None or len(seen) < max_urls):
|
|
a.replace_with(f"{label} ({href})")
|
|
seen.add(href)
|
|
elif links == "anchor":
|
|
anchors.append((label, href))
|
|
for node in soup.find_all(string=True):
|
|
if not node.strip() and "\n" in node and node.parent.name != "pre":
|
|
node.extract() # layout newlines between tags; we add our own
|
|
for br in soup.find_all("br"):
|
|
br.replace_with("\n")
|
|
for li in soup.find_all("li"):
|
|
li.insert(0, "• ")
|
|
li.append("\n")
|
|
for block in soup.find_all(_BLOCKS):
|
|
block.append("\n\n")
|
|
text = soup.get_text()
|
|
text = re.sub(r"[ \t]+\n", "\n", text)
|
|
return re.sub(r"\n{3,}", "\n\n", text).strip(), anchors
|
|
|
|
|
|
_URL_RE = re.compile(r"https?://[^\s<>\"]+")
|
|
_TAG_RE = re.compile(r"(?<![\w#])#(\w*[^\W\d]\w*)")
|
|
|
|
|
|
def bluesky_facets(text: str, anchors: List[tuple] = ()) -> List[dict]:
|
|
"""
|
|
Link and hashtag facets (UTF-8 byte ranges) for a Bluesky post.
|
|
`anchors` are (text, url) pairs: that text, where it appears, links to url.
|
|
"""
|
|
def span(start: int, end: int) -> dict:
|
|
return {"byteStart": len(text[:start].encode()), "byteEnd": len(text[:end].encode())}
|
|
|
|
facets = []
|
|
for m in _URL_RE.finditer(text):
|
|
url = _clean_url(m.group(0))
|
|
facets.append({"index": span(m.start(), m.start() + len(url)),
|
|
"features": [{"$type": "app.bsky.richtext.facet#link", "uri": url}]})
|
|
taken = [(m.start(), m.end()) for m in _URL_RE.finditer(text)]
|
|
pos = 0
|
|
for label, url in anchors:
|
|
start = text.find(label, pos) if label.strip() else -1
|
|
if start < 0:
|
|
continue # shortened away
|
|
end = start + len(label)
|
|
if any(a < end and start < b for a, b in taken):
|
|
continue
|
|
taken.append((start, end))
|
|
pos = end
|
|
facets.append({"index": span(start, end),
|
|
"features": [{"$type": "app.bsky.richtext.facet#link", "uri": url}]})
|
|
urls = [(f["index"]["byteStart"], f["index"]["byteEnd"]) for f in facets]
|
|
facets.sort(key=lambda f: f["index"]["byteStart"])
|
|
for m in _TAG_RE.finditer(text):
|
|
index = span(m.start(), m.end())
|
|
if any(a <= index["byteStart"] < b for a, b in urls):
|
|
continue # a #fragment inside a URL
|
|
facets.append({"index": index,
|
|
"features": [{"$type": "app.bsky.richtext.facet#tag", "tag": m.group(1)}]})
|
|
return facets
|
|
|
|
|
|
_MD_IMAGE_RE = re.compile(r"!\[[^\]]*\]\([^)]*\)")
|
|
_HTML_IMAGE_RE = re.compile(r"<img\b[^>]*>", re.IGNORECASE)
|
|
|
|
|
|
def inline_images(markdown_text: str) -> List[tuple]:
|
|
"""(url, alt) for each image embedded in a note's text, in reading order"""
|
|
images = []
|
|
for img in _note_soup(markdown_text).find_all("img"):
|
|
if img.get("src"):
|
|
images.append((img["src"], img.get("alt") or ""))
|
|
return images
|
|
|
|
|
|
def strip_images(markdown_text: str) -> str:
|
|
"""A note's Markdown without its embedded images"""
|
|
text = _HTML_IMAGE_RE.sub("", _MD_IMAGE_RE.sub("", markdown_text))
|
|
return re.sub(r"\n{3,}", "\n\n", text).strip()
|
|
|
|
|
|
def _own_media_path(url: str) -> Optional[str]:
|
|
"""Path in the media library for one of this site's /media/ URLs"""
|
|
site = current_app.config["SITE_URL"].rstrip("/")
|
|
if url.startswith(site):
|
|
url = url[len(site):]
|
|
return url[len("/media/"):] if url.startswith("/media/") else None
|
|
|
|
|
|
def _media_files(note_id: int, limit: int, content: str = "") -> List[dict]:
|
|
"""
|
|
Image files for a note: the 'large' variant when there is one.
|
|
|
|
Attached photos come first, then images embedded in the text that are in
|
|
this site's media library. Some Micropub clients (OutPost, for one)
|
|
upload a photo and put it in the content rather than in `photo` (v2.6.6).
|
|
"""
|
|
from starpunk.media import get_note_media
|
|
|
|
media_dir = Path(current_app.config["DATA_PATH"]) / "media"
|
|
site = current_app.config["SITE_URL"].rstrip("/")
|
|
items = [(m, m.get("caption") or "") for m in get_note_media(note_id)]
|
|
seen = {m["path"] for m, _ in items}
|
|
for url, alt in inline_images(content) if content else []:
|
|
path = _own_media_path(url)
|
|
if not path or path in seen:
|
|
continue
|
|
media = _media_by_path(path)
|
|
if media:
|
|
seen.add(path)
|
|
items.append((media, alt or media.get("caption") or ""))
|
|
|
|
files = []
|
|
for media, description in items[:limit]:
|
|
variant = (media.get("variants") or {}).get("large")
|
|
rel = variant["path"] if variant else media["path"]
|
|
files.append({
|
|
"path": media_dir / rel,
|
|
"url": f"{site}/media/{rel}", # for targets that fetch images themselves
|
|
"mime_type": media.get("mime_type") or "image/jpeg",
|
|
"description": description,
|
|
})
|
|
return files
|
|
|
|
|
|
def _media_by_path(path: str) -> Optional[dict]:
|
|
"""A media row with its variants, found by its path in the library"""
|
|
db = get_db(current_app)
|
|
row = db.execute("SELECT id, path, mime_type FROM media WHERE path = ?", (path,)).fetchone()
|
|
if not row:
|
|
return None
|
|
variants = {
|
|
v["variant_type"]: {"path": v["path"]}
|
|
for v in db.execute(
|
|
"SELECT variant_type, path FROM media_variants WHERE media_id = ?", (row["id"],)
|
|
)
|
|
}
|
|
return {"path": row["path"], "mime_type": row["mime_type"], "variants": variants, "caption": ""}
|
|
|
|
|
|
# --- Worker -----------------------------------------------------------------
|
|
|
|
def _claim(db, row_id: int) -> bool:
|
|
cursor = db.execute(
|
|
f"""
|
|
UPDATE syndications SET claimed_until = datetime('now', '+{CLAIM_MINUTES} minutes')
|
|
WHERE id = ? AND state = 'pending'
|
|
AND (claimed_until IS NULL OR claimed_until < datetime('now'))
|
|
""",
|
|
(row_id,),
|
|
)
|
|
db.commit()
|
|
return cursor.rowcount == 1
|
|
|
|
|
|
def _finish(db, row, remote_id: Optional[str], remote_url: Optional[str]) -> None:
|
|
# Only mark synced if nothing changed while we were sending
|
|
db.execute(
|
|
"""
|
|
UPDATE syndications
|
|
SET remote_id = ?, remote_url = ?, claimed_until = NULL, attempts = 0, last_error = NULL,
|
|
state = CASE WHEN version = ? THEN 'synced' ELSE 'pending' END,
|
|
updated_at = datetime('now')
|
|
WHERE id = ?
|
|
""",
|
|
(remote_id, remote_url, row["version"], row["id"]),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def _fail(db, row, error: SyndicationError) -> None:
|
|
attempts = row["attempts"] + 1
|
|
failed = error.permanent or attempts >= MAX_ATTEMPTS
|
|
delay = min(30 * (2 ** attempts), 6 * 3600)
|
|
db.execute(
|
|
f"""
|
|
UPDATE syndications
|
|
SET attempts = ?, last_error = ?, claimed_until = NULL,
|
|
state = CASE WHEN version = ? THEN ? ELSE 'pending' END,
|
|
next_attempt_at = datetime('now', '+{delay} seconds'), updated_at = datetime('now')
|
|
WHERE id = ?
|
|
""",
|
|
(attempts, str(error)[:500], row["version"], "failed" if failed else "pending", row["id"]),
|
|
)
|
|
db.commit()
|
|
current_app.logger.warning(
|
|
f"Syndication {row['target']} for note {row['note_id']} failed "
|
|
f"(attempt {attempts}{', giving up' if failed else ''}): {error}"
|
|
)
|
|
|
|
|
|
def _target_title(url: Optional[str]) -> Optional[str]:
|
|
"""Fetched title of a liked/reposted post, if we have one"""
|
|
if not url:
|
|
return None
|
|
try:
|
|
from starpunk.webmention_queries import get_reply_context
|
|
context = get_reply_context(url) or {}
|
|
except Exception:
|
|
return None
|
|
title = context.get("title") or ""
|
|
if not title and context.get("author_name"):
|
|
title = f"Post by {context['author_name']}"
|
|
return " ".join(title.split())[:200] or None
|
|
|
|
|
|
def _reconcile(row, target: Target) -> tuple:
|
|
"""Bring the remote post in line with the note. Returns (remote_id, remote_url)."""
|
|
from starpunk.notes import get_note
|
|
from starpunk.tags import get_note_tags
|
|
|
|
note = get_note(id=row["note_id"], load_content=True)
|
|
wanted = bool(row["enabled"]) and note is not None and note.published
|
|
|
|
if not wanted:
|
|
if row["remote_id"]:
|
|
target.client.delete(row["remote_id"])
|
|
return None, None
|
|
|
|
if row["remote_id"] and not target.client.can_edit:
|
|
return row["remote_id"], row["remote_url"] # posted already; edits don't follow
|
|
|
|
site = current_app.config["SITE_URL"].rstrip("/")
|
|
post = Post(
|
|
note=note,
|
|
permalink=f"{site}/note/{note.slug}",
|
|
tags=[t["display_name"] for t in get_note_tags(note.id, entered_order=True)],
|
|
max_chars=target.max_chars,
|
|
media=_media_files(note.id, target.max_media, note.content)
|
|
if note.kind == "note" else [],
|
|
kind=note.kind,
|
|
target_url=note.reacts_to,
|
|
target_title=_target_title(note.reacts_to),
|
|
)
|
|
|
|
if row["remote_id"]:
|
|
try:
|
|
result = target.client.update(row["remote_id"], post)
|
|
return result["id"], result["url"]
|
|
except RemoteGone:
|
|
pass # deleted on the remote side; post it again
|
|
|
|
result = target.client.create(
|
|
post, key=f"starpunk-{row['id']}-{row['version']}", retrying=row["attempts"] > 0
|
|
)
|
|
return result["id"], result["url"]
|
|
|
|
|
|
def process_due(limit: int = 5) -> int:
|
|
"""Reconcile up to `limit` pending rows. Returns how many were handled."""
|
|
targets = get_targets()
|
|
if not targets:
|
|
return 0
|
|
for target in targets.values():
|
|
maintain = getattr(target.client, "maintain", None)
|
|
if maintain:
|
|
try:
|
|
maintain()
|
|
except Exception as e:
|
|
current_app.logger.error(f"Syndication {target.id} upkeep failed: {e}")
|
|
db = get_db(current_app)
|
|
rows = db.execute(
|
|
"""
|
|
SELECT * FROM syndications
|
|
WHERE state = 'pending' AND next_attempt_at <= datetime('now')
|
|
AND (claimed_until IS NULL OR claimed_until < datetime('now'))
|
|
ORDER BY next_attempt_at LIMIT ?
|
|
""",
|
|
(limit,),
|
|
).fetchall()
|
|
|
|
handled = 0
|
|
for row in rows:
|
|
if not _claim(db, row["id"]):
|
|
continue # another worker took it
|
|
handled += 1
|
|
target = targets.get(row["target"])
|
|
if target is None:
|
|
_fail(db, row, SyndicationError(f"target {row['target']} is not configured", permanent=True))
|
|
continue
|
|
try:
|
|
remote_id, remote_url = _reconcile(row, target)
|
|
except SyndicationError as e:
|
|
_fail(db, row, e)
|
|
continue
|
|
except Exception as e: # never let one bad note stop the worker
|
|
_fail(db, row, SyndicationError(f"unexpected error: {e}"))
|
|
continue
|
|
_finish(db, row, remote_id, remote_url)
|
|
if remote_id:
|
|
current_app.logger.info(f"Syndicated note {row['note_id']} to {row['target']}: {remote_url}")
|
|
return handled
|
|
|
|
|
|
class SyndicationWorker(threading.Thread):
|
|
"""Daemon thread that drains the syndication queue"""
|
|
|
|
def __init__(self, app, poll_interval: int = 10):
|
|
super().__init__(daemon=True, name="syndication-worker")
|
|
self.app = app
|
|
self.poll_interval = poll_interval
|
|
self._stop_event = threading.Event()
|
|
|
|
def run(self):
|
|
while not self._stop_event.is_set():
|
|
try:
|
|
with self.app.app_context():
|
|
handled = process_due()
|
|
except Exception as e:
|
|
self.app.logger.error(f"Syndication worker error: {e}", exc_info=True)
|
|
handled = 0
|
|
if handled == 0:
|
|
self._stop_event.wait(self.poll_interval)
|
|
|
|
def stop(self):
|
|
self._stop_event.set()
|