Some Micropub clients (OutPost) upload a photo to the media endpoint and then embed it in content instead of sending it as `photo`. Those posts have no attached media, so crossposts went out without the image. Images in the text that are in this site's media library are now sent like attached photos, with their alt text, and GoToSocial's text no longer carries the image Markdown. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1394 lines
57 KiB
Python
1394 lines
57 KiB
Python
"""
|
|
Crossposting (POSSE) for StarPunk (v2.1.0 ADR-065, v2.2.0 ADR-066, v2.6.0 ADR-069, v2.6.3 ADR-070)
|
|
|
|
Published notes are copied to syndication targets, and later edits and
|
|
deletes follow them. Three target kinds exist: a GoToSocial (or any
|
|
Mastodon-API) account, a Bluesky account and a Threads account. Bluesky and
|
|
Threads posts can't be edited, so there only deletes follow.
|
|
|
|
Each (note, target) pair has a row in `syndications`. Changing a note marks
|
|
its rows pending; a background worker then reconciles the remote post with
|
|
the note:
|
|
|
|
note published + target enabled, no remote post -> create
|
|
note published + target enabled, remote post -> update
|
|
otherwise, remote post exists -> delete
|
|
|
|
Rows carry a version so an edit made while a send is in flight is not lost.
|
|
"""
|
|
|
|
import fcntl
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import threading
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional
|
|
|
|
import httpx
|
|
from flask import current_app
|
|
|
|
# httpx logs request URLs at INFO, and Threads takes its token in the URL
|
|
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
|
|
from starpunk.database import get_db
|
|
|
|
MAX_ATTEMPTS = 8
|
|
CLAIM_MINUTES = 5
|
|
|
|
|
|
class SyndicationError(Exception):
|
|
"""A send failed. permanent=True means retrying won't help."""
|
|
|
|
def __init__(self, message: str, permanent: bool = False):
|
|
super().__init__(message)
|
|
self.permanent = permanent
|
|
|
|
|
|
class RemoteGone(SyndicationError):
|
|
"""The remote post no longer exists"""
|
|
|
|
|
|
class MastodonClient:
|
|
"""Minimal Mastodon-API client (works with GoToSocial)"""
|
|
|
|
def __init__(self, base_url: str, token: str, visibility: str = "public",
|
|
http: Optional[httpx.Client] = None):
|
|
self.base_url = base_url.rstrip("/")
|
|
self.visibility = visibility
|
|
self.http = http or httpx.Client(timeout=30.0)
|
|
self.headers = {"Authorization": f"Bearer {token}"}
|
|
|
|
def _call(self, method: str, path: str, **kwargs) -> httpx.Response:
|
|
headers = {**self.headers, **kwargs.pop("headers", {})}
|
|
try:
|
|
response = self.http.request(method, self.base_url + path, headers=headers, **kwargs)
|
|
except httpx.HTTPError as e:
|
|
raise SyndicationError(f"{method} {path}: {e}")
|
|
if response.status_code == 404:
|
|
raise RemoteGone(f"{method} {path}: not found", permanent=True)
|
|
if response.status_code >= 400:
|
|
# 4xx other than rate limiting won't fix itself
|
|
permanent = 400 <= response.status_code < 500 and response.status_code != 429
|
|
raise SyndicationError(
|
|
f"{method} {path}: HTTP {response.status_code} {response.text[:200]}",
|
|
permanent=permanent,
|
|
)
|
|
return response
|
|
|
|
def upload_media(self, path: Path, mime_type: str, description: str) -> str:
|
|
with open(path, "rb") as f:
|
|
response = self._call(
|
|
"POST", "/api/v2/media",
|
|
files={"file": (path.name, f, mime_type)},
|
|
data={"description": description or ""},
|
|
)
|
|
return response.json()["id"]
|
|
|
|
def _status_body(self, text: str, media_ids: List[str]) -> dict:
|
|
return {
|
|
"status": text,
|
|
"media_ids": media_ids,
|
|
"visibility": self.visibility,
|
|
"content_type": "text/markdown",
|
|
}
|
|
|
|
def create_status(self, text: str, media_ids: List[str], idempotency_key: str) -> dict:
|
|
response = self._call(
|
|
"POST", "/api/v1/statuses",
|
|
json=self._status_body(text, media_ids),
|
|
headers={"Idempotency-Key": idempotency_key},
|
|
)
|
|
data = response.json()
|
|
return {"id": data["id"], "url": data.get("url") or data.get("uri")}
|
|
|
|
def update_status(self, remote_id: str, text: str, media_ids: List[str]) -> dict:
|
|
response = self._call("PUT", f"/api/v1/statuses/{remote_id}",
|
|
json=self._status_body(text, media_ids))
|
|
data = response.json()
|
|
return {"id": data["id"], "url": data.get("url") or data.get("uri")}
|
|
|
|
def delete_status(self, remote_id: str) -> None:
|
|
try:
|
|
self._call("DELETE", f"/api/v1/statuses/{remote_id}")
|
|
except RemoteGone:
|
|
pass # already gone
|
|
|
|
# Interface used by the reconcile loop
|
|
can_edit = True
|
|
NATIVE = ("favourite", "reblog") # remote_id prefixes for likes/boosts (v2.5.0)
|
|
|
|
def find_status(self, url: str) -> Optional[str]:
|
|
"""Local id of a fediverse post, fetching it if needed; None if the URL isn't one"""
|
|
try:
|
|
response = self._call("GET", "/api/v2/search", params={
|
|
"q": url, "resolve": "true", "type": "statuses", "limit": 1,
|
|
})
|
|
except RemoteGone:
|
|
return None
|
|
statuses = response.json().get("statuses") or []
|
|
return statuses[0]["id"] if statuses else None
|
|
|
|
def _prepare(self, post: "Post") -> tuple:
|
|
if post.kind == "repost":
|
|
return compose_repost_text(post, post.note.content, post.max_chars), []
|
|
# Images in the text are sent as attachments instead (v2.6.6)
|
|
text = compose_text(post.note, post.permalink, post.tags, post.max_chars,
|
|
body=strip_images(post.note.content))
|
|
media_ids = [self.upload_media(f["path"], f["mime_type"], f["description"]) for f in post.media]
|
|
return text, media_ids
|
|
|
|
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
|
|
if post.kind in ("like", "repost") and post.target_url:
|
|
# Native favourite/boost when the target is a fediverse post; a
|
|
# repost with a comment is posted as a status so the comment stays
|
|
commented = bool(post.note.content.strip())
|
|
if post.kind == "like" or not commented:
|
|
status_id = self.find_status(post.target_url)
|
|
if status_id:
|
|
action = "favourite" if post.kind == "like" else "reblog"
|
|
self._call("POST", f"/api/v1/statuses/{status_id}/{action}")
|
|
return {"id": f"{action}:{status_id}", "url": None}
|
|
if post.kind == "like":
|
|
return {"id": None, "url": None} # nothing to like here
|
|
text, media_ids = self._prepare(post)
|
|
return self.create_status(text, media_ids, idempotency_key=key)
|
|
|
|
def update(self, remote_id: str, post: "Post") -> dict:
|
|
if remote_id.split(":", 1)[0] in self.NATIVE:
|
|
return {"id": remote_id, "url": None} # a favourite/boost has nothing to edit
|
|
text, media_ids = self._prepare(post)
|
|
return self.update_status(remote_id, text, media_ids)
|
|
|
|
def delete(self, remote_id: str) -> None:
|
|
action, _, status_id = remote_id.partition(":")
|
|
if action in self.NATIVE and status_id:
|
|
try:
|
|
self._call("POST", f"/api/v1/statuses/{status_id}/un{action}")
|
|
except RemoteGone:
|
|
pass
|
|
return
|
|
self.delete_status(remote_id)
|
|
|
|
|
|
BLUESKY_BLOB_LIMIT = 1_000_000 # app.bsky.embed.images maxSize
|
|
_TID_CHARS = "234567abcdefghijklmnopqrstuvwxyz"
|
|
|
|
|
|
class BlueskyClient:
|
|
"""
|
|
Minimal AT Protocol client for posting to Bluesky with an app password
|
|
|
|
Posts are plain text with link and hashtag facets. Images go in an
|
|
images embed; a note without images gets a link card instead.
|
|
"""
|
|
|
|
can_edit = False # Bluesky has no edits; only deletes follow the note
|
|
POST = "app.bsky.feed.post"
|
|
|
|
def __init__(self, service: str, identifier: str, password: str,
|
|
http: Optional[httpx.Client] = None):
|
|
self.service = service.rstrip("/")
|
|
self.identifier = identifier
|
|
self.password = password
|
|
self.http = http or httpx.Client(timeout=30.0)
|
|
self.pds = self.service
|
|
self.did: Optional[str] = None
|
|
self.handle = identifier
|
|
self._access: Optional[str] = None
|
|
self._refresh: Optional[str] = None
|
|
self._lock = threading.Lock()
|
|
|
|
# --- XRPC plumbing ---
|
|
|
|
def _raw(self, method: str, nsid: str, base: str, token: Optional[str], **kwargs) -> httpx.Response:
|
|
headers = kwargs.pop("headers", {})
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
try:
|
|
return self.http.request(method, f"{base}/xrpc/{nsid}", headers=headers, **kwargs)
|
|
except httpx.HTTPError as e:
|
|
raise SyndicationError(f"{nsid}: {e}")
|
|
|
|
@staticmethod
|
|
def _error_name(response: httpx.Response) -> str:
|
|
try:
|
|
return response.json().get("error") or ""
|
|
except ValueError:
|
|
return ""
|
|
|
|
def _raise_for(self, nsid: str, response: httpx.Response) -> None:
|
|
if response.status_code < 400:
|
|
return
|
|
permanent = 400 <= response.status_code < 500 and response.status_code != 429
|
|
raise SyndicationError(
|
|
f"{nsid}: HTTP {response.status_code} {response.text[:200]}", permanent=permanent
|
|
)
|
|
|
|
def _login(self) -> None:
|
|
response = None
|
|
if self._refresh:
|
|
response = self._raw("POST", "com.atproto.server.refreshSession", self.service, self._refresh)
|
|
if response.status_code >= 400:
|
|
response = None
|
|
if response is None:
|
|
response = self._raw("POST", "com.atproto.server.createSession", self.service, None,
|
|
json={"identifier": self.identifier, "password": self.password})
|
|
self._raise_for("com.atproto.server.createSession", response)
|
|
data = response.json()
|
|
self._access, self._refresh = data["accessJwt"], data["refreshJwt"]
|
|
self.did = data["did"]
|
|
self.handle = data.get("handle") or self.handle
|
|
for service in (data.get("didDoc") or {}).get("service", []):
|
|
if service.get("id", "").endswith("#atproto_pds") and service.get("serviceEndpoint"):
|
|
self.pds = service["serviceEndpoint"].rstrip("/")
|
|
|
|
def _call(self, method: str, nsid: str, **kwargs) -> httpx.Response:
|
|
with self._lock:
|
|
if not self._access:
|
|
self._login()
|
|
response = self._raw(method, nsid, self.pds, self._access, **kwargs)
|
|
if response.status_code == 401 or self._error_name(response) == "ExpiredToken":
|
|
self._access = None
|
|
self._login()
|
|
response = self._raw(method, nsid, self.pds, self._access, **kwargs)
|
|
self._raise_for(nsid, response)
|
|
return response
|
|
|
|
# --- Posts ---
|
|
|
|
def upload_blob(self, data: bytes, mime_type: str) -> dict:
|
|
response = self._call("POST", "com.atproto.repo.uploadBlob", content=data,
|
|
headers={"Content-Type": mime_type})
|
|
return response.json()["blob"]
|
|
|
|
def _image(self, media: dict) -> dict:
|
|
data, mime_type, width, height = _fit_image(media["path"], media["mime_type"], BLUESKY_BLOB_LIMIT)
|
|
image = {"image": self.upload_blob(data, mime_type), "alt": media["description"] or ""}
|
|
if width and height:
|
|
image["aspectRatio"] = {"width": width, "height": height}
|
|
return image
|
|
|
|
@staticmethod
|
|
def _now() -> str:
|
|
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
|
|
|
def _link_card(self, url: str) -> dict:
|
|
"""External embed for a page, with its title, description and image when we can get them"""
|
|
card = page_card(self.http, url)
|
|
external = {"uri": url, "title": card["title"][:300], "description": card["description"][:1000]}
|
|
if card["image"]:
|
|
try:
|
|
data, mime_type, _ = _fetch_public(self.http, card["image"], CARD_IMAGE_LIMIT,
|
|
truncate=False)
|
|
data, mime_type, _, _ = _fit_image(data, mime_type.split(";")[0] or "image/jpeg",
|
|
BLUESKY_BLOB_LIMIT)
|
|
external["thumb"] = self.upload_blob(data, mime_type)
|
|
except Exception as e: # a card without a picture is fine
|
|
current_app.logger.info(f"No link card image for {url}: {e}")
|
|
return {"$type": "app.bsky.embed.external", "external": external}
|
|
|
|
def _record(self, post: "Post", text: Optional[str] = None, embed: Optional[dict] = None,
|
|
anchors: List[tuple] = ()) -> dict:
|
|
if text is None:
|
|
body, anchors = plain_text_links(post.note.content, links="anchor")
|
|
text = compose_text(post.note, post.permalink, post.tags, post.max_chars, body=body)
|
|
record = {"$type": self.POST, "text": text, "createdAt": self._now()}
|
|
facets = bluesky_facets(text, anchors)
|
|
if facets:
|
|
record["facets"] = facets
|
|
if embed is not None:
|
|
record["embed"] = embed
|
|
elif post.media:
|
|
record["embed"] = {"$type": "app.bsky.embed.images",
|
|
"images": [self._image(m) for m in post.media]}
|
|
elif note_links(post.note.content):
|
|
# The note links somewhere: the card shows that page, not the note
|
|
record["embed"] = self._link_card(note_links(post.note.content)[0])
|
|
else:
|
|
record["embed"] = {"$type": "app.bsky.embed.external", "external": {
|
|
"uri": post.permalink,
|
|
"title": post.note.title[:300],
|
|
"description": plain_text(post.note.content)[:300],
|
|
}}
|
|
return record
|
|
|
|
_POST_URL = re.compile(r"^https://bsky\.app/profile/([^/]+)/post/([^/?#]+)")
|
|
|
|
def resolve_post(self, url: str) -> Optional[dict]:
|
|
"""{'uri', 'cid'} of a Bluesky post URL; None if the URL isn't one"""
|
|
m = self._POST_URL.match(url or "")
|
|
if not m:
|
|
return None
|
|
actor, rkey = m.groups()
|
|
if not actor.startswith("did:"):
|
|
actor = self._call("GET", "com.atproto.identity.resolveHandle",
|
|
params={"handle": actor}).json()["did"]
|
|
uri = f"at://{actor}/{self.POST}/{rkey}"
|
|
posts = self._call("GET", "app.bsky.feed.getPosts", params={"uris": uri}).json().get("posts") or []
|
|
return {"uri": uri, "cid": posts[0]["cid"]} if posts else None
|
|
|
|
def _create(self, collection: str, record: dict) -> str:
|
|
response = self._call("POST", "com.atproto.repo.createRecord",
|
|
json={"repo": self.did or self.identifier, "collection": collection,
|
|
"record": record})
|
|
return response.json()["uri"]
|
|
|
|
def _react(self, post: "Post") -> dict:
|
|
"""Like, repost or quote a Bluesky post; fall back to a link post for reposts"""
|
|
comment, anchors = (plain_text_links(post.note.content, links="anchor")
|
|
if post.note.content.strip() else ("", []))
|
|
subject = self.resolve_post(post.target_url)
|
|
if subject and post.kind == "like":
|
|
uri = self._create("app.bsky.feed.like", {
|
|
"$type": "app.bsky.feed.like", "subject": subject, "createdAt": self._now()})
|
|
return {"id": uri, "url": None}
|
|
if subject and not comment:
|
|
uri = self._create("app.bsky.feed.repost", {
|
|
"$type": "app.bsky.feed.repost", "subject": subject, "createdAt": self._now()})
|
|
return {"id": uri, "url": None}
|
|
if subject: # repost with a comment -> quote post
|
|
text = comment if len(comment) <= post.max_chars else \
|
|
comment[:post.max_chars - 1].rsplit(" ", 1)[0].rstrip() + "…"
|
|
uri = self._create(self.POST, self._record(
|
|
post, text=text, embed={"$type": "app.bsky.embed.record", "record": subject},
|
|
anchors=anchors))
|
|
return {"id": uri, "url": self._web_url(uri)}
|
|
if post.kind == "like":
|
|
return {"id": None, "url": None} # not a Bluesky post: nothing to like
|
|
text = compose_repost_text(post, comment, post.max_chars)
|
|
embed = {"$type": "app.bsky.embed.external", "external": {
|
|
"uri": post.target_url, "title": (post.target_title or post.target_url)[:300],
|
|
"description": ""}}
|
|
uri = self._create(self.POST, self._record(post, text=text, embed=embed))
|
|
return {"id": uri, "url": self._web_url(uri)}
|
|
|
|
def _web_url(self, uri: str) -> str:
|
|
rkey = uri.rsplit("/", 1)[1]
|
|
return f"https://bsky.app/profile/{self.handle or self.did}/post/{rkey}"
|
|
|
|
def _find_existing(self, permalink: str) -> Optional[dict]:
|
|
"""A recent post that already links to this note (an earlier send whose reply was lost)"""
|
|
response = self._call("GET", "com.atproto.repo.listRecords",
|
|
params={"repo": self.did, "collection": self.POST, "limit": 25})
|
|
for item in response.json().get("records", []):
|
|
if permalink in (item.get("value") or {}).get("text", ""):
|
|
return {"id": item["uri"], "url": self._web_url(item["uri"])}
|
|
return None
|
|
|
|
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
|
|
if post.kind in ("like", "repost") and post.target_url:
|
|
if not self.did:
|
|
with self._lock:
|
|
self._login()
|
|
return self._react(post)
|
|
if retrying:
|
|
if not self.did:
|
|
with self._lock:
|
|
self._login()
|
|
existing = self._find_existing(post.permalink)
|
|
if existing:
|
|
return existing
|
|
record = self._record(post)
|
|
response = self._call("POST", "com.atproto.repo.createRecord",
|
|
json={"repo": self.did or self.identifier, "collection": self.POST,
|
|
"record": record})
|
|
uri = response.json()["uri"]
|
|
return {"id": uri, "url": self._web_url(uri)}
|
|
|
|
def update(self, remote_id: str, post: "Post") -> dict:
|
|
raise NotImplementedError("Bluesky posts can't be edited")
|
|
|
|
def delete(self, remote_id: str) -> None:
|
|
# at://<did>/app.bsky.feed.post/<rkey>; deleting a missing record succeeds
|
|
repo, collection, rkey = remote_id[len("at://"):].split("/", 2)
|
|
self._call("POST", "com.atproto.repo.deleteRecord",
|
|
json={"repo": repo, "collection": collection, "rkey": rkey})
|
|
|
|
|
|
def _fit_image(path, mime_type: str, limit: int) -> tuple:
|
|
"""
|
|
Image (a path, or bytes) no bigger than `limit`, re-encoded as JPEG if
|
|
needed. Returns (data, mime, w, h).
|
|
"""
|
|
from PIL import Image
|
|
|
|
data = path if isinstance(path, bytes) else Path(path).read_bytes()
|
|
name = "image" if isinstance(path, bytes) else Path(path).name
|
|
with Image.open(io.BytesIO(data)) as img:
|
|
width, height = img.size
|
|
if len(data) <= limit:
|
|
return data, mime_type, width, height
|
|
img = img.convert("RGB")
|
|
for max_side, quality in ((2000, 85), (2000, 75), (1600, 70), (1200, 65), (1000, 60)):
|
|
copy = img.copy()
|
|
copy.thumbnail((max_side, max_side))
|
|
buf = io.BytesIO()
|
|
copy.save(buf, format="JPEG", quality=quality, optimize=True)
|
|
if buf.tell() <= limit:
|
|
return buf.getvalue(), "image/jpeg", copy.size[0], copy.size[1]
|
|
raise SyndicationError(f"{name}: image too large for Bluesky", permanent=True)
|
|
|
|
|
|
# --- Link cards ---------------------------------------------------------------
|
|
|
|
CARD_PAGE_LIMIT = 512 * 1024 # the <head> is near the start; read no more
|
|
CARD_IMAGE_LIMIT = 5 * 1024 * 1024
|
|
|
|
|
|
def _fetch_public(http: httpx.Client, url: str, limit: int, truncate: bool) -> tuple:
|
|
"""
|
|
GET a public web URL, checking every redirect hop (no requests into the
|
|
LAN). Reads at most `limit` bytes: stops there if `truncate`, else fails.
|
|
Returns (data, content_type, final_url).
|
|
"""
|
|
from urllib.parse import urljoin, urlparse
|
|
from starpunk.micropub import _public_address
|
|
|
|
site = current_app.config.get("SITE_URL", "")
|
|
for _ in range(5):
|
|
parsed = urlparse(url)
|
|
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
|
raise SyndicationError(f"not a web URL: {url}", permanent=True)
|
|
if not current_app.config.get("TESTING") and not _public_address(parsed.hostname):
|
|
raise SyndicationError(f"not a public address: {url}", permanent=True)
|
|
try:
|
|
with http.stream("GET", url, follow_redirects=False, timeout=10.0,
|
|
headers={"User-Agent": f"StarPunk (+{site})"}) as response:
|
|
if response.is_redirect and "location" in response.headers:
|
|
url = urljoin(url, response.headers["location"])
|
|
continue
|
|
if response.status_code >= 400:
|
|
raise SyndicationError(f"{url}: HTTP {response.status_code}")
|
|
data = bytearray()
|
|
for chunk in response.iter_bytes():
|
|
data += chunk
|
|
if len(data) > limit:
|
|
if truncate:
|
|
break
|
|
raise SyndicationError(f"{url}: too large")
|
|
return bytes(data), response.headers.get("content-type", ""), url
|
|
except httpx.HTTPError as e:
|
|
raise SyndicationError(f"{url}: {type(e).__name__}")
|
|
raise SyndicationError(f"{url}: too many redirects")
|
|
|
|
|
|
def page_card(http: httpx.Client, url: str) -> dict:
|
|
"""
|
|
A page's title, description and image URL (Open Graph, Twitter or plain
|
|
HTML tags), for a link card. Missing parts come back empty; a page that
|
|
can't be fetched gives just the URL as the title.
|
|
"""
|
|
from urllib.parse import urljoin
|
|
from bs4 import BeautifulSoup
|
|
|
|
card = {"title": url, "description": "", "image": ""}
|
|
try:
|
|
data, content_type, final = _fetch_public(http, url, CARD_PAGE_LIMIT, truncate=True)
|
|
except SyndicationError as e:
|
|
current_app.logger.info(f"No link card details for {url}: {e}")
|
|
return card
|
|
if "html" not in content_type.lower():
|
|
return card
|
|
soup = BeautifulSoup(data, "html.parser")
|
|
|
|
def meta(*names):
|
|
for name in names:
|
|
tag = soup.find("meta", attrs={"property": name}) or soup.find("meta", attrs={"name": name})
|
|
if tag and (tag.get("content") or "").strip():
|
|
return " ".join(tag["content"].split())
|
|
return ""
|
|
|
|
title = meta("og:title", "twitter:title")
|
|
if not title and soup.title:
|
|
title = " ".join(soup.title.get_text(" ").split())
|
|
image = meta("og:image:secure_url", "og:image", "og:image:url", "twitter:image")
|
|
card.update(title=title or url,
|
|
description=meta("og:description", "twitter:description", "description"),
|
|
image=urljoin(final, image) if image else "")
|
|
return card
|
|
|
|
|
|
THREADS_API = "https://graph.threads.net/v1.0"
|
|
THREADS_IMAGE_TYPES = {"image/jpeg", "image/png"}
|
|
THREADS_IMAGE_LIMIT = 8_000_000
|
|
THREADS_MAX_LINKS = 5
|
|
THREADS_REFRESH_EVERY = 24 * 3600 # tokens last 60 days; Meta allows a refresh once one is a day old
|
|
# Graph API error codes that are temporary (rate limits, service hiccups)
|
|
_GRAPH_TRANSIENT = {1, 2, 4, 17, 32, 341, 613}
|
|
_GRAPH_AUTH = {10, 190, 200} # bad token or missing permission: retrying won't help
|
|
|
|
|
|
def threads_length(text: str) -> int:
|
|
"""Length as Threads counts it: emoji (and, to be safe, all non-ASCII) by UTF-8 bytes"""
|
|
return sum(1 if ord(ch) < 128 else len(ch.encode()) for ch in text)
|
|
|
|
|
|
def threads_topic(tags: List[str]) -> str:
|
|
"""The first tag usable as a Threads topic (one per post; no '.' or '&', 1-50 chars)"""
|
|
for tag in tags:
|
|
topic = re.sub(r"[.&]", "", tag).strip()[:50]
|
|
if topic:
|
|
return topic
|
|
return ""
|
|
|
|
|
|
class ThreadsClient:
|
|
"""
|
|
Minimal Threads API client (Meta Graph API)
|
|
|
|
Posting is two steps: create a container, wait for it to be ready, then
|
|
publish it. Meta fetches images from their public URL on this site.
|
|
Posts can't be edited through the API, so only deletes follow.
|
|
|
|
Long-lived tokens expire after 60 days unless refreshed. `maintain()`
|
|
refreshes the token once a day and keeps it in `token_file`, so it
|
|
survives restarts. A different token in the config replaces it.
|
|
"""
|
|
|
|
can_edit = False
|
|
|
|
def __init__(self, token: str, token_file: Optional[Path] = None, api: str = THREADS_API,
|
|
http: Optional[httpx.Client] = None, sleep=time.sleep):
|
|
self.config_token = token
|
|
self.token_file = Path(token_file) if token_file else None
|
|
self.api = api.rstrip("/")
|
|
self.http = http or httpx.Client(timeout=30.0)
|
|
self.sleep = sleep
|
|
self.user_id: Optional[str] = None
|
|
self.username: Optional[str] = None
|
|
self._lock = threading.Lock()
|
|
self._token = token
|
|
self._checked_at = 0.0
|
|
saved = self._read_file()
|
|
if saved:
|
|
self._token = saved["token"]
|
|
|
|
# --- Token ---
|
|
|
|
def _seed(self) -> str:
|
|
return hashlib.sha256(self.config_token.encode()).hexdigest()
|
|
|
|
def _read_file(self) -> Optional[dict]:
|
|
"""The saved token, if it descends from the configured one"""
|
|
if not self.token_file or not self.token_file.exists():
|
|
return None
|
|
try:
|
|
data = json.loads(self.token_file.read_text())
|
|
except (OSError, ValueError):
|
|
return None
|
|
return data if data.get("seed") == self._seed() and data.get("token") else None
|
|
|
|
def _write_file(self, token: str, refreshed_at: float) -> None:
|
|
tmp = self.token_file.with_suffix(".tmp")
|
|
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
with os.fdopen(fd, "w") as f:
|
|
json.dump({"seed": self._seed(), "token": token, "refreshed_at": refreshed_at}, f)
|
|
os.replace(tmp, self.token_file)
|
|
|
|
def maintain(self) -> None:
|
|
"""Refresh the token if a day has passed since the last try (cheap to call often)"""
|
|
now = time.time()
|
|
if now - self._checked_at < THREADS_REFRESH_EVERY:
|
|
return
|
|
self._checked_at = now
|
|
if not self.token_file:
|
|
return
|
|
# Gunicorn workers share the file: one refreshes, the others pick up its result
|
|
self.token_file.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(self.token_file.with_suffix(".lock"), "w") as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX)
|
|
saved = self._read_file()
|
|
if saved:
|
|
self._token = saved["token"]
|
|
if now - saved.get("refreshed_at", 0) < THREADS_REFRESH_EVERY:
|
|
self._checked_at = saved["refreshed_at"]
|
|
return
|
|
base = self.api.rsplit("/", 1)[0] # refresh lives outside the versioned path
|
|
try:
|
|
response = self.http.get(f"{base}/refresh_access_token", params={
|
|
"grant_type": "th_refresh_token", "access_token": self._token})
|
|
except httpx.HTTPError as e:
|
|
current_app.logger.warning(f"Threads token refresh failed: {type(e).__name__}")
|
|
self._checked_at = now - THREADS_REFRESH_EVERY + 3600 # try again in an hour
|
|
return
|
|
if response.status_code < 400:
|
|
self._token = response.json()["access_token"]
|
|
current_app.logger.info("Threads access token refreshed")
|
|
else:
|
|
# e.g. a token less than a day old: keep it, try again tomorrow
|
|
current_app.logger.warning(
|
|
f"Threads token refresh failed: HTTP {response.status_code} {response.text[:200]}")
|
|
self._write_file(self._token, now)
|
|
|
|
# --- Graph plumbing ---
|
|
|
|
def _call(self, method: str, path: str, **params) -> dict:
|
|
params["access_token"] = self._token
|
|
kwargs = {"params": params} if method in ("GET", "DELETE") else {"data": params}
|
|
try:
|
|
response = self.http.request(method, f"{self.api}/{path}", **kwargs)
|
|
except httpx.HTTPError as e:
|
|
raise SyndicationError(f"{method} {path}: {type(e).__name__}") # never echo the URL/token
|
|
try:
|
|
body = response.json()
|
|
except ValueError:
|
|
body = {}
|
|
error = body.get("error") if isinstance(body, dict) else None
|
|
if response.status_code >= 400 or error:
|
|
error = error if isinstance(error, dict) else {}
|
|
code = error.get("code")
|
|
detail = " / ".join(str(error[k]) for k in ("error_user_title", "error_user_msg")
|
|
if error.get(k))
|
|
message = (f"{method} {path}: HTTP {response.status_code} "
|
|
f"{error.get('message') or response.text[:200]}"
|
|
f" (code {code}, subcode {error.get('error_subcode')}"
|
|
f"{', ' + detail if detail else ''}, fbtrace {error.get('fbtrace_id')})")
|
|
if response.status_code == 404 or (code == 100 and error.get("error_subcode") == 33):
|
|
raise RemoteGone(message, permanent=True)
|
|
permanent = (400 <= response.status_code < 500 and response.status_code != 429
|
|
and code not in _GRAPH_TRANSIENT)
|
|
exc = SyndicationError(message, permanent=permanent)
|
|
exc.code = code
|
|
raise exc
|
|
return body
|
|
|
|
def _me(self) -> str:
|
|
with self._lock:
|
|
if not self.user_id:
|
|
me = self._call("GET", "me", fields="id,username")
|
|
self.user_id, self.username = me["id"], me.get("username")
|
|
return self.user_id
|
|
|
|
def _wait(self, container_id: str) -> None:
|
|
"""Wait until Meta has processed a container (images take a few seconds)"""
|
|
delay = 1
|
|
self.sleep(2) # Meta can report a container ready a moment too early
|
|
for _ in range(8):
|
|
status = self._call("GET", container_id, fields="status,error_message")
|
|
state = status.get("status")
|
|
if state in ("FINISHED", None):
|
|
return
|
|
if state in ("ERROR", "EXPIRED"):
|
|
raise SyndicationError(
|
|
f"Threads container {state}: {status.get('error_message') or 'no detail'}",
|
|
permanent=True)
|
|
self.sleep(delay)
|
|
delay = min(delay * 2, 15)
|
|
raise SyndicationError("Threads container still processing")
|
|
|
|
# --- Posts ---
|
|
|
|
def _text(self, post: "Post") -> str:
|
|
# Links written "text (url)"; the first one gets Threads' preview card.
|
|
# Meta allows 5 different links per post and the permalink is one.
|
|
body = plain_text(post.note.content, links="inline", max_urls=THREADS_MAX_LINKS - 1)
|
|
limit = post.max_chars
|
|
while True:
|
|
text = compose_text(post.note, post.permalink, [], limit, body=body)
|
|
over = threads_length(text) - post.max_chars
|
|
if over <= 0:
|
|
return text
|
|
limit -= over
|
|
|
|
def _image_container(self, uid: str, media: dict, **extra) -> str:
|
|
params = {"media_type": "IMAGE", "image_url": media["url"], **extra}
|
|
if media["description"]:
|
|
params["alt_text"] = media["description"][:1000]
|
|
return self._call("POST", f"{uid}/threads", **params)["id"]
|
|
|
|
def _repost_text(self, post: "Post") -> str:
|
|
comment = plain_text(post.note.content, links="inline", max_urls=THREADS_MAX_LINKS - 1)
|
|
limit = post.max_chars
|
|
while True:
|
|
text = compose_repost_text(post, comment, limit)
|
|
over = threads_length(text) - post.max_chars
|
|
if over <= 0:
|
|
return text
|
|
limit -= over
|
|
|
|
def _container(self, uid: str, post: "Post") -> str:
|
|
if post.kind == "repost":
|
|
# No native repost of outside content: post "🔁 title url"
|
|
return self._call("POST", f"{uid}/threads", media_type="TEXT",
|
|
text=self._repost_text(post))["id"]
|
|
common = {"text": self._text(post)}
|
|
topic = threads_topic(post.tags)
|
|
if topic:
|
|
common["topic_tag"] = topic
|
|
# Threads takes JPEG/PNG up to 8 MB; others (GIF, WebP) are left off
|
|
images = [m for m in post.media if m["mime_type"] in THREADS_IMAGE_TYPES
|
|
and Path(m["path"]).stat().st_size <= THREADS_IMAGE_LIMIT]
|
|
if not images:
|
|
# No explicit link_attachment: Meta rejects one for a page it hasn't
|
|
# seen yet ("Invalid Link Attachment"), but previews the text's
|
|
# first link by itself without failing
|
|
return self._call("POST", f"{uid}/threads", media_type="TEXT", **common)["id"]
|
|
if len(images) == 1:
|
|
return self._image_container(uid, images[0], **common)
|
|
children = [self._image_container(uid, m, is_carousel_item="true") for m in images]
|
|
for child in children:
|
|
self._wait(child)
|
|
return self._call("POST", f"{uid}/threads", media_type="CAROUSEL",
|
|
children=",".join(children), **common)["id"]
|
|
|
|
def _profile_url(self) -> str:
|
|
return f"https://www.threads.com/@{self.username}" if self.username else "https://www.threads.com/"
|
|
|
|
def _find_existing(self, uid: str, link: str) -> Optional[dict]:
|
|
"""A recent post that already has this link (an earlier send whose reply was lost)"""
|
|
listing = self._call("GET", f"{uid}/threads", fields="id,text,permalink", limit=25)
|
|
for item in listing.get("data", []):
|
|
if link in (item.get("text") or ""):
|
|
return {"id": item["id"], "url": item.get("permalink") or self._profile_url()}
|
|
return None
|
|
|
|
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
|
|
if post.kind == "like" and post.target_url:
|
|
return {"id": None, "url": None} # the API can't like things; likes stay on the site
|
|
uid = self._me()
|
|
if retrying:
|
|
marker = post.target_url if post.kind == "repost" and post.target_url else post.permalink
|
|
existing = self._find_existing(uid, marker)
|
|
if existing:
|
|
return existing
|
|
container = self._container(uid, post)
|
|
self._wait(container)
|
|
try:
|
|
media_id = self._call("POST", f"{uid}/threads_publish", creation_id=container)["id"]
|
|
except SyndicationError as e:
|
|
if getattr(e, "code", None) in _GRAPH_AUTH:
|
|
raise
|
|
# Meta's publish step fails now and then ("Fatal") and works on a
|
|
# later try; the retry checks for an existing post first
|
|
raise SyndicationError(str(e))
|
|
try:
|
|
url = self._call("GET", media_id, fields="permalink").get("permalink")
|
|
except SyndicationError:
|
|
url = None # it's published; don't fail (and post twice) over the link
|
|
return {"id": media_id, "url": url or self._profile_url()}
|
|
|
|
def update(self, remote_id: str, post: "Post") -> dict:
|
|
raise NotImplementedError("Threads posts can't be edited through the API")
|
|
|
|
def delete(self, remote_id: str) -> None:
|
|
try:
|
|
self._call("DELETE", remote_id)
|
|
except RemoteGone:
|
|
pass # already gone
|
|
|
|
|
|
@dataclass
|
|
class Target:
|
|
id: str # stored in syndications.target
|
|
uid: str # Micropub syndicate-to uid
|
|
name: str # shown in the editor and Micropub clients
|
|
client: object # MastodonClient or BlueskyClient
|
|
max_chars: int = 5000
|
|
max_media: int = 4
|
|
|
|
|
|
@dataclass
|
|
class Post:
|
|
"""What a client needs to publish a note"""
|
|
note: object
|
|
permalink: str
|
|
tags: List[str]
|
|
max_chars: int
|
|
media: List[dict] = field(default_factory=list)
|
|
kind: str = "note" # note, like or repost (v2.5.0)
|
|
target_url: Optional[str] = None # what a like/repost points at
|
|
target_title: Optional[str] = None # its fetched title, if any
|
|
|
|
|
|
def compose_repost_text(post: "Post", comment: str, max_chars: int) -> str:
|
|
"""'comment' then '🔁 title' and the URL; the comment is shortened to fit"""
|
|
title = post.target_title or "Reposted"
|
|
tail = f"🔁 {title}\n{post.target_url}"
|
|
if len(tail) > max_chars:
|
|
tail = f"🔁 {post.target_url}"
|
|
comment = (comment or "").strip()
|
|
if not comment:
|
|
return tail
|
|
room = max_chars - len(tail) - 2
|
|
if len(comment) > room:
|
|
comment = comment[:max(room - 1, 0)].rsplit(" ", 1)[0].rstrip() + "…"
|
|
return f"{comment}\n\n{tail}" if room > 1 else tail
|
|
|
|
|
|
_client_cache: Dict[str, object] = {}
|
|
|
|
|
|
def targets_configured(config) -> bool:
|
|
"""Whether any crossposting target has credentials (used to start the worker)"""
|
|
return bool(
|
|
(config.get("GOTOSOCIAL_URL") and config.get("GOTOSOCIAL_ACCESS_TOKEN"))
|
|
or (config.get("BLUESKY_HANDLE") and config.get("BLUESKY_APP_PASSWORD"))
|
|
or config.get("THREADS_ACCESS_TOKEN")
|
|
)
|
|
|
|
|
|
def get_targets() -> Dict[str, Target]:
|
|
"""Configured syndication targets, by id"""
|
|
config = current_app.config
|
|
targets: Dict[str, Target] = {}
|
|
|
|
url = (config.get("GOTOSOCIAL_URL") or "").rstrip("/")
|
|
token = config.get("GOTOSOCIAL_ACCESS_TOKEN") or ""
|
|
if url and token:
|
|
client = _client_cache.get(url)
|
|
if client is None:
|
|
client = MastodonClient(url, token, config.get("GOTOSOCIAL_VISIBILITY", "public"))
|
|
_client_cache[url] = client
|
|
targets["gotosocial"] = Target(
|
|
id="gotosocial",
|
|
uid=url + "/",
|
|
name=config.get("GOTOSOCIAL_NAME", "GoToSocial"),
|
|
client=client,
|
|
max_chars=int(config.get("GOTOSOCIAL_MAX_CHARS", 5000)),
|
|
max_media=int(config.get("GOTOSOCIAL_MAX_MEDIA", 4)),
|
|
)
|
|
|
|
handle = (config.get("BLUESKY_HANDLE") or "").lstrip("@")
|
|
password = config.get("BLUESKY_APP_PASSWORD") or ""
|
|
if handle and password:
|
|
service = (config.get("BLUESKY_SERVICE") or "https://bsky.social").rstrip("/")
|
|
key = f"bluesky:{service}:{handle}"
|
|
client = _client_cache.get(key)
|
|
if client is None:
|
|
client = BlueskyClient(service, handle, password)
|
|
_client_cache[key] = client
|
|
targets["bluesky"] = Target(
|
|
id="bluesky",
|
|
uid=f"https://bsky.app/profile/{handle}",
|
|
name=config.get("BLUESKY_NAME", "Bluesky"),
|
|
client=client,
|
|
max_chars=int(config.get("BLUESKY_MAX_CHARS", 300)),
|
|
max_media=4,
|
|
)
|
|
|
|
token = config.get("THREADS_ACCESS_TOKEN") or ""
|
|
if token:
|
|
client = _client_cache.get("threads")
|
|
if client is None or client.config_token != token:
|
|
client = ThreadsClient(token, Path(config["DATA_PATH"]) / "threads_token.json",
|
|
api=config.get("THREADS_API") or THREADS_API)
|
|
_client_cache["threads"] = client
|
|
username = (config.get("THREADS_USERNAME") or "").lstrip("@")
|
|
targets["threads"] = Target(
|
|
id="threads",
|
|
uid=f"https://www.threads.com/@{username}" if username else "https://www.threads.com/",
|
|
name=config.get("THREADS_NAME", "Threads"),
|
|
client=client,
|
|
max_chars=int(config.get("THREADS_MAX_CHARS", 500)),
|
|
max_media=int(config.get("THREADS_MAX_MEDIA", 4)),
|
|
)
|
|
return targets
|
|
|
|
|
|
def resolve_targets(requested: List[str]) -> List[str]:
|
|
"""Map Micropub uids or target ids to configured target ids"""
|
|
targets = get_targets()
|
|
by_uid = {t.uid.rstrip("/"): t.id for t in targets.values()}
|
|
chosen = []
|
|
for value in requested:
|
|
value = (value or "").strip()
|
|
target_id = value if value in targets else by_uid.get(value.rstrip("/"))
|
|
if target_id and target_id not in chosen:
|
|
chosen.append(target_id)
|
|
return chosen
|
|
|
|
|
|
# --- Queue -----------------------------------------------------------------
|
|
|
|
def select_targets(note_id: int, target_ids: List[str]) -> None:
|
|
"""
|
|
Set which targets a note goes to, and queue the reconcile
|
|
|
|
Targets not listed are disabled (their remote post, if any, is deleted).
|
|
Nothing is recorded for unlisted targets the note never went to.
|
|
"""
|
|
db = get_db(current_app)
|
|
for target_id in get_targets():
|
|
enabled = 1 if target_id in target_ids else 0
|
|
existing = db.execute(
|
|
"SELECT id FROM syndications WHERE note_id = ? AND target = ?",
|
|
(note_id, target_id),
|
|
).fetchone()
|
|
if existing:
|
|
db.execute(
|
|
"""
|
|
UPDATE syndications
|
|
SET enabled = ?, state = 'pending', version = version + 1, attempts = 0,
|
|
last_error = NULL, next_attempt_at = datetime('now'), updated_at = datetime('now')
|
|
WHERE id = ?
|
|
""",
|
|
(enabled, existing["id"]),
|
|
)
|
|
elif enabled:
|
|
db.execute(
|
|
"INSERT INTO syndications (note_id, target, enabled) VALUES (?, ?, 1)",
|
|
(note_id, target_id),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def note_changed(note_id: int) -> None:
|
|
"""Queue a reconcile for every target the note was sent to"""
|
|
db = get_db(current_app)
|
|
db.execute(
|
|
"""
|
|
UPDATE syndications
|
|
SET state = 'pending', version = version + 1, attempts = 0, last_error = NULL,
|
|
next_attempt_at = datetime('now'), updated_at = datetime('now')
|
|
WHERE note_id = ?
|
|
""",
|
|
(note_id,),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def retry(syndication_id: int) -> None:
|
|
"""Retry a failed or pending row now"""
|
|
db = get_db(current_app)
|
|
db.execute(
|
|
"""
|
|
UPDATE syndications
|
|
SET state = 'pending', attempts = 0, last_error = NULL, next_attempt_at = datetime('now')
|
|
WHERE id = ?
|
|
""",
|
|
(syndication_id,),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def get_note_syndications(note_id: int) -> List[dict]:
|
|
"""Rows for a note, with the target's display name"""
|
|
targets = get_targets()
|
|
rows = get_db(current_app).execute(
|
|
"SELECT * FROM syndications WHERE note_id = ? ORDER BY target", (note_id,)
|
|
).fetchall()
|
|
result = []
|
|
for row in rows:
|
|
item = dict(row)
|
|
item["name"] = targets[row["target"]].name if row["target"] in targets else row["target"]
|
|
result.append(item)
|
|
return result
|
|
|
|
|
|
# --- Composing --------------------------------------------------------------
|
|
|
|
def _hashtag(display_name: str) -> str:
|
|
word = re.sub(r"[^\w]", "", display_name)
|
|
return f"#{word}" if word else ""
|
|
|
|
|
|
def compose_text(note, permalink: str, tags: List[str], max_chars: int,
|
|
body: Optional[str] = None) -> str:
|
|
"""
|
|
Full note (Markdown unless `body` is given), then the permalink and
|
|
hashtags; shortened at a word boundary if too long. On short targets the
|
|
hashtags may use at most a third of the space.
|
|
"""
|
|
hashtags = ""
|
|
for h in (_hashtag(t) for t in tags):
|
|
if h and len(hashtags) + len(h) + 1 <= max(max_chars // 3, 1):
|
|
hashtags = f"{hashtags} {h}".strip()
|
|
tail = f"\n\n{permalink}" + (f"\n\n{hashtags}" if hashtags else "")
|
|
body = (note.content if body is None else body).strip()
|
|
if len(body) + len(tail) > max_chars:
|
|
room = max_chars - len(tail) - 1
|
|
body = body[:room].rsplit(" ", 1)[0].rstrip() + "…"
|
|
# A photo post may have no text: start with the link (v2.6.5)
|
|
return (body + tail).lstrip()
|
|
|
|
|
|
_BLOCKS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "pre", "blockquote", "ul", "ol", "hr", "table"]
|
|
|
|
|
|
def _note_soup(markdown_text: str):
|
|
import markdown
|
|
from bs4 import BeautifulSoup
|
|
|
|
return BeautifulSoup(markdown.markdown(markdown_text, extensions=["fenced_code"]), "html.parser")
|
|
|
|
|
|
def _clean_url(url: str) -> str:
|
|
return url.rstrip(".,;:!?)\"'…")
|
|
|
|
|
|
def _is_web(url: str) -> bool:
|
|
return url.startswith(("http://", "https://"))
|
|
|
|
|
|
def note_links(markdown_text: str) -> List[str]:
|
|
"""Web links in a note, in reading order: Markdown links, <autolinks> and bare URLs"""
|
|
from bs4 import NavigableString
|
|
|
|
links = []
|
|
for node in _note_soup(markdown_text).descendants:
|
|
if getattr(node, "name", None) == "a" and _is_web(node.get("href", "")):
|
|
links.append(node["href"])
|
|
elif isinstance(node, NavigableString) and not node.find_parent("a"):
|
|
links.extend(_clean_url(m.group(0)) for m in _URL_RE.finditer(str(node)))
|
|
return list(dict.fromkeys(links))
|
|
|
|
|
|
def plain_text(markdown_text: str, links: str = "text", max_urls: Optional[int] = None) -> str:
|
|
"""Markdown as readable plain text (for targets without Markdown). See `plain_text_links`."""
|
|
return plain_text_links(markdown_text, links, max_urls)[0]
|
|
|
|
|
|
def plain_text_links(markdown_text: str, links: str = "text",
|
|
max_urls: Optional[int] = None) -> tuple:
|
|
"""
|
|
Markdown as plain text, and the links whose text stands in for a URL.
|
|
|
|
A Markdown link `[text](url)` becomes:
|
|
links="text" just its text
|
|
links="inline" "text (url)", for targets with no clickable link text;
|
|
after `max_urls` different URLs, just the text
|
|
links="anchor" its text, listed in the returned anchors as (text, url)
|
|
so the target can make that text clickable
|
|
A link whose text is the URL itself stays a bare URL.
|
|
"""
|
|
from bs4 import NavigableString
|
|
|
|
soup = _note_soup(markdown_text)
|
|
for img in soup.find_all("img"):
|
|
img.decompose()
|
|
anchors = []
|
|
seen = set()
|
|
for node in list(soup.descendants): # bare URLs count towards max_urls first
|
|
if isinstance(node, NavigableString) and not node.find_parent("a"):
|
|
seen.update(_clean_url(m.group(0)) for m in _URL_RE.finditer(str(node)))
|
|
for a in soup.find_all("a"):
|
|
href, label = a.get("href", ""), a.get_text()
|
|
if not _is_web(href):
|
|
continue
|
|
bare = re.sub(r"^https?://", "", href).rstrip("/")
|
|
if label.strip().rstrip("/") in (href.rstrip("/"), bare):
|
|
a.replace_with(href)
|
|
seen.add(href)
|
|
elif links == "inline" and (href in seen or max_urls is None or len(seen) < max_urls):
|
|
a.replace_with(f"{label} ({href})")
|
|
seen.add(href)
|
|
elif links == "anchor":
|
|
anchors.append((label, href))
|
|
for node in soup.find_all(string=True):
|
|
if not node.strip() and "\n" in node and node.parent.name != "pre":
|
|
node.extract() # layout newlines between tags; we add our own
|
|
for br in soup.find_all("br"):
|
|
br.replace_with("\n")
|
|
for li in soup.find_all("li"):
|
|
li.insert(0, "• ")
|
|
li.append("\n")
|
|
for block in soup.find_all(_BLOCKS):
|
|
block.append("\n\n")
|
|
text = soup.get_text()
|
|
text = re.sub(r"[ \t]+\n", "\n", text)
|
|
return re.sub(r"\n{3,}", "\n\n", text).strip(), anchors
|
|
|
|
|
|
_URL_RE = re.compile(r"https?://[^\s<>\"]+")
|
|
_TAG_RE = re.compile(r"(?<![\w#])#(\w*[^\W\d]\w*)")
|
|
|
|
|
|
def bluesky_facets(text: str, anchors: List[tuple] = ()) -> List[dict]:
|
|
"""
|
|
Link and hashtag facets (UTF-8 byte ranges) for a Bluesky post.
|
|
`anchors` are (text, url) pairs: that text, where it appears, links to url.
|
|
"""
|
|
def span(start: int, end: int) -> dict:
|
|
return {"byteStart": len(text[:start].encode()), "byteEnd": len(text[:end].encode())}
|
|
|
|
facets = []
|
|
for m in _URL_RE.finditer(text):
|
|
url = _clean_url(m.group(0))
|
|
facets.append({"index": span(m.start(), m.start() + len(url)),
|
|
"features": [{"$type": "app.bsky.richtext.facet#link", "uri": url}]})
|
|
taken = [(m.start(), m.end()) for m in _URL_RE.finditer(text)]
|
|
pos = 0
|
|
for label, url in anchors:
|
|
start = text.find(label, pos) if label.strip() else -1
|
|
if start < 0:
|
|
continue # shortened away
|
|
end = start + len(label)
|
|
if any(a < end and start < b for a, b in taken):
|
|
continue
|
|
taken.append((start, end))
|
|
pos = end
|
|
facets.append({"index": span(start, end),
|
|
"features": [{"$type": "app.bsky.richtext.facet#link", "uri": url}]})
|
|
urls = [(f["index"]["byteStart"], f["index"]["byteEnd"]) for f in facets]
|
|
facets.sort(key=lambda f: f["index"]["byteStart"])
|
|
for m in _TAG_RE.finditer(text):
|
|
index = span(m.start(), m.end())
|
|
if any(a <= index["byteStart"] < b for a, b in urls):
|
|
continue # a #fragment inside a URL
|
|
facets.append({"index": index,
|
|
"features": [{"$type": "app.bsky.richtext.facet#tag", "tag": m.group(1)}]})
|
|
return facets
|
|
|
|
|
|
_MD_IMAGE_RE = re.compile(r"!\[[^\]]*\]\([^)]*\)")
|
|
_HTML_IMAGE_RE = re.compile(r"<img\b[^>]*>", re.IGNORECASE)
|
|
|
|
|
|
def inline_images(markdown_text: str) -> List[tuple]:
|
|
"""(url, alt) for each image embedded in a note's text, in reading order"""
|
|
images = []
|
|
for img in _note_soup(markdown_text).find_all("img"):
|
|
if img.get("src"):
|
|
images.append((img["src"], img.get("alt") or ""))
|
|
return images
|
|
|
|
|
|
def strip_images(markdown_text: str) -> str:
|
|
"""A note's Markdown without its embedded images"""
|
|
text = _HTML_IMAGE_RE.sub("", _MD_IMAGE_RE.sub("", markdown_text))
|
|
return re.sub(r"\n{3,}", "\n\n", text).strip()
|
|
|
|
|
|
def _own_media_path(url: str) -> Optional[str]:
|
|
"""Path in the media library for one of this site's /media/ URLs"""
|
|
site = current_app.config["SITE_URL"].rstrip("/")
|
|
if url.startswith(site):
|
|
url = url[len(site):]
|
|
return url[len("/media/"):] if url.startswith("/media/") else None
|
|
|
|
|
|
def _media_files(note_id: int, limit: int, content: str = "") -> List[dict]:
|
|
"""
|
|
Image files for a note: the 'large' variant when there is one.
|
|
|
|
Attached photos come first, then images embedded in the text that are in
|
|
this site's media library. Some Micropub clients (OutPost, for one)
|
|
upload a photo and put it in the content rather than in `photo` (v2.6.6).
|
|
"""
|
|
from starpunk.media import get_note_media
|
|
|
|
media_dir = Path(current_app.config["DATA_PATH"]) / "media"
|
|
site = current_app.config["SITE_URL"].rstrip("/")
|
|
items = [(m, m.get("caption") or "") for m in get_note_media(note_id)]
|
|
seen = {m["path"] for m, _ in items}
|
|
for url, alt in inline_images(content) if content else []:
|
|
path = _own_media_path(url)
|
|
if not path or path in seen:
|
|
continue
|
|
media = _media_by_path(path)
|
|
if media:
|
|
seen.add(path)
|
|
items.append((media, alt or media.get("caption") or ""))
|
|
|
|
files = []
|
|
for media, description in items[:limit]:
|
|
variant = (media.get("variants") or {}).get("large")
|
|
rel = variant["path"] if variant else media["path"]
|
|
files.append({
|
|
"path": media_dir / rel,
|
|
"url": f"{site}/media/{rel}", # for targets that fetch images themselves
|
|
"mime_type": media.get("mime_type") or "image/jpeg",
|
|
"description": description,
|
|
})
|
|
return files
|
|
|
|
|
|
def _media_by_path(path: str) -> Optional[dict]:
|
|
"""A media row with its variants, found by its path in the library"""
|
|
db = get_db(current_app)
|
|
row = db.execute("SELECT id, path, mime_type FROM media WHERE path = ?", (path,)).fetchone()
|
|
if not row:
|
|
return None
|
|
variants = {
|
|
v["variant_type"]: {"path": v["path"]}
|
|
for v in db.execute(
|
|
"SELECT variant_type, path FROM media_variants WHERE media_id = ?", (row["id"],)
|
|
)
|
|
}
|
|
return {"path": row["path"], "mime_type": row["mime_type"], "variants": variants, "caption": ""}
|
|
|
|
|
|
# --- Worker -----------------------------------------------------------------
|
|
|
|
def _claim(db, row_id: int) -> bool:
|
|
cursor = db.execute(
|
|
f"""
|
|
UPDATE syndications SET claimed_until = datetime('now', '+{CLAIM_MINUTES} minutes')
|
|
WHERE id = ? AND state = 'pending'
|
|
AND (claimed_until IS NULL OR claimed_until < datetime('now'))
|
|
""",
|
|
(row_id,),
|
|
)
|
|
db.commit()
|
|
return cursor.rowcount == 1
|
|
|
|
|
|
def _finish(db, row, remote_id: Optional[str], remote_url: Optional[str]) -> None:
|
|
# Only mark synced if nothing changed while we were sending
|
|
db.execute(
|
|
"""
|
|
UPDATE syndications
|
|
SET remote_id = ?, remote_url = ?, claimed_until = NULL, attempts = 0, last_error = NULL,
|
|
state = CASE WHEN version = ? THEN 'synced' ELSE 'pending' END,
|
|
updated_at = datetime('now')
|
|
WHERE id = ?
|
|
""",
|
|
(remote_id, remote_url, row["version"], row["id"]),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def _fail(db, row, error: SyndicationError) -> None:
|
|
attempts = row["attempts"] + 1
|
|
failed = error.permanent or attempts >= MAX_ATTEMPTS
|
|
delay = min(30 * (2 ** attempts), 6 * 3600)
|
|
db.execute(
|
|
f"""
|
|
UPDATE syndications
|
|
SET attempts = ?, last_error = ?, claimed_until = NULL,
|
|
state = CASE WHEN version = ? THEN ? ELSE 'pending' END,
|
|
next_attempt_at = datetime('now', '+{delay} seconds'), updated_at = datetime('now')
|
|
WHERE id = ?
|
|
""",
|
|
(attempts, str(error)[:500], row["version"], "failed" if failed else "pending", row["id"]),
|
|
)
|
|
db.commit()
|
|
current_app.logger.warning(
|
|
f"Syndication {row['target']} for note {row['note_id']} failed "
|
|
f"(attempt {attempts}{', giving up' if failed else ''}): {error}"
|
|
)
|
|
|
|
|
|
def _target_title(url: Optional[str]) -> Optional[str]:
|
|
"""Fetched title of a liked/reposted post, if we have one"""
|
|
if not url:
|
|
return None
|
|
try:
|
|
from starpunk.webmention_queries import get_reply_context
|
|
context = get_reply_context(url) or {}
|
|
except Exception:
|
|
return None
|
|
title = context.get("title") or ""
|
|
if not title and context.get("author_name"):
|
|
title = f"Post by {context['author_name']}"
|
|
return " ".join(title.split())[:200] or None
|
|
|
|
|
|
def _reconcile(row, target: Target) -> tuple:
|
|
"""Bring the remote post in line with the note. Returns (remote_id, remote_url)."""
|
|
from starpunk.notes import get_note
|
|
from starpunk.tags import get_note_tags
|
|
|
|
note = get_note(id=row["note_id"], load_content=True)
|
|
wanted = bool(row["enabled"]) and note is not None and note.published
|
|
|
|
if not wanted:
|
|
if row["remote_id"]:
|
|
target.client.delete(row["remote_id"])
|
|
return None, None
|
|
|
|
if row["remote_id"] and not target.client.can_edit:
|
|
return row["remote_id"], row["remote_url"] # posted already; edits don't follow
|
|
|
|
site = current_app.config["SITE_URL"].rstrip("/")
|
|
post = Post(
|
|
note=note,
|
|
permalink=f"{site}/note/{note.slug}",
|
|
tags=[t["display_name"] for t in get_note_tags(note.id, entered_order=True)],
|
|
max_chars=target.max_chars,
|
|
media=_media_files(note.id, target.max_media, note.content)
|
|
if note.kind == "note" else [],
|
|
kind=note.kind,
|
|
target_url=note.reacts_to,
|
|
target_title=_target_title(note.reacts_to),
|
|
)
|
|
|
|
if row["remote_id"]:
|
|
try:
|
|
result = target.client.update(row["remote_id"], post)
|
|
return result["id"], result["url"]
|
|
except RemoteGone:
|
|
pass # deleted on the remote side; post it again
|
|
|
|
result = target.client.create(
|
|
post, key=f"starpunk-{row['id']}-{row['version']}", retrying=row["attempts"] > 0
|
|
)
|
|
return result["id"], result["url"]
|
|
|
|
|
|
def process_due(limit: int = 5) -> int:
|
|
"""Reconcile up to `limit` pending rows. Returns how many were handled."""
|
|
targets = get_targets()
|
|
if not targets:
|
|
return 0
|
|
for target in targets.values():
|
|
maintain = getattr(target.client, "maintain", None)
|
|
if maintain:
|
|
try:
|
|
maintain()
|
|
except Exception as e:
|
|
current_app.logger.error(f"Syndication {target.id} upkeep failed: {e}")
|
|
db = get_db(current_app)
|
|
rows = db.execute(
|
|
"""
|
|
SELECT * FROM syndications
|
|
WHERE state = 'pending' AND next_attempt_at <= datetime('now')
|
|
AND (claimed_until IS NULL OR claimed_until < datetime('now'))
|
|
ORDER BY next_attempt_at LIMIT ?
|
|
""",
|
|
(limit,),
|
|
).fetchall()
|
|
|
|
handled = 0
|
|
for row in rows:
|
|
if not _claim(db, row["id"]):
|
|
continue # another worker took it
|
|
handled += 1
|
|
target = targets.get(row["target"])
|
|
if target is None:
|
|
_fail(db, row, SyndicationError(f"target {row['target']} is not configured", permanent=True))
|
|
continue
|
|
try:
|
|
remote_id, remote_url = _reconcile(row, target)
|
|
except SyndicationError as e:
|
|
_fail(db, row, e)
|
|
continue
|
|
except Exception as e: # never let one bad note stop the worker
|
|
_fail(db, row, SyndicationError(f"unexpected error: {e}"))
|
|
continue
|
|
_finish(db, row, remote_id, remote_url)
|
|
if remote_id:
|
|
current_app.logger.info(f"Syndicated note {row['note_id']} to {row['target']}: {remote_url}")
|
|
return handled
|
|
|
|
|
|
class SyndicationWorker(threading.Thread):
|
|
"""Daemon thread that drains the syndication queue"""
|
|
|
|
def __init__(self, app, poll_interval: int = 10):
|
|
super().__init__(daemon=True, name="syndication-worker")
|
|
self.app = app
|
|
self.poll_interval = poll_interval
|
|
self._stop_event = threading.Event()
|
|
|
|
def run(self):
|
|
while not self._stop_event.is_set():
|
|
try:
|
|
with self.app.app_context():
|
|
handled = process_due()
|
|
except Exception as e:
|
|
self.app.logger.error(f"Syndication worker error: {e}", exc_info=True)
|
|
handled = 0
|
|
if handled == 0:
|
|
self._stop_event.wait(self.poll_interval)
|
|
|
|
def stop(self):
|
|
self._stop_event.set()
|