From 89c90fba56dd0670ec7eb54f474518586164bfc4 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Thu, 17 Nov 2022 09:18:06 +0100 Subject: [PATCH] Start to merge IndieWeb and AP interactions --- ...d359ce78_add_webmention_webmention_type.py | 32 +++++ app/boxes.py | 125 ++++++++++++++++-- app/main.py | 20 +-- app/models.py | 11 ++ app/webmentions.py | 69 +++++++++- 5 files changed, 222 insertions(+), 35 deletions(-) create mode 100644 alembic/versions/2022_11_16_1942-fadfd359ce78_add_webmention_webmention_type.py diff --git a/alembic/versions/2022_11_16_1942-fadfd359ce78_add_webmention_webmention_type.py b/alembic/versions/2022_11_16_1942-fadfd359ce78_add_webmention_webmention_type.py new file mode 100644 index 0000000..a7753be --- /dev/null +++ b/alembic/versions/2022_11_16_1942-fadfd359ce78_add_webmention_webmention_type.py @@ -0,0 +1,32 @@ +"""Add Webmention.webmention_type + +Revision ID: fadfd359ce78 +Revises: b28c0551c236 +Create Date: 2022-11-16 19:42:56.925512+00:00 + +""" +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision = 'fadfd359ce78' +down_revision = 'b28c0551c236' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('webmention', schema=None) as batch_op: + batch_op.add_column(sa.Column('webmention_type', sa.Enum('UNKNOWN', 'LIKE', 'REPLY', 'REPOST', name='webmentiontype'), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('webmention', schema=None) as batch_op: + batch_op.drop_column('webmention_type') + + # ### end Alembic commands ### diff --git a/app/boxes.py b/app/boxes.py index 0f5c7a4..f258934 100644 --- a/app/boxes.py +++ b/app/boxes.py @@ -201,7 +201,7 @@ async def send_delete(db_session: AsyncSession, ap_object_id: str) -> None: raise ValueError("Should never happen") outbox_object_to_delete.is_deleted = True - await db_session.commit() + await db_session.flush() # Compute the original recipients recipients = await _compute_recipients( @@ -216,14 +216,17 @@ async def send_delete(db_session: AsyncSession, ap_object_id: str) -> None: db_session, outbox_object_to_delete.in_reply_to ) if replied_object: - new_replies_count = await _get_replies_count( - db_session, replied_object.ap_id - ) + if replied_object.is_from_outbox: + # Different helper here because we also count webmentions + new_replies_count = await _get_outbox_replies_count( + db_session, replied_object # type: ignore + ) + else: + new_replies_count = await _get_replies_count( + db_session, replied_object.ap_id + ) replied_object.replies_count = new_replies_count - if replied_object.replies_count < 0: - logger.warning("negative replies count for {replied_object.ap_id}") - replied_object.replies_count = 0 else: logger.info(f"{outbox_object_to_delete.in_reply_to} not found") @@ -1048,6 +1051,32 @@ async def get_outbox_object_by_ap_id( ) # type: ignore +async def get_outbox_object_by_slug_and_short_id( + db_session: AsyncSession, + slug: str, + short_id: str, +) -> models.OutboxObject | None: + return ( + ( + await db_session.execute( + select(models.OutboxObject) + .options( + joinedload(models.OutboxObject.outbox_object_attachments).options( + joinedload(models.OutboxObjectAttachment.upload) + ) + ) + .where( + models.OutboxObject.public_id.like(f"{short_id}%"), + models.OutboxObject.slug == slug, + models.OutboxObject.is_deleted.is_(False), + ) + ) + ) + .unique() + .scalar_one_or_none() + ) + + async def get_anybox_object_by_ap_id( db_session: AsyncSession, ap_id: str ) -> AnyboxObject | None: @@ -1201,6 +1230,67 @@ async def _get_replies_count( ) +async def _get_outbox_replies_count( + db_session: AsyncSession, + outbox_object: models.OutboxObject, +) -> int: + return (await _get_replies_count(db_session, outbox_object.ap_id)) + ( + await db_session.scalar( + select(func.count(models.Webmention.id)).where( + models.Webmention.is_deleted.is_(False), + models.Webmention.outbox_object_id == outbox_object.id, + models.Webmention.webmention_type == models.WebmentionType.REPLY, + ) + ) + ) + + +async def _get_outbox_likes_count( + db_session: AsyncSession, + outbox_object: models.OutboxObject, +) -> int: + return ( + await db_session.scalar( + select(func.count(models.InboxObject.id)).where( + models.InboxObject.ap_type == "Like", + models.InboxObject.relates_to_outbox_object_id == outbox_object.id, + models.InboxObject.is_deleted.is_(False), + ) + ) + ) + ( + await db_session.scalar( + select(func.count(models.Webmention.id)).where( + models.Webmention.is_deleted.is_(False), + models.Webmention.outbox_object_id == outbox_object.id, + models.Webmention.webmention_type == models.WebmentionType.LIKE, + ) + ) + ) + + +async def _get_outbox_announces_count( + db_session: AsyncSession, + outbox_object: models.OutboxObject, +) -> int: + return ( + await db_session.scalar( + select(func.count(models.InboxObject.id)).where( + models.InboxObject.ap_type == "Announce", + models.InboxObject.relates_to_outbox_object_id == outbox_object.id, + models.InboxObject.is_deleted.is_(False), + ) + ) + ) + ( + await db_session.scalar( + select(func.count(models.Webmention.id)).where( + models.Webmention.is_deleted.is_(False), + models.Webmention.outbox_object_id == outbox_object.id, + models.Webmention.webmention_type == models.WebmentionType.REPOST, + ) + ) + ) + + async def _revert_side_effect_for_deleted_object( db_session: AsyncSession, delete_activity: models.InboxObject | None, @@ -1231,8 +1321,8 @@ async def _revert_side_effect_for_deleted_object( # also needs to be forwarded is_delete_needs_to_be_forwarded = True - new_replies_count = await _get_replies_count( - db_session, replied_object.ap_id + new_replies_count = await _get_outbox_replies_count( + db_session, replied_object # type: ignore ) await db_session.execute( @@ -1262,12 +1352,13 @@ async def _revert_side_effect_for_deleted_object( ) if related_object: if related_object.is_from_outbox: + likes_count = await _get_outbox_likes_count(db_session, related_object) await db_session.execute( update(models.OutboxObject) .where( models.OutboxObject.id == related_object.id, ) - .values(likes_count=models.OutboxObject.likes_count - 1) + .values(likes_count=likes_count - 1) ) elif ( deleted_ap_object.ap_type == "Annouce" @@ -1279,12 +1370,15 @@ async def _revert_side_effect_for_deleted_object( ) if related_object: if related_object.is_from_outbox: + announces_count = await _get_outbox_announces_count( + db_session, related_object + ) await db_session.execute( update(models.OutboxObject) .where( models.OutboxObject.id == related_object.id, ) - .values(announces_count=models.OutboxObject.announces_count - 1) + .values(announces_count=announces_count - 1) ) # Delete any Like/Announce @@ -1826,8 +1920,8 @@ async def _process_note_object( replied_object, # type: ignore # outbox check below ) else: - new_replies_count = await _get_replies_count( - db_session, replied_object.ap_id + new_replies_count = await _get_outbox_replies_count( + db_session, replied_object # type: ignore ) await db_session.execute( @@ -2073,7 +2167,10 @@ async def _handle_like_activity( ) await db_session.delete(like_activity) else: - relates_to_outbox_object.likes_count = models.OutboxObject.likes_count + 1 + relates_to_outbox_object.likes_count = await _get_outbox_likes_count( + db_session, + relates_to_outbox_object, + ) notif = models.Notification( notification_type=models.NotificationType.LIKE, diff --git a/app/main.py b/app/main.py index 115e3a3..8325ada 100644 --- a/app/main.py +++ b/app/main.py @@ -799,24 +799,8 @@ async def article_by_slug( db_session: AsyncSession = Depends(get_db_session), httpsig_info: httpsig.HTTPSigInfo = Depends(httpsig.httpsig_checker), ) -> ActivityPubResponse | templates.TemplateResponse | RedirectResponse: - maybe_object = ( - ( - await db_session.execute( - select(models.OutboxObject) - .options( - joinedload(models.OutboxObject.outbox_object_attachments).options( - joinedload(models.OutboxObjectAttachment.upload) - ) - ) - .where( - models.OutboxObject.public_id.like(f"{short_id}%"), - models.OutboxObject.slug == slug, - models.OutboxObject.is_deleted.is_(False), - ) - ) - ) - .unique() - .scalar_one_or_none() + maybe_object = await boxes.get_outbox_object_by_slug_and_short_id( + db_session, slug, short_id ) if not maybe_object: raise HTTPException(status_code=404) diff --git a/app/models.py b/app/models.py index 391f420..13b1b0b 100644 --- a/app/models.py +++ b/app/models.py @@ -468,6 +468,14 @@ class IndieAuthAccessToken(Base): is_revoked = Column(Boolean, nullable=False, default=False) +@enum.unique +class WebmentionType(str, enum.Enum): + UNKNOWN = "unknown" + LIKE = "like" + REPLY = "reply" + REPOST = "repost" + + class Webmention(Base): __tablename__ = "webmention" __table_args__ = (UniqueConstraint("source", "target", name="uix_source_target"),) @@ -484,6 +492,8 @@ class Webmention(Base): outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=False) outbox_object = relationship(OutboxObject, uselist=False) + webmention_type = Column(Enum(WebmentionType), nullable=True) + @property def as_facepile_item(self) -> webmentions.Webmention | None: if not self.source_microformats: @@ -493,6 +503,7 @@ class Webmention(Base): self.source_microformats["items"], self.source ) except Exception: + # TODO: return a facepile with the unknown image logger.warning( f"Failed to generate facefile item for Webmention id={self.id}" ) diff --git a/app/webmentions.py b/app/webmentions.py index 20c4808..1c429be 100644 --- a/app/webmentions.py +++ b/app/webmentions.py @@ -1,3 +1,5 @@ +from urllib.parse import urlparse + import httpx from bs4 import BeautifulSoup # type: ignore from fastapi import APIRouter @@ -9,7 +11,11 @@ from loguru import logger from sqlalchemy import select from app import models +from app.boxes import _get_outbox_announces_count +from app.boxes import _get_outbox_likes_count +from app.boxes import _get_outbox_replies_count from app.boxes import get_outbox_object_by_ap_id +from app.boxes import get_outbox_object_by_slug_and_short_id from app.database import AsyncSession from app.database import get_db_session from app.utils import microformats @@ -47,6 +53,7 @@ async def webmention_endpoint( check_url(source) check_url(target) + parsed_target_url = urlparse(target) except Exception: logger.exception("Invalid webmention request") raise HTTPException(status_code=400, detail="Invalid payload") @@ -65,6 +72,16 @@ async def webmention_endpoint( logger.info("Found existing Webmention, will try to update or delete") mentioned_object = await get_outbox_object_by_ap_id(db_session, target) + + if not mentioned_object and parsed_target_url.path.startswith("/articles/"): + try: + _, _, short_id, slug = parsed_target_url.path.split("/") + mentioned_object = await get_outbox_object_by_slug_and_short_id( + db_session, slug, short_id + ) + except Exception: + logger.exception(f"Failed to match {target}") + if not mentioned_object: logger.info(f"Invalid target {target=}") @@ -90,8 +107,13 @@ async def webmention_endpoint( logger.warning(f"target {target=} not found in source") if existing_webmention_in_db: logger.info("Deleting existing Webmention") - mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1 existing_webmention_in_db.is_deleted = True + await db_session.flush() + + # Revert side effects + await _handle_webmention_side_effects( + db_session, existing_webmention_in_db, mentioned_object + ) notif = models.Notification( notification_type=models.NotificationType.DELETED_WEBMENTION, @@ -110,10 +132,25 @@ async def webmention_endpoint( else: return JSONResponse(content={}, status_code=200) + webmention_type = models.WebmentionType.UNKNOWN + for item in data.get("items", []): + if target in item.get("properties", {}).get("in-reply-to", []): + webmention_type = models.WebmentionType.REPLY + break + elif target in item.get("properties", {}).get("like-of", []): + webmention_type = models.WebmentionType.LIKE + break + elif target in item.get("properties", {}).get("repost-of", []): + webmention_type = models.WebmentionType.REPOST + break + + webmention: models.Webmention if existing_webmention_in_db: # Undelete if needed existing_webmention_in_db.is_deleted = False existing_webmention_in_db.source_microformats = data + await db_session.flush() + webmention = existing_webmention_in_db notif = models.Notification( notification_type=models.NotificationType.UPDATED_WEBMENTION, @@ -127,9 +164,11 @@ async def webmention_endpoint( target=target, source_microformats=data, outbox_object_id=mentioned_object.id, + webmention_type=webmention_type, ) db_session.add(new_webmention) await db_session.flush() + webmention = new_webmention notif = models.Notification( notification_type=models.NotificationType.NEW_WEBMENTION, @@ -138,8 +177,32 @@ async def webmention_endpoint( ) db_session.add(notif) - mentioned_object.webmentions_count = mentioned_object.webmentions_count + 1 - + # Handle side effect + await _handle_webmention_side_effects(db_session, webmention, mentioned_object) await db_session.commit() return JSONResponse(content={}, status_code=200) + + +async def _handle_webmention_side_effects( + db_session: AsyncSession, + webmention: models.Webmention, + mentioned_object: models.OutboxObject, +) -> None: + if webmention.webmention_type == models.WebmentionType.UNKNOWN: + # TODO: recount everything + mentioned_object.webmentions_count = mentioned_object.webmentions_count + 1 + elif webmention.webmention_type == models.WebmentionType.LIKE: + mentioned_object.likes_count = await _get_outbox_likes_count( + db_session, mentioned_object + ) + elif webmention.webmention_type == models.WebmentionType.REPOST: + mentioned_object.announces_count = await _get_outbox_announces_count( + db_session, mentioned_object + ) + elif webmention.webmention_type == models.WebmentionType.REPLY: + mentioned_object.replies_count = await _get_outbox_replies_count( + db_session, mentioned_object + ) + else: + raise ValueError(f"Unhandled {webmention.webmention_type} webmention")