Webmention improvements

- Tweak design for IndieAuth login flow
 - Webmentions notifications support
 - Refactor webmentions processing
main
Thomas Sileo 2022-07-19 20:38:32 +02:00
parent 9882fc555c
commit 0f6915fdbb
9 changed files with 204 additions and 90 deletions

View File

@ -49,6 +49,7 @@ def run_migrations_offline() -> None:
target_metadata=target_metadata, target_metadata=target_metadata,
literal_binds=True, literal_binds=True,
dialect_opts={"paramstyle": "named"}, dialect_opts={"paramstyle": "named"},
render_as_batch=True,
) )
with context.begin_transaction(): with context.begin_transaction():
@ -69,7 +70,11 @@ def run_migrations_online() -> None:
) )
with connectable.connect() as connection: with connectable.connect() as connection:
context.configure(connection=connection, target_metadata=target_metadata) context.configure(
connection=connection,
target_metadata=target_metadata,
render_as_batch=True,
)
with context.begin_transaction(): with context.begin_transaction():
context.run_migrations() context.run_migrations()

View File

@ -0,0 +1,32 @@
"""Webmention notifications
Revision ID: 2b51ae7047cb
Revises: e58c1ffadf2e
Create Date: 2022-07-19 20:22:06.968951
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = '2b51ae7047cb'
down_revision = 'e58c1ffadf2e'
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('notifications', schema=None) as batch_op:
batch_op.create_foreign_key('fk_webmention_id', 'webmention', ['webmention_id'], ['id'])
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('notifications', schema=None) as batch_op:
batch_op.drop_constraint('fk_webmention_id', type_='foreignkey')
# ### end Alembic commands ###

View File

@ -435,6 +435,7 @@ async def get_notifications(
models.OutboxObject.outbox_object_attachments models.OutboxObject.outbox_object_attachments
).options(joinedload(models.OutboxObjectAttachment.upload)), ).options(joinedload(models.OutboxObjectAttachment.upload)),
), ),
joinedload(models.Notification.webmention),
) )
.order_by(models.Notification.created_at.desc()) .order_by(models.Notification.created_at.desc())
) )

View File

@ -76,6 +76,8 @@ _RESIZED_CACHE: MutableMapping[tuple[str, int], tuple[bytes, str, Any]] = LFUCac
# TODO(ts): # TODO(ts):
# #
# Next: # Next:
# - Webmention notification
# - Page support
# - Article support # - Article support
# - indieauth tweaks # - indieauth tweaks
# - API for posting notes # - API for posting notes

View File

@ -300,35 +300,6 @@ class Following(Base):
ap_actor_id = Column(String, nullable=False, unique=True) ap_actor_id = Column(String, nullable=False, unique=True)
@enum.unique
class NotificationType(str, enum.Enum):
NEW_FOLLOWER = "new_follower"
UNFOLLOW = "unfollow"
LIKE = "like"
UNDO_LIKE = "undo_like"
ANNOUNCE = "announce"
UNDO_ANNOUNCE = "undo_announce"
MENTION = "mention"
class Notification(Base):
__tablename__ = "notifications"
id = Column(Integer, primary_key=True, index=True)
created_at = Column(DateTime(timezone=True), nullable=False, default=now)
notification_type = Column(Enum(NotificationType), nullable=True)
is_new = Column(Boolean, nullable=False, default=True)
actor_id = Column(Integer, ForeignKey("actor.id"), nullable=True)
actor = relationship(Actor, uselist=False)
outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=True)
outbox_object = relationship(OutboxObject, uselist=False)
inbox_object_id = Column(Integer, ForeignKey("inbox.id"), nullable=True)
inbox_object = relationship(InboxObject, uselist=False)
class IncomingActivity(Base): class IncomingActivity(Base):
__tablename__ = "incoming_activity" __tablename__ = "incoming_activity"
@ -503,6 +474,43 @@ class Webmention(Base):
return None return None
@enum.unique
class NotificationType(str, enum.Enum):
NEW_FOLLOWER = "new_follower"
UNFOLLOW = "unfollow"
LIKE = "like"
UNDO_LIKE = "undo_like"
ANNOUNCE = "announce"
UNDO_ANNOUNCE = "undo_announce"
MENTION = "mention"
NEW_WEBMENTION = "new_webmention"
UPDATED_WEBMENTION = "updated_webmention"
DELETED_WEBMENTION = "deleted_webmention"
class Notification(Base):
__tablename__ = "notifications"
id = Column(Integer, primary_key=True, index=True)
created_at = Column(DateTime(timezone=True), nullable=False, default=now)
notification_type = Column(Enum(NotificationType), nullable=True)
is_new = Column(Boolean, nullable=False, default=True)
actor_id = Column(Integer, ForeignKey("actor.id"), nullable=True)
actor = relationship(Actor, uselist=False)
outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=True)
outbox_object = relationship(OutboxObject, uselist=False)
inbox_object_id = Column(Integer, ForeignKey("inbox.id"), nullable=True)
inbox_object = relationship(InboxObject, uselist=False)
webmention_id = Column(
Integer, ForeignKey("webmention.id", name="fk_webmention_id"), nullable=True
)
webmention = relationship(Webmention, uselist=False)
outbox_fts = Table( outbox_fts = Table(
"outbox_fts", "outbox_fts",
metadata_obj, metadata_obj,

View File

@ -2,41 +2,40 @@
{% extends "layout.html" %} {% extends "layout.html" %}
{% block content %} {% block content %}
<div class="box"> <div class="box">
<div style="display:flex"> <div style="display:flex;column-gap: 20px;">
{% if client.logo %} {% if client.logo %}
<div style="flex:initial;width:100px;"> <div style="flex:initial;width:100px;">
<img src="{{client.logo | media_proxy_url }}" style="max-width:100px;"> <img src="{{client.logo | media_proxy_url }}" style="max-width:100px;" alt="{{ client.name }} logo">
</div> </div>
{% endif %} {% endif %}
<div style="flex:1;"> <div style="flex:1;">
<div style="margin-top:20px"> <div style="padding-left: 20px;">
<a class="lcolor" style="font-size:1.2em;font-weight:600;text-decoration:none;" href="{{ client.url }}">{{ client.name }}</a> <a class="lcolor" style="font-size:1.2em;font-weight:600;" href="{{ client.url }}">{{ client.name }}</a>
<p>wants you to login as <strong class="lcolor">{{ me }}</strong> with the following redirect URI: <code>{{ redirect_uri }}</code>.</p> <p>wants you to login as <strong class="lcolor">{{ me }}</strong> with the following redirect URI: <code>{{ redirect_uri }}</code>.</p>
</div>
</div>
</div>
<form method="POST" action="{{ url_for('indieauth_flow') }}">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
{% if scopes %}
<h3>Scopes</h3>
<ul>
{% for scope in scopes %}
<li><input type="checkbox" name="scopes" value="{{scope}}" id="scope-{{scope}}"><label for="scope-{{scope}}">{{ scope }}</label>
</li>
{% endfor %}
</ul>
{% endif %}
<input type="hidden" name="redirect_uri" value="{{ redirect_uri }}">
<input type="hidden" name="state" value="{{ state }}">
<input type="hidden" name="client_id" value="{{ client_id }}">
<input type="hidden" name="me" value="{{ me }}">
<input type="hidden" name="response_type" value="{{ response_type }}">
<input type="hidden" name="code_challenge" value="{{ code_challenge }}">
<input type="hidden" name="code_challenge_method" value="{{ code_challenge_method }}">
<input type="submit" value="login">
</form>
</div> <form method="POST" action="{{ url_for('indieauth_flow') }}" class="form">
{{ utils.embed_csrf_token() }}
{% if scopes %}
<h3>Scopes</h3>
<ul>
{% for scope in scopes %}
<li><input type="checkbox" name="scopes" value="{{scope}}" id="scope-{{scope}}"><label for="scope-{{scope}}">{{ scope }}</label>
</li>
{% endfor %}
</ul>
{% endif %}
<input type="hidden" name="redirect_uri" value="{{ redirect_uri }}">
<input type="hidden" name="state" value="{{ state }}">
<input type="hidden" name="client_id" value="{{ client_id }}">
<input type="hidden" name="me" value="{{ me }}">
<input type="hidden" name="response_type" value="{{ response_type }}">
<input type="hidden" name="code_challenge" value="{{ code_challenge }}">
<input type="hidden" name="code_challenge_method" value="{{ code_challenge_method }}">
<input type="submit" value="login">
</form>
</div>
</div>
</div>
</div> </div>
{% endblock %} {% endblock %}

View File

@ -47,7 +47,36 @@
<a style="font-weight:bold;" href="{{ notif.actor.url }}">{{ notif.actor.display_name }}</a> mentioned you <a style="font-weight:bold;" href="{{ notif.actor.url }}">{{ notif.actor.display_name }}</a> mentioned you
</div> </div>
{{ utils.display_object(notif.inbox_object) }} {{ utils.display_object(notif.inbox_object) }}
{% elif notif.notification_type.value == "new_webmention" %}
<div class="actor-action" title="{{ notif.created_at.isoformat() }}">
new webmention from
{% set facepile_item = notif.webmention.as_facepile_item %}
{% if facepile_item %}
<a href="{{ facepile_item.actor_url }}">{{ facepile_item.actor_name }}</a>
{% endif %}
<a style="font-weight:bold;" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
</div>
{{ utils.display_object(notif.outbox_object) }}
{% elif notif.notification_type.value == "updated_webmention" %}
<div class="actor-action" title="{{ notif.created_at.isoformat() }}">
updated webmention from
{% set facepile_item = notif.webmention.as_facepile_item %}
{% if facepile_item %}
<a href="{{ facepile_item.actor_url }}">{{ facepile_item.actor_name }}</a>
{% endif %}
<a style="font-weight:bold;" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
</div>
{{ utils.display_object(notif.outbox_object) }}
{% elif notif.notification_type.value == "deleted_webmention" %}
<div class="actor-action" title="{{ notif.created_at.isoformat() }}">
deleted webmention from
{% set facepile_item = notif.webmention.as_facepile_item %}
{% if facepile_item %}
<a href="{{ facepile_item.actor_url }}">{{ facepile_item.actor_name }}</a>
{% endif %}
<a style="font-weight:bold;" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
</div>
{{ utils.display_object(notif.outbox_object) }}
{% else %} {% else %}
<div class="actor-action"> <div class="actor-action">
Implement {{ notif.notification_type }} Implement {{ notif.notification_type }}

View File

@ -7,19 +7,28 @@ from loguru import logger
from app import config from app import config
async def fetch_and_parse(url: str) -> tuple[dict[str, Any], str] | None: class URLNotFoundOrGone(Exception):
pass
async def fetch_and_parse(url: str) -> tuple[dict[str, Any], str]:
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
resp = await client.get(
url,
headers={
"User-Agent": config.USER_AGENT,
},
follow_redirects=True,
)
if resp.status_code in [404, 410]:
raise URLNotFoundOrGone
try: try:
resp = await client.get(
url,
headers={
"User-Agent": config.USER_AGENT,
},
follow_redirects=True,
)
resp.raise_for_status() resp.raise_for_status()
except (httpx.HTTPError, httpx.HTTPStatusError): except httpx.HTTPStatusError:
logger.exception(f"Failed to discover webmention endpoint for {url}") logger.error(
return None f"Failed to parse microformats for {url}: " f"got {resp.status_code}"
)
raise
return mf2py.parse(doc=resp.text), resp.text return mf2py.parse(doc=resp.text), resp.text

View File

@ -1,3 +1,4 @@
import httpx
from bs4 import BeautifulSoup # type: ignore from bs4 import BeautifulSoup # type: ignore
from fastapi import APIRouter from fastapi import APIRouter
from fastapi import Depends from fastapi import Depends
@ -73,33 +74,53 @@ async def webmention_endpoint(
await db_session.commit() await db_session.commit()
raise HTTPException(status_code=400, detail="Invalid target") raise HTTPException(status_code=400, detail="Invalid target")
maybe_data_and_html = await microformats.fetch_and_parse(source) is_webmention_deleted = False
if not maybe_data_and_html: try:
logger.info("failed to fetch source") data_and_html = await microformats.fetch_and_parse(source)
except microformats.URLNotFoundOrGone:
is_webmention_deleted = True
except httpx.HTTPError:
raise HTTPException(status_code=500, detail=f"Fetch to process {source}")
data, html = data_and_html
is_target_found_in_source = is_source_containing_target(html, target)
data, html = data_and_html
if is_webmention_deleted or not is_target_found_in_source:
logger.warning(f"target {target=} not found in source")
if existing_webmention_in_db: if existing_webmention_in_db:
logger.info("Deleting existing Webmention") logger.info("Deleting existing Webmention")
mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1 mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1
existing_webmention_in_db.is_deleted = True existing_webmention_in_db.is_deleted = True
await db_session.commit()
raise HTTPException(status_code=400, detail="failed to fetch source")
data, html = maybe_data_and_html notif = models.Notification(
notification_type=models.NotificationType.DELETED_WEBMENTION,
outbox_object_id=mentioned_object.id,
webmention_id=existing_webmention_in_db.id,
)
db_session.add(notif)
if not is_source_containing_target(html, target):
logger.warning("target not found in source")
if existing_webmention_in_db:
logger.info("Deleting existing Webmention")
mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1
existing_webmention_in_db.is_deleted = True
await db_session.commit() await db_session.commit()
raise HTTPException(status_code=400, detail="target not found in source") if not is_target_found_in_source:
raise HTTPException(
status_code=400,
detail="target not found in source",
)
else:
return JSONResponse(content={}, status_code=200)
if existing_webmention_in_db: if existing_webmention_in_db:
# Undelete if needed
existing_webmention_in_db.is_deleted = False existing_webmention_in_db.is_deleted = False
existing_webmention_in_db.source_microformats = data existing_webmention_in_db.source_microformats = data
notif = models.Notification(
notification_type=models.NotificationType.UPDATED_WEBMENTION,
outbox_object_id=mentioned_object.id,
webmention_id=existing_webmention_in_db.id,
)
db_session.add(notif)
else: else:
new_webmention = models.Webmention( new_webmention = models.Webmention(
source=source, source=source,
@ -108,6 +129,14 @@ async def webmention_endpoint(
outbox_object_id=mentioned_object.id, outbox_object_id=mentioned_object.id,
) )
db_session.add(new_webmention) db_session.add(new_webmention)
await db_session.flush()
notif = models.Notification(
notification_type=models.NotificationType.NEW_WEBMENTION,
outbox_object_id=mentioned_object.id,
webmention_id=new_webmention.id,
)
db_session.add(notif)
mentioned_object.webmentions_count = mentioned_object.webmentions_count + 1 mentioned_object.webmentions_count = mentioned_object.webmentions_count + 1