zaklaus
/
microblog
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Webmention improvements 

- Tweak design for IndieAuth login flow
 - Webmentions notifications support
 - Refactor webmentions processing
main
Thomas Sileo 2022-07-19 20:38:32 +02:00
parent 9882fc555c
commit 0f6915fdbb
9 changed files with 204 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
alembic/env.py
View File

@ -49,6 +49,7 @@ def run_migrations_offline() -> None:
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
render_as_batch=True,
)
with context.begin_transaction():
@ -69,7 +70,11 @@ def run_migrations_online() -> None:
)
with connectable.connect() as connection:
context.configure(connection=connection, target_metadata=target_metadata)
context.configure(
connection=connection,
target_metadata=target_metadata,
render_as_batch=True,
)
with context.begin_transaction():
context.run_migrations()

32
alembic/versions/2b51ae7047cb_webmention_notifications.py 100644
View File

@ -0,0 +1,32 @@
"""Webmention notifications
Revision ID: 2b51ae7047cb
Revises: e58c1ffadf2e
Create Date: 2022-07-19 20:22:06.968951
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = '2b51ae7047cb'
down_revision = 'e58c1ffadf2e'
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('notifications', schema=None) as batch_op:
batch_op.create_foreign_key('fk_webmention_id', 'webmention', ['webmention_id'], ['id'])
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('notifications', schema=None) as batch_op:
batch_op.drop_constraint('fk_webmention_id', type_='foreignkey')
# ### end Alembic commands ###

1
app/admin.py
View File

@ -435,6 +435,7 @@ async def get_notifications(
models.OutboxObject.outbox_object_attachments
).options(joinedload(models.OutboxObjectAttachment.upload)),
),
joinedload(models.Notification.webmention),
)
.order_by(models.Notification.created_at.desc())
)

2
app/main.py
View File

@ -76,6 +76,8 @@ _RESIZED_CACHE: MutableMapping[tuple[str, int], tuple[bytes, str, Any]] = LFUCac
# TODO(ts):
#
# Next:
# - Webmention notification
# - Page support
# - Article support
# - indieauth tweaks
# - API for posting notes

66
app/models.py
View File

@ -300,35 +300,6 @@ class Following(Base):
ap_actor_id = Column(String, nullable=False, unique=True)
@enum.unique
class NotificationType(str, enum.Enum):
NEW_FOLLOWER = "new_follower"
UNFOLLOW = "unfollow"
LIKE = "like"
UNDO_LIKE = "undo_like"
ANNOUNCE = "announce"
UNDO_ANNOUNCE = "undo_announce"
MENTION = "mention"
class Notification(Base):
__tablename__ = "notifications"
id = Column(Integer, primary_key=True, index=True)
created_at = Column(DateTime(timezone=True), nullable=False, default=now)
notification_type = Column(Enum(NotificationType), nullable=True)
is_new = Column(Boolean, nullable=False, default=True)
actor_id = Column(Integer, ForeignKey("actor.id"), nullable=True)
actor = relationship(Actor, uselist=False)
outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=True)
outbox_object = relationship(OutboxObject, uselist=False)
inbox_object_id = Column(Integer, ForeignKey("inbox.id"), nullable=True)
inbox_object = relationship(InboxObject, uselist=False)
class IncomingActivity(Base):
__tablename__ = "incoming_activity"
@ -503,6 +474,43 @@ class Webmention(Base):
return None
@enum.unique
class NotificationType(str, enum.Enum):
NEW_FOLLOWER = "new_follower"
UNFOLLOW = "unfollow"
LIKE = "like"
UNDO_LIKE = "undo_like"
ANNOUNCE = "announce"
UNDO_ANNOUNCE = "undo_announce"
MENTION = "mention"
NEW_WEBMENTION = "new_webmention"
UPDATED_WEBMENTION = "updated_webmention"
DELETED_WEBMENTION = "deleted_webmention"
class Notification(Base):
__tablename__ = "notifications"
id = Column(Integer, primary_key=True, index=True)
created_at = Column(DateTime(timezone=True), nullable=False, default=now)
notification_type = Column(Enum(NotificationType), nullable=True)
is_new = Column(Boolean, nullable=False, default=True)
actor_id = Column(Integer, ForeignKey("actor.id"), nullable=True)
actor = relationship(Actor, uselist=False)
outbox_object_id = Column(Integer, ForeignKey("outbox.id"), nullable=True)
outbox_object = relationship(OutboxObject, uselist=False)
inbox_object_id = Column(Integer, ForeignKey("inbox.id"), nullable=True)
inbox_object = relationship(InboxObject, uselist=False)
webmention_id = Column(
Integer, ForeignKey("webmention.id", name="fk_webmention_id"), nullable=True
)
webmention = relationship(Webmention, uselist=False)
outbox_fts = Table(
"outbox_fts",
metadata_obj,

67
app/templates/indieauth_flow.html
View File

@ -2,41 +2,40 @@
{% extends "layout.html" %}
{% block content %}
<div class="box">
<div style="display:flex">
{% if client.logo %}
<div style="flex:initial;width:100px;">
<img src="{{client.logo | media_proxy_url }}" style="max-width:100px;">
</div>
{% endif %}
<div style="flex:1;">
<div style="margin-top:20px">
<a class="lcolor" style="font-size:1.2em;font-weight:600;text-decoration:none;" href="{{ client.url }}">{{ client.name }}</a>
<p>wants you to login as <strong class="lcolor">{{ me }}</strong> with the following redirect URI: <code>{{ redirect_uri }}</code>.</p>
</div>
</div>
</div>
<div style="display:flex;column-gap: 20px;">
{% if client.logo %}
<div style="flex:initial;width:100px;">
<img src="{{client.logo | media_proxy_url }}" style="max-width:100px;" alt="{{ client.name }} logo">
</div>
{% endif %}
<div style="flex:1;">
<div style="padding-left: 20px;">
<a class="lcolor" style="font-size:1.2em;font-weight:600;" href="{{ client.url }}">{{ client.name }}</a>
<p>wants you to login as <strong class="lcolor">{{ me }}</strong> with the following redirect URI: <code>{{ redirect_uri }}</code>.</p>
<form method="POST" action="{{ url_for('indieauth_flow') }}">
<input type="hidden" name="csrf_token" value="{{ csrf_token }}">
{% if scopes %}
<h3>Scopes</h3>
<ul>
{% for scope in scopes %}
<li><input type="checkbox" name="scopes" value="{{scope}}" id="scope-{{scope}}"><label for="scope-{{scope}}">{{ scope }}</label>
</li>
{% endfor %}
</ul>
{% endif %}
<input type="hidden" name="redirect_uri" value="{{ redirect_uri }}">
<input type="hidden" name="state" value="{{ state }}">
<input type="hidden" name="client_id" value="{{ client_id }}">
<input type="hidden" name="me" value="{{ me }}">
<input type="hidden" name="response_type" value="{{ response_type }}">
<input type="hidden" name="code_challenge" value="{{ code_challenge }}">
<input type="hidden" name="code_challenge_method" value="{{ code_challenge_method }}">
<input type="submit" value="login">
</form>
</div>
<form method="POST" action="{{ url_for('indieauth_flow') }}" class="form">
{{ utils.embed_csrf_token() }}
{% if scopes %}
<h3>Scopes</h3>
<ul>
{% for scope in scopes %}
<li><input type="checkbox" name="scopes" value="{{scope}}" id="scope-{{scope}}"><label for="scope-{{scope}}">{{ scope }}</label>
</li>
{% endfor %}
</ul>
{% endif %}
<input type="hidden" name="redirect_uri" value="{{ redirect_uri }}">
<input type="hidden" name="state" value="{{ state }}">
<input type="hidden" name="client_id" value="{{ client_id }}">
<input type="hidden" name="me" value="{{ me }}">
<input type="hidden" name="response_type" value="{{ response_type }}">
<input type="hidden" name="code_challenge" value="{{ code_challenge }}">
<input type="hidden" name="code_challenge_method" value="{{ code_challenge_method }}">
<input type="submit" value="login">
</form>
</div>
</div>
</div>
</div>
{% endblock %}

31
app/templates/notifications.html
View File

@ -47,7 +47,36 @@
<a style="font-weight:bold;" href="{{ notif.actor.url }}">{{ notif.actor.display_name }}</a> mentioned you
</div>
{{ utils.display_object(notif.inbox_object) }}
{% elif notif.notification_type.value == "new_webmention" %}
<div class="actor-action" title="{{ notif.created_at.isoformat() }}">
new webmention from
{% set facepile_item = notif.webmention.as_facepile_item %}
{% if facepile_item %}
<a href="{{ facepile_item.actor_url }}">{{ facepile_item.actor_name }}</a>
{% endif %}
<a style="font-weight:bold;" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
</div>
{{ utils.display_object(notif.outbox_object) }}
{% elif notif.notification_type.value == "updated_webmention" %}
<div class="actor-action" title="{{ notif.created_at.isoformat() }}">
updated webmention from
{% set facepile_item = notif.webmention.as_facepile_item %}
{% if facepile_item %}
<a href="{{ facepile_item.actor_url }}">{{ facepile_item.actor_name }}</a>
{% endif %}
<a style="font-weight:bold;" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
</div>
{{ utils.display_object(notif.outbox_object) }}
{% elif notif.notification_type.value == "deleted_webmention" %}
<div class="actor-action" title="{{ notif.created_at.isoformat() }}">
deleted webmention from
{% set facepile_item = notif.webmention.as_facepile_item %}
{% if facepile_item %}
<a href="{{ facepile_item.actor_url }}">{{ facepile_item.actor_name }}</a>
{% endif %}
<a style="font-weight:bold;" href="{{ notif.webmention.source }}">{{ notif.webmention.source }}</a>
</div>
{{ utils.display_object(notif.outbox_object) }}
{% else %}
<div class="actor-action">
Implement {{ notif.notification_type }}

31
app/utils/microformats.py
View File

@ -7,19 +7,28 @@ from loguru import logger
from app import config
async def fetch_and_parse(url: str) -> tuple[dict[str, Any], str] | None:
class URLNotFoundOrGone(Exception):
pass
async def fetch_and_parse(url: str) -> tuple[dict[str, Any], str]:
async with httpx.AsyncClient() as client:
resp = await client.get(
url,
headers={
"User-Agent": config.USER_AGENT,
},
follow_redirects=True,
)
if resp.status_code in [404, 410]:
raise URLNotFoundOrGone
try:
resp = await client.get(
url,
headers={
"User-Agent": config.USER_AGENT,
},
follow_redirects=True,
)
resp.raise_for_status()
except (httpx.HTTPError, httpx.HTTPStatusError):
logger.exception(f"Failed to discover webmention endpoint for {url}")
return None
except httpx.HTTPStatusError:
logger.error(
f"Failed to parse microformats for {url}: " f"got {resp.status_code}"
)
raise
return mf2py.parse(doc=resp.text), resp.text

57
app/webmentions.py
View File

@ -1,3 +1,4 @@
import httpx
from bs4 import BeautifulSoup # type: ignore
from fastapi import APIRouter
from fastapi import Depends
@ -73,33 +74,53 @@ async def webmention_endpoint(
await db_session.commit()
raise HTTPException(status_code=400, detail="Invalid target")
maybe_data_and_html = await microformats.fetch_and_parse(source)
if not maybe_data_and_html:
logger.info("failed to fetch source")
is_webmention_deleted = False
try:
data_and_html = await microformats.fetch_and_parse(source)
except microformats.URLNotFoundOrGone:
is_webmention_deleted = True
except httpx.HTTPError:
raise HTTPException(status_code=500, detail=f"Fetch to process {source}")
data, html = data_and_html
is_target_found_in_source = is_source_containing_target(html, target)
data, html = data_and_html
if is_webmention_deleted or not is_target_found_in_source:
logger.warning(f"target {target=} not found in source")
if existing_webmention_in_db:
logger.info("Deleting existing Webmention")
mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1
existing_webmention_in_db.is_deleted = True
await db_session.commit()
raise HTTPException(status_code=400, detail="failed to fetch source")
data, html = maybe_data_and_html
notif = models.Notification(
notification_type=models.NotificationType.DELETED_WEBMENTION,
outbox_object_id=mentioned_object.id,
webmention_id=existing_webmention_in_db.id,
)
db_session.add(notif)
if not is_source_containing_target(html, target):
logger.warning("target not found in source")
if existing_webmention_in_db:
logger.info("Deleting existing Webmention")
mentioned_object.webmentions_count = mentioned_object.webmentions_count - 1
existing_webmention_in_db.is_deleted = True
await db_session.commit()
raise HTTPException(status_code=400, detail="target not found in source")
if not is_target_found_in_source:
raise HTTPException(
status_code=400,
detail="target not found in source",
)
else:
return JSONResponse(content={}, status_code=200)
if existing_webmention_in_db:
# Undelete if needed
existing_webmention_in_db.is_deleted = False
existing_webmention_in_db.source_microformats = data
notif = models.Notification(
notification_type=models.NotificationType.UPDATED_WEBMENTION,
outbox_object_id=mentioned_object.id,
webmention_id=existing_webmention_in_db.id,
)
db_session.add(notif)
else:
new_webmention = models.Webmention(
source=source,
@ -108,6 +129,14 @@ async def webmention_endpoint(
outbox_object_id=mentioned_object.id,
)
db_session.add(new_webmention)
await db_session.flush()
notif = models.Notification(
notification_type=models.NotificationType.NEW_WEBMENTION,
outbox_object_id=mentioned_object.id,
webmention_id=new_webmention.id,
)
db_session.add(notif)
mentioned_object.webmentions_count = mentioned_object.webmentions_count + 1