Bugfixes and cleanup

alembic/versions/79b5bcc918ce_tweak_inbox_objects_metadata.py

@@ -0,0 +1,34 @@
+"""Tweak inbox objects metadata
+
+Revision ID: 79b5bcc918ce
+Revises: 93e36ff5c691
+Create Date: 2022-07-07 18:03:46.945044
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '79b5bcc918ce'
+down_revision = '93e36ff5c691'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('inbox', sa.Column('is_deleted', sa.Boolean(), nullable=False, server_default="0"))
+    op.add_column('inbox', sa.Column('replies_count', sa.Integer(), nullable=False, server_default="0"))
+    op.drop_column('inbox', 'has_replies')
+    op.create_index(op.f('ix_outgoing_activity_id'), 'outgoing_activity', ['id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_outgoing_activity_id'), table_name='outgoing_activity')
+    op.add_column('inbox', sa.Column('has_replies', sa.BOOLEAN(), nullable=False))
+    op.drop_column('inbox', 'replies_count')
+    op.drop_column('inbox', 'is_deleted')
+    # ### end Alembic commands ###

app/admin.py

@@ -172,6 +172,7 @@ async def admin_bookmarks(
                         ["Note", "Article", "Video", "Announce"]
                     ),
                     models.InboxObject.is_bookmarked.is_(True),
+                    models.InboxObject.is_deleted.is_(False),
                 )
                 .options(
                     joinedload(models.InboxObject.relates_to_inbox_object),
@@ -199,6 +200,77 @@ async def admin_bookmarks(
     )
 
 
+@router.get("/stream")
+async def admin_stream(
+    request: Request,
+    db_session: AsyncSession = Depends(get_db_session),
+    cursor: str | None = None,
+) -> templates.TemplateResponse:
+    where = [
+        models.InboxObject.is_hidden_from_stream.is_(False),
+        models.InboxObject.is_deleted.is_(False),
+    ]
+    if cursor:
+        where.append(
+            models.InboxObject.ap_published_at < pagination.decode_cursor(cursor)
+        )
+
+    page_size = 20
+    remaining_count = await db_session.scalar(
+        select(func.count(models.InboxObject.id)).where(*where)
+    )
+    q = select(models.InboxObject).where(*where)
+
+    inbox = (
+        (
+            await db_session.scalars(
+                q.options(
+                    joinedload(models.InboxObject.relates_to_inbox_object).options(
+                        joinedload(models.InboxObject.actor)
+                    ),
+                    joinedload(models.InboxObject.relates_to_outbox_object).options(
+                        joinedload(
+                            models.OutboxObject.outbox_object_attachments
+                        ).options(joinedload(models.OutboxObjectAttachment.upload)),
+                    ),
+                    joinedload(models.InboxObject.actor),
+                )
+                .order_by(models.InboxObject.ap_published_at.desc())
+                .limit(20)
+            )
+        )
+        .unique()
+        .all()
+    )
+
+    next_cursor = (
+        pagination.encode_cursor(inbox[-1].ap_published_at)
+        if inbox and remaining_count > page_size
+        else None
+    )
+
+    actors_metadata = await get_actors_metadata(
+        db_session,
+        [
+            inbox_object.actor
+            for inbox_object in inbox
+            if inbox_object.ap_type == "Follow"
+        ],
+    )
+
+    return await templates.render_template(
+        db_session,
+        request,
+        "admin_inbox.html",
+        {
+            "inbox": inbox,
+            "actors_metadata": actors_metadata,
+            "next_cursor": next_cursor,
+            "show_filters": False,
+        },
+    )
+
+
 @router.get("/inbox")
 async def admin_inbox(
     request: Request,
@@ -207,7 +279,10 @@ async def admin_inbox(
     cursor: str | None = None,
 ) -> templates.TemplateResponse:
     where = [
-        models.InboxObject.ap_type.not_in(["Accept", "Delete", "Create", "Update"])
+        models.InboxObject.ap_type.not_in(
+            ["Accept", "Delete", "Create", "Update", "Undo"]
+        ),
+        models.InboxObject.is_deleted.is_(False),
     ]
     if filter_by:
         where.append(models.InboxObject.ap_type == filter_by)
@@ -267,6 +342,7 @@ async def admin_inbox(
             "inbox": inbox,
             "actors_metadata": actors_metadata,
             "next_cursor": next_cursor,
+            "show_filters": True,
         },
     )
 
@@ -425,6 +501,7 @@ async def admin_profile(
         await db_session.scalars(
             select(models.InboxObject)
             .where(
+                models.InboxObject.is_deleted.is_(False),
                 models.InboxObject.actor_id == actor.id,
                 models.InboxObject.ap_type.in_(["Note", "Article", "Video"]),
             )

app/ap_object.py

@@ -103,7 +103,19 @@ class Object:
                             )
                         )
                         break
-
+                    elif link.get("mediaType", "") == "application/x-mpegURL":
+                        for tag in ap.as_list(link.get("tag", [])):
+                            if tag.get("mediaType", "").startswith("video"):
+                                proxied_url = proxied_media_url(tag["href"])
+                                attachments.append(
+                                    Attachment(
+                                        type="Video",
+                                        mediaType=tag["mediaType"],
+                                        url=tag["href"],
+                                        proxiedUrl=proxied_url,
+                                    )
+                                )
+                                break
         return attachments
 
     @property

app/boxes.py

@@ -517,11 +517,8 @@ async def _handle_delete_activity(
                 inbox_object_id=delete_activity.id,
             )
 
-    # TODO(ts): do we need to delete related activities? should we keep
-    # bookmarked objects with a deleted flag?
     logger.info(f"Deleting {ap_object_to_delete.ap_type}/{ap_object_to_delete.ap_id}")
-    await db_session.delete(ap_object_to_delete)
+    ap_object_to_delete.is_deleted = True
-    await db_session.flush()
 
 
 async def _handle_follow_follow_activity(
@@ -575,6 +572,7 @@ async def _handle_undo_activity(
         return
 
     ap_activity_to_undo.undone_by_inbox_object_id = undo_activity.id
+    ap_activity_to_undo.is_deleted = True
 
     if ap_activity_to_undo.ap_type == "Follow":
         logger.info(f"Undo follow from {from_actor.ap_id}")
@@ -635,8 +633,6 @@ async def _handle_undo_activity(
                     inbox_object_id=ap_activity_to_undo.id,
                 )
                 db_session.add(notif)
-
-        # FIXME(ts): what to do with ap_activity_to_undo? flag? delete?
     else:
         logger.warning(f"Don't know how to undo {ap_activity_to_undo.ap_type} activity")
 
@@ -687,6 +683,14 @@ async def _handle_create_activity(
     if "published" in ro.ap_object:
         ap_published_at = isoparse(ro.ap_object["published"])
 
+    is_reply = bool(ro.in_reply_to)
+    is_local_reply = ro.in_reply_to and ro.in_reply_to.startswith(BASE_URL)
+    is_mention = False
+    tags = ro.ap_object.get("tag", [])
+    for tag in tags:
+        if tag.get("name") == LOCAL_ACTOR.handle or tag.get("href") == LOCAL_ACTOR.url:
+            is_mention = True
+
     inbox_object = models.InboxObject(
         server=urlparse(ro.ap_id).netloc,
         actor_id=from_actor.id,
@@ -701,11 +705,7 @@ async def _handle_create_activity(
         relates_to_outbox_object_id=None,
         activity_object_ap_id=ro.activity_object_ap_id,
         # Hide replies from the stream
-        is_hidden_from_stream=(
+        is_hidden_from_stream=not (not is_reply or is_mention or is_local_reply),
-            True
-            if (ro.in_reply_to and not ro.in_reply_to.startswith(BASE_URL))
-            else False
-        ),  # TODO: handle mentions
     )
 
     db_session.add(inbox_object)
@@ -714,28 +714,35 @@ async def _handle_create_activity(
 
     create_activity.relates_to_inbox_object_id = inbox_object.id
 
-    tags = inbox_object.ap_object.get("tag")
+    if inbox_object.in_reply_to:
-
+        replied_object = await get_anybox_object_by_ap_id(
-    if not tags:
+            db_session, inbox_object.in_reply_to
-        logger.info("No tags to process")
-        return None
-
-    if not isinstance(tags, list):
-        logger.info(f"Invalid tags: {tags}")
-        return None
-
-    if inbox_object.in_reply_to and inbox_object.in_reply_to.startswith(BASE_URL):
-        await db_session.execute(
-            update(models.OutboxObject)
-            .where(
-                models.OutboxObject.ap_id == inbox_object.in_reply_to,
-            )
-            .values(replies_count=models.OutboxObject.replies_count + 1)
         )
+        if replied_object:
+            if replied_object.is_from_outbox:
+                await db_session.execute(
+                    update(models.OutboxObject)
+                    .where(
+                        models.OutboxObject.id == replied_object.id,
+                    )
+                    .values(replies_count=models.OutboxObject.replies_count + 1)
+                )
+            else:
+                await db_session.execute(
+                    update(models.InboxObject)
+                    .where(
+                        models.InboxObject.id == replied_object.id,
+                    )
+                    .values(replies_count=models.InboxObject.replies_count + 1)
+                )
 
         # This object is a reply of a local object, we may need to forward it
         # to our followers (we can only forward JSON-LD signed activities)
-        if create_activity.has_ld_signature:
+        if (
+            replied_object
+            and replied_object.is_from_outbox
+            and create_activity.has_ld_signature
+        ):
             logger.info("Forwarding Create activity as it's a local reply")
             recipients = await _get_followers_recipients(db_session)
             for rcp in recipients:
@@ -746,14 +753,13 @@ async def _handle_create_activity(
                     inbox_object_id=create_activity.id,
                 )
 
-    for tag in tags:
+    if is_mention:
-        if tag.get("name") == LOCAL_ACTOR.handle or tag.get("href") == LOCAL_ACTOR.url:
+        notif = models.Notification(
-            notif = models.Notification(
+            notification_type=models.NotificationType.MENTION,
-                notification_type=models.NotificationType.MENTION,
+            actor_id=from_actor.id,
-                actor_id=from_actor.id,
+            inbox_object_id=inbox_object.id,
-                inbox_object_id=inbox_object.id,
+        )
-            )
+        db_session.add(notif)
-            db_session.add(notif)
 
 
 async def save_to_inbox(
@@ -825,7 +831,6 @@ async def save_to_inbox(
         if relates_to_outbox_object
         else None,
         activity_object_ap_id=activity_ro.activity_object_ap_id,
-        # Hide replies from the stream
         is_hidden_from_stream=True,
     )
 
@@ -921,7 +926,9 @@ async def save_to_inbox(
         else:
             # This is announce for a maybe unknown object
             if relates_to_inbox_object:
-                logger.info("Nothing to do, we already know about this object")
+                # We already know about this object, show the announce in the
+                # streal
+                inbox_object.is_hidden_from_stream = False
             else:
                 # Save it as an inbox object
                 if not activity_ro.activity_object_ap_id:
@@ -946,6 +953,7 @@ async def save_to_inbox(
                 db_session.add(announced_inbox_object)
                 await db_session.flush()
                 inbox_object.relates_to_inbox_object_id = announced_inbox_object.id
+                inbox_object.is_hidden_from_stream = False
     elif activity_ro.ap_type in ["Like", "Announce"]:
         if not relates_to_outbox_object:
             logger.info(
@@ -1030,40 +1038,44 @@ async def get_replies_tree(
     db_session: AsyncSession,
     requested_object: AnyboxObject,
 ) -> ReplyTreeNode:
-    # TODO: handle visibility
+    # XXX: PeerTube video don't use context
     tree_nodes: list[AnyboxObject] = []
-    tree_nodes.extend(
+    if requested_object.ap_context is None:
-        (
+        tree_nodes = [requested_object]
-            await db_session.scalars(
+    else:
-                select(models.InboxObject)
+        # TODO: handle visibility
-                .where(
+        tree_nodes.extend(
-                    models.InboxObject.ap_context == requested_object.ap_context,
+            (
-                    models.InboxObject.ap_type.not_in(["Announce"]),
+                await db_session.scalars(
+                    select(models.InboxObject)
+                    .where(
+                        models.InboxObject.ap_context == requested_object.ap_context,
+                        models.InboxObject.ap_type.not_in(["Announce"]),
+                    )
+                    .options(joinedload(models.InboxObject.actor))
                 )
-                .options(joinedload(models.InboxObject.actor))
             )
+            .unique()
+            .all()
         )
-        .unique()
+        tree_nodes.extend(
-        .all()
+            (
-    )
+                await db_session.scalars(
-    tree_nodes.extend(
+                    select(models.OutboxObject)
-        (
+                    .where(
-            await db_session.scalars(
+                        models.OutboxObject.ap_context == requested_object.ap_context,
-                select(models.OutboxObject)
+                        models.OutboxObject.is_deleted.is_(False),
-                .where(
+                    )
-                    models.OutboxObject.ap_context == requested_object.ap_context,
+                    .options(
-                    models.OutboxObject.is_deleted.is_(False),
+                        joinedload(
-                )
+                            models.OutboxObject.outbox_object_attachments
-                .options(
+                        ).options(joinedload(models.OutboxObjectAttachment.upload))
-                    joinedload(models.OutboxObject.outbox_object_attachments).options(
-                        joinedload(models.OutboxObjectAttachment.upload)
                     )
                 )
             )
+            .unique()
+            .all()
         )
-        .unique()
-        .all()
-    )
     nodes_by_in_reply_to = defaultdict(list)
     for node in tree_nodes:
         nodes_by_in_reply_to[node.in_reply_to].append(node)

app/config.py

@@ -15,7 +15,7 @@ from app.utils.emoji import _load_emojis
 
 ROOT_DIR = Path().parent.resolve()
 
-_CONFIG_FILE = os.getenv("MICROBLOGPUB_CONFIG_FILE", "me.toml")
+_CONFIG_FILE = os.getenv("MICROBLOGPUB_CONFIG_FILE", "profile.toml")
 
 VERSION_COMMIT = (
     subprocess.check_output(["git", "rev-parse", "--short=8", "HEAD"])

app/models.py

@@ -100,8 +100,10 @@ class InboxObject(Base, BaseObject):
 
     is_bookmarked = Column(Boolean, nullable=False, default=False)
 
-    # FIXME(ts): do we need this?
+    # Used to mark deleted objects, but also activities that were undone
-    has_replies = Column(Boolean, nullable=False, default=False)
+    is_deleted = Column(Boolean, nullable=False, default=False)
+
+    replies_count = Column(Integer, nullable=False, default=0)
 
     og_meta: Mapped[list[dict[str, Any]] | None] = Column(JSON, nullable=True)
 

app/scss/main.scss

@@ -174,6 +174,17 @@ nav.flexbox {
     }
 }
 
+.activity-expanded {
+  .activity-attachment {
+img, audio, video {
+      width: 100%;
+      max-width: 740px;
+      margin: 30px 0;
+    }
+
+  }
+}
+
 
 .activity-wrap {
   margin: 0 auto;

app/templates/admin_inbox.html

@@ -2,7 +2,9 @@
 {% extends "layout.html" %}
 {% block content %}
 
+{% if show_filters %}
 {{ utils.display_box_filters("admin_inbox") }}
+{% endif %}
 
 {% macro actor_action(inbox_object, text) %}
     <div class="actor-action">
@@ -33,7 +35,7 @@
 {% endfor %}
 
 {% if next_cursor %}
-<p><a href="{{ url_for("admin_inbox") }}?cursor={{ next_cursor }}{% if request.query_params.filter_by %}&filter_by={{ request.query_params.filter_by }}{% endif %}">See more</a></p>
+<p><a href="{{ request.url._path }}?cursor={{ next_cursor }}{% if request.query_params.filter_by %}&filter_by={{ request.query_params.filter_by }}{% endif %}">See more</a></p>
 {% endif %}
 
 {% endblock %}

app/templates/layout.html

@@ -28,6 +28,7 @@
         <li>Admin</li>
         <li>{{ admin_link("index", "Public") }}</li>
         <li>{{ admin_link("admin_new", "New") }}</li>
+        <li>{{ admin_link("admin_stream", "Stream") }}</li>
         <li>{{ admin_link("admin_inbox", "Inbox") }}/{{ admin_link("admin_outbox", "Outbox") }}</li>
         <li>{{ admin_link("get_notifications", "Notifications") }} {% if notifications_count %}({{ notifications_count }}){% endif %}</li>
         <li>{{ admin_link("get_lookup", "Lookup") }}</li>

app/templates/utils.html

@@ -210,7 +210,7 @@
     {% if attachment.type == "Image" or (attachment | has_media_type("image")) %}
     <img src="{{ attachment.resized_url or attachment.proxied_url }}"{% if attachment.name %} alt="{{ attachment.name }}"{% endif %} class="attachment">
     {% elif attachment.type == "Video" or (attachment | has_media_type("video")) %}
-    <video controls preload="metadata"  src="{{ attachment.url | media_proxy_url }}"{% if attachment.name %} title="{{ attachment.name }}"{% endif %} class="attachmeent"></video>
+    <video controls preload="metadata"  src="{{ attachment.url | media_proxy_url }}"{% if attachment.name %} title="{{ attachment.name }}"{% endif %} class="attachment"></video>
     {% elif attachment.type == "Audio" or (attachment | has_media_type("audio")) %}
     <audio controls preload="metadata"  src="{{ attachment.url | media_proxy_url }}"{% if attachment.name%} title="{{ attachment.name }}"{% endif %} style="width:480px;" class="attachment"></audio>
     {% else %}
@@ -264,7 +264,9 @@
 </ul>
 </nav>
 
+<div class="activity-attachment">
 {{ display_attachments(object) }}
+</div>
 
 {% if likes or shares %}
 <div style="display: flex;column-gap: 20px;margin:20px 0;">

scripts/config_wizard.py

@@ -29,11 +29,11 @@ def main() -> None:
     else:
         generate_key(_KEY_PATH)
 
-    config_file = Path("data/me.toml")
+    config_file = Path("data/profile.toml")
 
     if config_file.exists():
         # Spit out the relative path for the "config artifacts"
-        rconfig_file = "data/me.toml"
+        rconfig_file = "data/profile.toml"
         print(
             f"Existing setup detected, please delete {rconfig_file} "
             "before restarting the wizard"