From c160b75851baaa7c6a287dcad9344f2809b18d30 Mon Sep 17 00:00:00 2001 From: Thomas Sileo Date: Fri, 15 Jul 2022 20:50:27 +0200 Subject: [PATCH] More thorough URL checks --- app/activitypub.py | 5 +++++ app/main.py | 15 +++++++++++---- app/outgoing_activities.py | 2 ++ app/utils/url.py | 4 ++-- app/utils/webmentions.py | 3 +++ app/webfinger.py | 2 ++ 6 files changed, 25 insertions(+), 6 deletions(-) diff --git a/app/activitypub.py b/app/activitypub.py index 11a1989..603e28a 100644 --- a/app/activitypub.py +++ b/app/activitypub.py @@ -10,6 +10,7 @@ from app import config from app.config import AP_CONTENT_TYPE # noqa: F401 from app.httpsig import auth from app.key import get_pubkey_as_pem +from app.utils.url import check_url if TYPE_CHECKING: from app.actor import Actor @@ -112,6 +113,8 @@ async def fetch( params: dict[str, Any] | None = None, disable_httpsig: bool = False, ) -> RawObject: + check_url(url) + async with httpx.AsyncClient() as client: resp = await client.get( url, @@ -291,6 +294,8 @@ def remove_context(raw_object: RawObject) -> RawObject: def post(url: str, payload: dict[str, Any]) -> httpx.Response: + check_url(url) + resp = httpx.post( url, headers={ diff --git a/app/main.py b/app/main.py index 9b6502d..cbdd7a3 100644 --- a/app/main.py +++ b/app/main.py @@ -66,6 +66,7 @@ from app.templates import is_current_user_admin from app.uploads import UPLOAD_DIR from app.utils import pagination from app.utils.emoji import EMOJIS_BY_NAME +from app.utils.url import check_url from app.webfinger import get_remote_follow_template _RESIZED_CACHE: MutableMapping[tuple[str, int], tuple[bytes, str, Any]] = LFUCache(32) @@ -76,15 +77,12 @@ _RESIZED_CACHE: MutableMapping[tuple[str, int], tuple[bytes, str, Any]] = LFUCac # Next: # - fix stream (only content from follows + mention, and dedup shares) # - custom emoji in data/ -# - handle remove activity -# - retries httpx? -# - DB models for webmentions # - allow to undo follow requests # - indieauth tweaks # - API for posting notes # - allow to block servers # - FT5 text search -# - support update post with history +# - support update post with history? # # - [ ] block support # - [ ] prevent SSRF (urlutils from little-boxes) @@ -93,6 +91,12 @@ _RESIZED_CACHE: MutableMapping[tuple[str, int], tuple[bytes, str, Any]] = LFUCac class CustomMiddleware: + """Raw ASGI middleware as using starlette base middleware causes issues + with both: + - Jinja2: https://github.com/encode/starlette/issues/472 + - async SQLAchemy: https://github.com/tiangolo/fastapi/issues/4719 + """ + def __init__( self, app: ASGI3Application, @@ -808,6 +812,8 @@ proxy_client = httpx.AsyncClient(follow_redirects=True, http2=True) async def serve_proxy_media(request: Request, encoded_url: str) -> StreamingResponse: # Decode the base64-encoded URL url = base64.urlsafe_b64decode(encoded_url).decode() + check_url(url) + # Request the URL (and filter request headers) proxy_req = proxy_client.build_request( request.method, @@ -856,6 +862,7 @@ async def serve_proxy_media_resized( # Decode the base64-encoded URL url = base64.urlsafe_b64decode(encoded_url).decode() + check_url(url) if cached_resp := _RESIZED_CACHE.get((url, size)): resized_content, resized_mimetype, resp_headers = cached_resp diff --git a/app/outgoing_activities.py b/app/outgoing_activities.py index e1e1d6e..a2a773e 100644 --- a/app/outgoing_activities.py +++ b/app/outgoing_activities.py @@ -22,6 +22,7 @@ from app.database import AsyncSession from app.database import SessionLocal from app.key import Key from app.utils.datetime import now +from app.utils.url import check_url _MAX_RETRIES = 16 @@ -218,6 +219,7 @@ def process_next_outgoing_activity(db: Session) -> bool: "target": next_activity.webmention_target, } logger.info(f"{webmention_payload=}") + check_url(next_activity.recipient) resp = httpx.post( next_activity.recipient, data=webmention_payload, diff --git a/app/utils/url.py b/app/utils/url.py index 16d97ab..352fc59 100644 --- a/app/utils/url.py +++ b/app/utils/url.py @@ -24,7 +24,7 @@ class InvalidURLError(Exception): pass -@functools.lru_cache +@functools.lru_cache(maxsize=256) def _getaddrinfo(hostname: str, port: int) -> str: try: ip_address = str(ipaddress.ip_address(hostname)) @@ -65,7 +65,7 @@ def is_url_valid(url: str) -> bool: return True -def check_url(url: str, debug: bool = False) -> None: +def check_url(url: str) -> None: logger.debug(f"check_url {url=}") if not is_url_valid(url): raise InvalidURLError(f'"{url}" is invalid') diff --git a/app/utils/webmentions.py b/app/utils/webmentions.py index 3f986d1..772d10e 100644 --- a/app/utils/webmentions.py +++ b/app/utils/webmentions.py @@ -8,6 +8,7 @@ from loguru import logger from app import config from app.utils.datetime import now +from app.utils.url import check_url from app.utils.url import is_url_valid from app.utils.url import make_abs @@ -46,6 +47,8 @@ async def discover_webmention_endpoint(url: str) -> str | None: Passes all the tests at https://webmention.rocks! """ + check_url(url) + wurl = await _discover_webmention_endoint(url) if wurl is None: return None diff --git a/app/webfinger.py b/app/webfinger.py index e6bb213..8daf9d0 100644 --- a/app/webfinger.py +++ b/app/webfinger.py @@ -5,6 +5,7 @@ import httpx from loguru import logger from app import config +from app.utils.url import check_url async def webfinger( @@ -32,6 +33,7 @@ async def webfinger( for i, proto in enumerate(protos): try: url = f"{proto}://{host}/.well-known/webfinger" + check_url(url) resp = await client.get( url, params={"resource": resource},