HTTP clients tweaks

main
Thomas Sileo 2022-07-13 20:05:15 +02:00
parent 6616343cd3
commit 88b57f29af
5 changed files with 49 additions and 6 deletions

View File

@ -14,6 +14,8 @@ from app.key import get_pubkey_as_pem
if TYPE_CHECKING: if TYPE_CHECKING:
from app.actor import Actor from app.actor import Actor
_HTTPX_TRANSPORT = httpx.AsyncHTTPTransport(retries=1)
RawObject = dict[str, Any] RawObject = dict[str, Any]
AS_CTX = "https://www.w3.org/ns/activitystreams" AS_CTX = "https://www.w3.org/ns/activitystreams"
AS_PUBLIC = "https://www.w3.org/ns/activitystreams#Public" AS_PUBLIC = "https://www.w3.org/ns/activitystreams#Public"
@ -49,6 +51,10 @@ class ObjectIsGoneError(Exception):
pass pass
class ObjectNotFoundError(Exception):
pass
class VisibilityEnum(str, enum.Enum): class VisibilityEnum(str, enum.Enum):
PUBLIC = "public" PUBLIC = "public"
UNLISTED = "unlisted" UNLISTED = "unlisted"
@ -104,7 +110,7 @@ class NotAnObjectError(Exception):
async def fetch(url: str, params: dict[str, Any] | None = None) -> RawObject: async def fetch(url: str, params: dict[str, Any] | None = None) -> RawObject:
async with httpx.AsyncClient() as client: async with httpx.AsyncClient(transport=_HTTPX_TRANSPORT) as client:
resp = await client.get( resp = await client.get(
url, url,
headers={ headers={
@ -119,6 +125,8 @@ async def fetch(url: str, params: dict[str, Any] | None = None) -> RawObject:
# Special handling for deleted object # Special handling for deleted object
if resp.status_code == 410: if resp.status_code == 410:
raise ObjectIsGoneError(f"{url} is gone") raise ObjectIsGoneError(f"{url} is gone")
elif resp.status_code == 404:
raise ObjectNotFoundError(f"{url} not found")
resp.raise_for_status() resp.raise_for_status()
try: try:

View File

@ -138,8 +138,8 @@ async def httpsig_checker(
try: try:
k = await _get_public_key(db_session, hsig["keyId"]) k = await _get_public_key(db_session, hsig["keyId"])
except ap.ObjectIsGoneError: except (ap.ObjectIsGoneError, ap.ObjectNotFoundError):
logger.info("Actor is gone") logger.info("Actor is gone or not found")
return HTTPSigInfo(has_valid_signature=False, is_ap_actor_gone=True) return HTTPSigInfo(has_valid_signature=False, is_ap_actor_gone=True)
except Exception: except Exception:
logger.exception(f'Failed to fetch HTTP sig key {hsig["keyId"]}') logger.exception(f'Failed to fetch HTTP sig key {hsig["keyId"]}')

View File

@ -66,6 +66,9 @@ _RESIZED_CACHE: MutableMapping[tuple[str, int], tuple[bytes, str, Any]] = LFUCac
# TODO(ts): # TODO(ts):
# #
# Next: # Next:
# - incoming activity worker
# - handle remove activity
# - retries httpx?
# - DB models for webmentions # - DB models for webmentions
# - allow to undo follow requests # - allow to undo follow requests
# - indieauth tweaks # - indieauth tweaks
@ -760,7 +763,7 @@ async def nodeinfo(
) )
proxy_client = httpx.AsyncClient(follow_redirects=True) proxy_client = httpx.AsyncClient(follow_redirects=True, http2=True)
@app.get("/proxy/media/{encoded_url}") @app.get("/proxy/media/{encoded_url}")

34
poetry.lock generated
View File

@ -365,6 +365,26 @@ category = "main"
optional = false optional = false
python-versions = ">=3.6" python-versions = ">=3.6"
[[package]]
name = "h2"
version = "4.1.0"
description = "HTTP/2 State-Machine based protocol implementation"
category = "main"
optional = false
python-versions = ">=3.6.1"
[package.dependencies]
hpack = ">=4.0,<5"
hyperframe = ">=6.0,<7"
[[package]]
name = "hpack"
version = "4.0.0"
description = "Pure-Python HPACK header compression"
category = "main"
optional = false
python-versions = ">=3.6.1"
[[package]] [[package]]
name = "html2text" name = "html2text"
version = "2020.1.16" version = "2020.1.16"
@ -419,6 +439,7 @@ python-versions = ">=3.7"
[package.dependencies] [package.dependencies]
certifi = "*" certifi = "*"
h2 = {version = ">=3,<5", optional = true, markers = "extra == \"http2\""}
httpcore = ">=0.15.0,<0.16.0" httpcore = ">=0.15.0,<0.16.0"
rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]} rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]}
sniffio = "*" sniffio = "*"
@ -440,6 +461,14 @@ python-versions = ">=3.7"
[package.extras] [package.extras]
tests = ["freezegun", "pytest", "pytest-cov"] tests = ["freezegun", "pytest", "pytest-cov"]
[[package]]
name = "hyperframe"
version = "6.0.1"
description = "HTTP/2 framing layer for Python"
category = "main"
optional = false
python-versions = ">=3.6.1"
[[package]] [[package]]
name = "idna" name = "idna"
version = "3.3" version = "3.3"
@ -1173,7 +1202,7 @@ dev = ["pytest (>=4.6.2)", "black (>=19.3b0)"]
[metadata] [metadata]
lock-version = "1.1" lock-version = "1.1"
python-versions = "^3.10" python-versions = "^3.10"
content-hash = "fd741c6c1c1e85cb1b39150df503bc64b28244b65222180c6768409fcfd1d70a" content-hash = "7bc5ba65a004438ac015dcd01c27e1d327dbf491f9f881a48a2a790bb0bbf710"
[metadata.files] [metadata.files]
aiosqlite = [ aiosqlite = [
@ -1454,6 +1483,8 @@ h11 = [
{file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"}, {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"},
{file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"}, {file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"},
] ]
h2 = []
hpack = []
html2text = [ html2text = [
{file = "html2text-2020.1.16-py3-none-any.whl", hash = "sha256:c7c629882da0cf377d66f073329ccf34a12ed2adf0169b9285ae4e63ef54c82b"}, {file = "html2text-2020.1.16-py3-none-any.whl", hash = "sha256:c7c629882da0cf377d66f073329ccf34a12ed2adf0169b9285ae4e63ef54c82b"},
{file = "html2text-2020.1.16.tar.gz", hash = "sha256:e296318e16b059ddb97f7a8a1d6a5c1d7af4544049a01e261731d2d5cc277bbb"}, {file = "html2text-2020.1.16.tar.gz", hash = "sha256:e296318e16b059ddb97f7a8a1d6a5c1d7af4544049a01e261731d2d5cc277bbb"},
@ -1474,6 +1505,7 @@ humanize = [
{file = "humanize-4.2.3-py3-none-any.whl", hash = "sha256:bed628920d45cd5018abb095710f0c03a8336d6ac0790e7647c6a328f3880b81"}, {file = "humanize-4.2.3-py3-none-any.whl", hash = "sha256:bed628920d45cd5018abb095710f0c03a8336d6ac0790e7647c6a328f3880b81"},
{file = "humanize-4.2.3.tar.gz", hash = "sha256:2bc1fdd831cd00557d3010abdd84d3e41b4a96703a3eaf6c24ee290b26b75a44"}, {file = "humanize-4.2.3.tar.gz", hash = "sha256:2bc1fdd831cd00557d3010abdd84d3e41b4a96703a3eaf6c24ee290b26b75a44"},
] ]
hyperframe = []
idna = [ idna = [
{file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"},
{file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"},

View File

@ -15,7 +15,7 @@ bcrypt = "^3.2.2"
itsdangerous = "^2.1.2" itsdangerous = "^2.1.2"
python-multipart = "^0.0.5" python-multipart = "^0.0.5"
tomli = "^2.0.1" tomli = "^2.0.1"
httpx = "^0.23.0" httpx = {extras = ["http2"], version = "^0.23.0"}
SQLAlchemy = {extras = ["asyncio"], version = "^1.4.39"} SQLAlchemy = {extras = ["asyncio"], version = "^1.4.39"}
alembic = "^1.8.0" alembic = "^1.8.0"
bleach = "^5.0.0" bleach = "^5.0.0"