Skip to content

Commit baccb05

Browse files
committed
WIP: Async support in pulp-glue
Replaces requests with aiohttp and changes the api.
1 parent 76ede0c commit baccb05

File tree

21 files changed

+744
-565
lines changed

21 files changed

+744
-565
lines changed

.ci/settings/settings.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
ALLOWED_EXPORT_PATHS = ["/tmp"]
44
ANALYTICS = False
55
ALLOWED_CONTENT_CHECKSUMS = ["sha1", "sha256", "sha512"]
6+
TASK_DIAGNOSTICS = ["memory"]
67

78
pulp_https = os.environ.get("PULP_HTTPS", "false").lower() == "true"
89
pulp_oauth2 = os.environ.get("PULP_OAUTH2", "false").lower() == "true"

CHANGES/pulp-glue/+aiohttp.feature

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
WIP: Added async api to Pulp glue.

CHANGES/pulp-glue/+aiohttp.removal

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Replaced requests with aiohttp.
2+
Breaking change: Reworked the contract around the `AuthProvider` to allow authentication to be coded independently of the underlying library.

lint_requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@ mypy==1.19.0
77
shellcheck-py==0.11.0.1
88

99
# Type annotation stubs
10+
types-aiofiles
1011
types-pygments
1112
types-PyYAML
12-
types-requests
1313
types-setuptools
1414
types-toml
1515

lower_bounds_constraints.lock

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
aiofiles==25.1.0
2+
aiohttp==3.12.0
13
click==8.0.0
24
packaging==20.0
35
PyYAML==5.3

pulp-glue/docs/dev/learn/architecture.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ To this end, `pulp-glue` is the go-to place for all known version-dependent Pulp
88

99
## OpenAPI
1010

11-
This is the part in `pulp_glue` that uses [`requests`](https://requests.readthedocs.io/) to perform low level communication with an `OpenAPI 3` compatible server.
11+
This is the part in `pulp_glue` that uses http to perform low level communication with an `OpenAPI 3` compatible server.
1212
It is not anticipated that users of Pulp Glue need to interact with this abstraction layer.
1313

1414
## Contexts
Lines changed: 133 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -1,97 +1,146 @@
11
import typing as t
2-
from datetime import datetime, timedelta
2+
from datetime import datetime
33

4-
import requests
54

6-
7-
class OAuth2ClientCredentialsAuth(requests.auth.AuthBase):
8-
"""
9-
This implements the OAuth2 ClientCredentials Grant authentication flow.
10-
https://datatracker.ietf.org/doc/html/rfc6749#section-4.4
5+
class AuthProviderBase:
116
"""
7+
Base class for auth providers.
128
13-
def __init__(
14-
self,
15-
client_id: str,
16-
client_secret: str,
17-
token_url: str,
18-
scopes: list[str] | None = None,
19-
verify_ssl: str | bool | None = None,
20-
):
21-
self._token_server_auth = requests.auth.HTTPBasicAuth(client_id, client_secret)
22-
self._token_url = token_url
23-
self._scopes = scopes
24-
self._verify_ssl = verify_ssl
9+
This abstract base class will analyze the authentication proposals of the openapi specs.
10+
Different authentication schemes can be implemented in subclasses.
11+
"""
2512

26-
self._access_token: str | None = None
27-
self._expire_at: datetime | None = None
13+
def __init__(self) -> None:
14+
self._oauth2_token: str | None = None
15+
self._oauth2_expires: datetime = datetime.now()
16+
17+
def can_complete_http_basic(self) -> bool:
18+
return False
19+
20+
def can_complete_mutualTLS(self) -> bool:
21+
return False
22+
23+
def can_complete_oauth2_client_credentials(self, scopes: list[str]) -> bool:
24+
return False
25+
26+
def can_complete_scheme(self, scheme: dict[str, t.Any], scopes: list[str]) -> bool:
27+
if scheme["type"] == "http":
28+
if scheme["scheme"] == "basic":
29+
return self.can_complete_http_basic()
30+
elif scheme["type"] == "mutualTLS":
31+
return self.can_complete_mutualTLS()
32+
elif scheme["type"] == "oauth2":
33+
for flow_name, flow in scheme["flows"].items():
34+
if (
35+
flow_name == "clientCredentials"
36+
and self.can_complete_oauth2_client_credentials(flow["scopes"])
37+
):
38+
return True
39+
return False
40+
41+
def can_complete(
42+
self, proposal: dict[str, list[str]], security_schemes: dict[str, dict[str, t.Any]]
43+
) -> bool:
44+
for name, scopes in proposal.items():
45+
scheme = security_schemes.get(name)
46+
if scheme is None or not self.can_complete_scheme(scheme, scopes):
47+
return False
48+
# This covers the case where `[]` allows for no auth at all.
49+
return True
50+
51+
async def auth_success_hook(
52+
self, proposal: dict[str, list[str]], security_schemes: dict[str, dict[str, t.Any]]
53+
) -> None:
54+
pass
55+
56+
async def auth_failure_hook(
57+
self, proposal: dict[str, list[str]], security_schemes: dict[str, dict[str, t.Any]]
58+
) -> None:
59+
pass
60+
61+
async def http_basic_credentials(self) -> tuple[bytes, bytes]:
62+
raise NotImplementedError()
63+
64+
async def oauth2_client_credentials(self) -> tuple[bytes, bytes]:
65+
raise NotImplementedError()
66+
67+
def tls_credentials(self) -> tuple[str, str | None]:
68+
raise NotImplementedError()
69+
70+
71+
class BasicAuthProvider(AuthProviderBase):
72+
"""
73+
AuthProvider providing basic auth with fixed `username`, `password`.
74+
"""
2875

29-
def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest:
30-
if self._expire_at is None or self._expire_at < datetime.now():
31-
self._retrieve_token()
76+
def __init__(self, username: t.AnyStr, password: t.AnyStr):
77+
super().__init__()
78+
self.username: bytes = username.encode("latin1") if isinstance(username, str) else username
79+
self.password: bytes = password.encode("latin1") if isinstance(password, str) else password
3280

33-
assert self._access_token is not None
81+
def can_complete_http_basic(self) -> bool:
82+
return True
3483

35-
request.headers["Authorization"] = f"Bearer {self._access_token}"
84+
async def http_basic_credentials(self) -> tuple[bytes, bytes]:
85+
return self.username, self.password
3686

37-
# Call to untyped function "register_hook" in typed context
38-
request.register_hook("response", self._handle401) # type: ignore[no-untyped-call]
3987

40-
return request
88+
class GlueAuthProvider(AuthProviderBase):
89+
"""
90+
AuthProvider allowing to be used with prepared credentials.
91+
"""
4192

42-
def _handle401(
93+
def __init__(
4394
self,
44-
response: requests.Response,
45-
**kwargs: t.Any,
46-
) -> requests.Response:
47-
if response.status_code != 401:
48-
return response
49-
50-
# If we get this far, probably the token is not valid anymore.
51-
52-
# Try to reach for a new token once.
53-
self._retrieve_token()
54-
55-
assert self._access_token is not None
56-
57-
# Consume content and release the original connection
58-
# to allow our new request to reuse the same one.
59-
response.content
60-
response.close()
61-
prepared_new_request = response.request.copy()
62-
63-
prepared_new_request.headers["Authorization"] = f"Bearer {self._access_token}"
64-
65-
# Avoid to enter into an infinity loop.
66-
# Call to untyped function "deregister_hook" in typed context
67-
prepared_new_request.deregister_hook( # type: ignore[no-untyped-call]
68-
"response", self._handle401
69-
)
70-
71-
# "Response" has no attribute "connection"
72-
new_response: requests.Response = response.connection.send(prepared_new_request, **kwargs)
73-
new_response.history.append(response)
74-
new_response.request = prepared_new_request
75-
76-
return new_response
77-
78-
def _retrieve_token(self) -> None:
79-
data = {
80-
"grant_type": "client_credentials",
81-
}
82-
83-
if self._scopes:
84-
data["scope"] = " ".join(self._scopes)
85-
86-
response: requests.Response = requests.post(
87-
self._token_url,
88-
data=data,
89-
auth=self._token_server_auth,
90-
verify=self._verify_ssl,
91-
)
92-
93-
response.raise_for_status()
94-
95-
token = response.json()
96-
self._expire_at = datetime.now() + timedelta(seconds=token["expires_in"])
97-
self._access_token = token["access_token"]
95+
*,
96+
username: t.AnyStr | None = None,
97+
password: t.AnyStr | None = None,
98+
client_id: t.AnyStr | None = None,
99+
client_secret: t.AnyStr | None = None,
100+
cert: str | None = None,
101+
key: str | None = None,
102+
):
103+
super().__init__()
104+
self.username: bytes | None = None
105+
self.password: bytes | None = None
106+
self.client_id: bytes | None = None
107+
self.client_secret: bytes | None = None
108+
self.cert: str | None = cert
109+
self.key: str | None = key
110+
111+
if username is not None:
112+
assert password is not None
113+
self.username = username.encode("latin1") if isinstance(username, str) else username
114+
self.password = password.encode("latin1") if isinstance(password, str) else password
115+
if client_id is not None:
116+
assert client_secret is not None
117+
self.client_id = client_id.encode("latin1") if isinstance(client_id, str) else client_id
118+
self.client_secret = (
119+
client_secret.encode("latin1") if isinstance(client_secret, str) else client_secret
120+
)
121+
122+
if cert is None and key is not None:
123+
raise RuntimeError("Key can only be used together with a cert.")
124+
125+
def can_complete_http_basic(self) -> bool:
126+
return self.username is not None
127+
128+
def can_complete_oauth2_client_credentials(self, scopes: list[str]) -> bool:
129+
return self.client_id is not None
130+
131+
def can_complete_mutualTLS(self) -> bool:
132+
return self.cert is not None
133+
134+
async def http_basic_credentials(self) -> tuple[bytes, bytes]:
135+
assert self.username is not None
136+
assert self.password is not None
137+
return self.username, self.password
138+
139+
async def oauth2_client_credentials(self) -> tuple[bytes, bytes]:
140+
assert self.client_id is not None
141+
assert self.client_secret is not None
142+
return self.client_id, self.client_secret
143+
144+
def tls_credentials(self) -> tuple[str, str | None]:
145+
assert self.cert is not None
146+
return (self.cert, self.key)

pulp-glue/pulp_glue/common/context.py

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
from packaging.specifiers import SpecifierSet
1111

12+
from pulp_glue.common.authentication import GlueAuthProvider
1213
from pulp_glue.common.exceptions import (
1314
NotImplementedFake,
1415
OpenAPIError,
@@ -19,7 +20,7 @@
1920
UnsafeCallError,
2021
)
2122
from pulp_glue.common.i18n import get_translation
22-
from pulp_glue.common.openapi import BasicAuthProvider, OpenAPI
23+
from pulp_glue.common.openapi import OpenAPI
2324

2425
if sys.version_info >= (3, 11):
2526
import tomllib
@@ -202,6 +203,20 @@ def patch_upstream_pulp_replicate_request_body(api: OpenAPI) -> None:
202203
operation.pop("requestBody", None)
203204

204205

206+
@api_quirk(PluginRequirement("core", specifier="<3.85"))
207+
def patch_security_scheme_mutual_tls(api: OpenAPI) -> None:
208+
# Trick to allow tls cert auth on older Pulp.
209+
if (components := api.api_spec.get("components")) is not None:
210+
if (security_schemes := components.get("securitySchemes")) is not None:
211+
# Only if it is going to be idempotent...
212+
if "gluePatchTLS" not in security_schemes:
213+
security_schemes["gluePatchTLS"] = {"type": "mutualTLS"}
214+
for method, path in api.operations.values():
215+
operation = api.api_spec["paths"][path][method]
216+
if "security" in operation:
217+
operation["security"].append({"gluePatchTLS": []})
218+
219+
205220
class PulpContext:
206221
"""
207222
Abstract class for the global PulpContext object.
@@ -335,8 +350,13 @@ def from_config(cls, config: dict[str, t.Any]) -> "t.Self":
335350
api_kwargs: dict[str, t.Any] = {
336351
"base_url": config["base_url"],
337352
}
338-
if "username" in config:
339-
api_kwargs["auth_provider"] = BasicAuthProvider(config["username"], config["password"])
353+
api_kwargs["auth_provider"] = GlueAuthProvider(
354+
**{
355+
k: v
356+
for k, v in config.items()
357+
if k in {"username", "password", "client_id", "client_secret", "cert", "key"}
358+
}
359+
)
340360
if "headers" in config:
341361
api_kwargs["headers"] = dict(
342362
(header.split(":", maxsplit=1) for header in config["headers"])
@@ -385,7 +405,9 @@ def api(self) -> OpenAPI:
385405
# Deprecated for 'auth'.
386406
if not password:
387407
password = self.prompt("password", hide_input=True)
388-
self._api_kwargs["auth_provider"] = BasicAuthProvider(username, password)
408+
self._api_kwargs["auth_provider"] = GlueAuthProvider(
409+
username=username, password=password
410+
)
389411
warnings.warn(
390412
"Using 'username' and 'password' with 'PulpContext' is deprecated. "
391413
"Use an auth provider with the 'auth_provider' argument instead.",
@@ -399,10 +421,10 @@ def api(self) -> OpenAPI:
399421
)
400422
except OpenAPIError as e:
401423
raise PulpException(str(e))
424+
self._patch_api_spec()
402425
# Rerun scheduled version checks
403426
for plugin_requirement in self._needed_plugins:
404427
self.needs_plugin(plugin_requirement)
405-
self._patch_api_spec()
406428
return self._api
407429

408430
@property

0 commit comments

Comments
 (0)