adjust aws-replicator code imports for v3 changes (#43)

Author: whummer

.github/workflows/aws-replicator.yml

@@ -82,8 +82,6 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_AWS_SECRET_ACCESS_KEY }}
           LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }}
         run: |
-          # TODO tmp fix for https://github.com/localstack/localstack/issues/8267:
-          pip install --upgrade 'botocore<1.31.81'
           cd aws-replicator/example
           make test
 

aws-replicator/README.md

@@ -115,6 +115,7 @@ localstack extensions install "git+https://github.com/localstack/localstack-exte
 
 ## Change Log
 
+* `0.1.3`: Adjust code imports for recent LocalStack v3.0 module changes
 * `0.1.2`: Remove deprecated ProxyListener for starting local aws-replicator proxy server
 * `0.1.1`: Add simple configuration Web UI
 * `0.1.0`: Initial version of extension

aws-replicator/aws_replicator/client/auth_proxy.py

@@ -17,7 +17,7 @@
 from localstack.aws.api import HttpRequest
 from localstack.aws.protocol.parser import create_parser
 from localstack.aws.spec import load_service
-from localstack.config import get_edge_url
+from localstack.config import internal_service_url
 from localstack.constants import AWS_REGION_US_EAST_1, DOCKER_IMAGE_NAME_PRO
 from localstack.http import Request
 from localstack.utils.aws.aws_responses import requests_response
@@ -30,8 +30,8 @@
 from localstack.utils.net import get_free_tcp_port
 from localstack.utils.server.http2_server import run_server
 from localstack.utils.serving import Server
-from localstack.utils.strings import short_uid, to_str, truncate
-from localstack_ext.bootstrap.licensing import ENV_LOCALSTACK_API_KEY
+from localstack.utils.strings import short_uid, to_bytes, to_str, truncate
+from localstack_ext.bootstrap.licensingv2 import ENV_LOCALSTACK_API_KEY
 from requests import Response
 
 from aws_replicator.client.utils import truncate_content
@@ -138,7 +138,7 @@ def register_in_instance(self):
         port = getattr(self, "port", None)
         if not port:
             raise Exception("Proxy currently not running")
-        url = f"{get_edge_url()}{HANDLER_PATH_PROXIES}"
+        url = f"{internal_service_url()}{HANDLER_PATH_PROXIES}"
         data = AddProxyRequest(port=port, config=self.config)
         try:
             response = requests.post(url, json=data)
@@ -214,13 +214,22 @@ def _adjust_request_dict(self, service_name: str, request_dict: Dict):
                     '<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
                     f"<LocationConstraint>{region}</LocationConstraint></CreateBucketConfiguration>"
                 )
+
         if service_name == "sqs" and isinstance(req_body, dict):
             account_id = self._query_account_id_from_aws()
             if "QueueUrl" in req_body:
                 queue_name = req_body["QueueUrl"].split("/")[-1]
                 req_body["QueueUrl"] = f"https://queue.amazonaws.com/{account_id}/{queue_name}"
             if "QueueOwnerAWSAccountId" in req_body:
                 req_body["QueueOwnerAWSAccountId"] = account_id
+        if service_name == "sqs" and request_dict.get("url"):
+            req_json = run_safe(lambda: json.loads(body_str)) or {}
+            account_id = self._query_account_id_from_aws()
+            queue_name = req_json.get("QueueName")
+            if account_id and queue_name:
+                request_dict["url"] = f"https://queue.amazonaws.com/{account_id}/{queue_name}"
+                req_json["QueueOwnerAWSAccountId"] = account_id
+                request_dict["body"] = to_bytes(json.dumps(req_json))
 
     def _fix_headers(self, request: HttpRequest, service_name: str):
         if service_name == "s3":

aws-replicator/aws_replicator/server/aws_request_forwarder.py

@@ -61,7 +61,8 @@ def select_proxy(self, context: RequestContext) -> Optional[ProxyInstance]:
             proxy = self.PROXY_INSTANCES[port]
             proxy_config = proxy.get("config") or {}
             services = proxy_config.get("services") or {}
-            service_config = services.get(context.service.service_name)
+            service_name = self._get_canonical_service_name(context.service.service_name)
+            service_config = services.get(service_name)
             if not service_config:
                 continue
 
@@ -96,11 +97,12 @@ def select_proxy(self, context: RequestContext) -> Optional[ProxyInstance]:
     def _request_matches_resource(
         self, context: RequestContext, resource_name_pattern: str
     ) -> bool:
-        if context.service.service_name == "s3":
+        service_name = self._get_canonical_service_name(context.service.service_name)
+        if service_name == "s3":
             bucket_name = context.service_request.get("Bucket") or ""
             s3_bucket_arn = arns.s3_bucket_arn(bucket_name)
             return bool(re.match(resource_name_pattern, s3_bucket_arn))
-        if context.service.service_name == "sqs":
+        if service_name == "sqs":
             queue_name = context.service_request.get("QueueName") or ""
             queue_url = context.service_request.get("QueueUrl") or ""
             queue_name = queue_name or queue_url.split("/")[-1]
@@ -200,3 +202,9 @@ def _get_resource_names(cls, service_config: ProxyServiceConfig) -> list[str]:
         default_names = [".*"]
         result = service_config.get("resources") or default_names
         return ensure_list(result)
+
+    @classmethod
+    def _get_canonical_service_name(cls, service_name: str) -> str:
+        if service_name == "sqs-query":
+            return "sqs"
+        return service_name

aws-replicator/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = localstack-extension-aws-replicator
-version = 0.1.2
+version = 0.1.3
 summary = LocalStack Extension: AWS replicator
 description = Replicate AWS resources into your LocalStack instance
 long_description = file: README.md
@@ -16,8 +16,7 @@ install_requires =
     # TODO: currently requires a version pin, see note in auth_proxy.py
     boto3>=1.26.151
     # TODO: currently requires a version pin, see note in auth_proxy.py
-    # TODO: upper version pin due to https://github.com/localstack/localstack/issues/8267
-    botocore>=1.29.151,<1.31.81
+    botocore>=1.29.151
     flask
     localstack
     localstack-client