feat(serverless): Python Serverless nocode instrumentation (#1004)

* Moved logic from aws_lambda.py to aws_lambda.__init__

* Added init function that revokes original handler

* Added documentation

* fix: Formatting

* Added test definition for serverless no code instrumentation

* TODO comments

* Refactored AWSLambda Layer script and fixed missing dir bug

* Removed redunant line

* Organized import

* Moved build-aws-layer script to integrations/aws_lambda

* Added check if path fails

* Renamed script to have underscore rather than dashes

* Fixed naming change for calling script

* Tests to ensure lambda check does not fail existing tests

* Added dest abs path as an arg

* Testing init script

* Modifying tests to accomodate addtion of layer

* Added test that ensures serverless auto instrumentation works as expected

* Removed redundant test arg from sentry_sdk init in serverless init

* Removed redundant todo statement

* Refactored layer and function creation into its own function

* Linting fixes

* Linting fixes

* Moved scripts from within sdk to scripts dir

* Updated documentation

* Pinned dependency to fix CI issue

Co-authored-by: sentry-bot <markus+ghbot@sentry.io>

Author: ahmedetefy

Makefile

@@ -63,5 +63,5 @@ apidocs-hotfix: apidocs
 aws-lambda-layer-build: dist
 	$(VENV_PATH)/bin/pip install urllib3
 	$(VENV_PATH)/bin/pip install certifi
-	$(VENV_PATH)/bin/python -m scripts.build-awslambda-layer
+	$(VENV_PATH)/bin/python -m scripts.build_awslambda_layer
 .PHONY: aws-lambda-layer-build

scripts/build-awslambda-layer.py

@@ -0,0 +1,115 @@
+import os
+import subprocess
+import tempfile
+import shutil
+
+from sentry_sdk.consts import VERSION as SDK_VERSION
+from sentry_sdk._types import MYPY
+
+if MYPY:
+    from typing import Union
+
+
+class PackageBuilder:
+    def __init__(
+        self,
+        base_dir,  # type: str
+        pkg_parent_dir,  # type: str
+        dist_rel_path,  # type: str
+    ):
+        # type: (...) -> None
+        self.base_dir = base_dir
+        self.pkg_parent_dir = pkg_parent_dir
+        self.dist_rel_path = dist_rel_path
+        self.packages_dir = self.get_relative_path_of(pkg_parent_dir)
+
+    def make_directories(self):
+        # type: (...) -> None
+        os.makedirs(self.packages_dir)
+
+    def install_python_binaries(self):
+        # type: (...) -> None
+        wheels_filepath = os.path.join(
+            self.dist_rel_path, f"sentry_sdk-{SDK_VERSION}-py2.py3-none-any.whl"
+        )
+        subprocess.run(
+            [
+                "pip",
+                "install",
+                "--no-cache-dir",  # Disables the cache -> always accesses PyPI
+                "-q",  # Quiet
+                wheels_filepath,  # Copied to the target directory before installation
+                "-t",  # Target directory flag
+                self.packages_dir,
+            ],
+            check=True,
+        )
+
+    def create_init_serverless_sdk_package(self):
+        # type: (...) -> None
+        """
+        Method that creates the init_serverless_sdk pkg in the
+        sentry-python-serverless zip
+        """
+        serverless_sdk_path = f'{self.packages_dir}/sentry_sdk/' \
+                              f'integrations/init_serverless_sdk'
+        if not os.path.exists(serverless_sdk_path):
+            os.makedirs(serverless_sdk_path)
+        shutil.copy('scripts/init_serverless_sdk.py',
+                    f'{serverless_sdk_path}/__init__.py')
+
+    def zip(
+        self, filename  # type: str
+    ):
+        # type: (...) -> None
+        subprocess.run(
+            [
+                "zip",
+                "-q",  # Quiet
+                "-x",  # Exclude files
+                "**/__pycache__/*",  # Files to be excluded
+                "-r",  # Recurse paths
+                filename,  # Output filename
+                self.pkg_parent_dir,  # Files to be zipped
+            ],
+            cwd=self.base_dir,
+            check=True,  # Raises CalledProcessError if exit status is non-zero
+        )
+
+    def get_relative_path_of(
+        self, subfile  # type: str
+    ):
+        # type: (...) -> str
+        return os.path.join(self.base_dir, subfile)
+
+
+# Ref to `pkg_parent_dir` Top directory in the ZIP file.
+# Placing the Sentry package in `/python` avoids
+# creating a directory for a specific version. For more information, see
+#  https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html#configuration-layers-path
+def build_packaged_zip(
+    dist_rel_path="dist",  # type: str
+    dest_zip_filename=f"sentry-python-serverless-{SDK_VERSION}.zip",  # type: str
+    pkg_parent_dir="python",  # type: str
+    dest_abs_path=None,  # type: Union[str, None]
+):
+    # type: (...) -> None
+    if dest_abs_path is None:
+        dest_abs_path = os.path.abspath(
+            os.path.join(os.path.dirname(__file__), "..", dist_rel_path)
+        )
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        package_builder = PackageBuilder(tmp_dir, pkg_parent_dir, dist_rel_path)
+        package_builder.make_directories()
+        package_builder.install_python_binaries()
+        package_builder.create_init_serverless_sdk_package()
+        package_builder.zip(dest_zip_filename)
+        if not os.path.exists(dist_rel_path):
+            os.makedirs(dist_rel_path)
+        shutil.copy(
+            package_builder.get_relative_path_of(dest_zip_filename), dest_abs_path
+        )
+
+
+if __name__ == "__main__":
+    build_packaged_zip()

scripts/build_awslambda_layer.py

@@ -0,0 +1,37 @@
+"""
+For manual instrumentation,
+The Handler function string of an aws lambda function should be added as an
+environment variable with a key of 'INITIAL_HANDLER' along with the 'DSN'
+Then the Handler function sstring should be replaced with
+'sentry_sdk.integrations.init_serverless_sdk.sentry_lambda_handler'
+"""
+import os
+
+import sentry_sdk
+from sentry_sdk._types import MYPY
+from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
+
+if MYPY:
+    from typing import Any
+
+
+# Configure Sentry SDK
+sentry_sdk.init(
+    dsn=os.environ["DSN"],
+    integrations=[AwsLambdaIntegration(timeout_warning=True)],
+)
+
+
+def sentry_lambda_handler(event, context):
+    # type: (Any, Any) -> None
+    """
+    Handler function that invokes a lambda handler which path is defined in
+    environment vairables as "INITIAL_HANDLER"
+    """
+    try:
+        module_name, handler_name = os.environ["INITIAL_HANDLER"].rsplit(".", 1)
+    except ValueError:
+        raise ValueError("Incorrect AWS Handler path (Not a path)")
+    lambda_function = __import__(module_name)
+    lambda_handler = getattr(lambda_function, handler_name)
+    lambda_handler(event, context)

scripts/init_serverless_sdk.py

@@ -17,6 +17,46 @@ def get_boto_client():
     )
 
 
+def build_no_code_serverless_function_and_layer(
+    client, tmpdir, fn_name, runtime, timeout
+):
+    """
+    Util function that auto instruments the no code implementation of the python
+    sdk by creating a layer containing the Python-sdk, and then creating a func
+    that uses that layer
+    """
+    from scripts.build_awslambda_layer import (
+        build_packaged_zip,
+    )
+
+    build_packaged_zip(dest_abs_path=tmpdir, dest_zip_filename="serverless-ball.zip")
+
+    with open(os.path.join(tmpdir, "serverless-ball.zip"), "rb") as serverless_zip:
+        response = client.publish_layer_version(
+            LayerName="python-serverless-sdk-test",
+            Description="Created as part of testsuite for getsentry/sentry-python",
+            Content={"ZipFile": serverless_zip.read()},
+        )
+
+    with open(os.path.join(tmpdir, "ball.zip"), "rb") as zip:
+        client.create_function(
+            FunctionName=fn_name,
+            Runtime=runtime,
+            Timeout=timeout,
+            Environment={
+                "Variables": {
+                    "INITIAL_HANDLER": "test_lambda.test_handler",
+                    "DSN": "https://123abc@example.com/123",
+                }
+            },
+            Role=os.environ["SENTRY_PYTHON_TEST_AWS_IAM_ROLE"],
+            Handler="sentry_sdk.integrations.init_serverless_sdk.sentry_lambda_handler",
+            Layers=[response["LayerVersionArn"]],
+            Code={"ZipFile": zip.read()},
+            Description="Created as part of testsuite for getsentry/sentry-python",
+        )
+
+
 def run_lambda_function(
     client,
     runtime,
@@ -25,6 +65,7 @@ def run_lambda_function(
     add_finalizer,
     syntax_check=True,
     timeout=30,
+    layer=None,
     subprocess_kwargs=(),
 ):
     subprocess_kwargs = dict(subprocess_kwargs)
@@ -40,39 +81,53 @@ def run_lambda_function(
             # such as chalice's)
             subprocess.check_call([sys.executable, test_lambda_py])
 
-        setup_cfg = os.path.join(tmpdir, "setup.cfg")
-        with open(setup_cfg, "w") as f:
-            f.write("[install]\nprefix=")
+        fn_name = "test_function_{}".format(uuid.uuid4())
 
-        subprocess.check_call(
-            [sys.executable, "setup.py", "sdist", "-d", os.path.join(tmpdir, "..")],
-            **subprocess_kwargs
-        )
+        if layer is None:
+            setup_cfg = os.path.join(tmpdir, "setup.cfg")
+            with open(setup_cfg, "w") as f:
+                f.write("[install]\nprefix=")
 
-        subprocess.check_call(
-            "pip install mock==3.0.0 funcsigs -t .",
-            cwd=tmpdir,
-            shell=True,
-            **subprocess_kwargs
-        )
+            subprocess.check_call(
+                [sys.executable, "setup.py", "sdist", "-d", os.path.join(tmpdir, "..")],
+                **subprocess_kwargs
+            )
 
-        # https://docs.aws.amazon.com/lambda/latest/dg/lambda-python-how-to-create-deployment-package.html
-        subprocess.check_call(
-            "pip install ../*.tar.gz -t .", cwd=tmpdir, shell=True, **subprocess_kwargs
-        )
-        shutil.make_archive(os.path.join(tmpdir, "ball"), "zip", tmpdir)
+            subprocess.check_call(
+                "pip install mock==3.0.0 funcsigs -t .",
+                cwd=tmpdir,
+                shell=True,
+                **subprocess_kwargs
+            )
 
-        fn_name = "test_function_{}".format(uuid.uuid4())
+            # https://docs.aws.amazon.com/lambda/latest/dg/lambda-python-how-to-create-deployment-package.html
+            subprocess.check_call(
+                "pip install ../*.tar.gz -t .",
+                cwd=tmpdir,
+                shell=True,
+                **subprocess_kwargs
+            )
 
-        with open(os.path.join(tmpdir, "ball.zip"), "rb") as zip:
-            client.create_function(
-                FunctionName=fn_name,
-                Runtime=runtime,
-                Timeout=timeout,
-                Role=os.environ["SENTRY_PYTHON_TEST_AWS_IAM_ROLE"],
-                Handler="test_lambda.test_handler",
-                Code={"ZipFile": zip.read()},
-                Description="Created as part of testsuite for getsentry/sentry-python",
+            shutil.make_archive(os.path.join(tmpdir, "ball"), "zip", tmpdir)
+
+            with open(os.path.join(tmpdir, "ball.zip"), "rb") as zip:
+                client.create_function(
+                    FunctionName=fn_name,
+                    Runtime=runtime,
+                    Timeout=timeout,
+                    Role=os.environ["SENTRY_PYTHON_TEST_AWS_IAM_ROLE"],
+                    Handler="test_lambda.test_handler",
+                    Code={"ZipFile": zip.read()},
+                    Description="Created as part of testsuite for getsentry/sentry-python",
+                )
+        else:
+            subprocess.run(
+                ["zip", "-q", "-x", "**/__pycache__/*", "-r", "ball.zip", "./"],
+                cwd=tmpdir,
+                check=True,
+            )
+            build_no_code_serverless_function_and_layer(
+                client, tmpdir, fn_name, runtime, timeout
             )
 
         @add_finalizer