Initial revision

Author: mheffner

.github/workflows/benchmark.yml

@@ -0,0 +1,63 @@
+name: Run Benchmark
+
+on:
+  workflow_dispatch:
+    inputs:
+      aws-region:
+        description: 'AWS Region'
+        required: true
+        default: 'us-east-1'
+      otel-layer:
+        description: 'OTEL Lambda Layer'
+        required: true
+        default: 'arn:aws:lambda:us-east-1:184161586896:layer:opentelemetry-collector-amd64-0_13_0:1'
+      rotel-layer:
+        description: 'Rotel Lambda Layer'
+        required: true
+        default: 'arn:aws:lambda:us-east-1:418653438961:layer:rotel-extension-amd64-alpha:12'
+      python-version:
+        description: 'Python version to use'
+        required: true
+        default: '3.13'
+
+jobs:
+  run-benchmark:
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ github.event.inputs.python-version }}
+
+      - name: Verify UV installation
+        run: uv --version
+
+      - name: Install deps
+        run: |
+          cd benchmark && uv sync
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_LAMBDA_BENCHMARK_ROLE_ARN }}
+          aws-region: us-east-1
+
+      - name: run
+        env:
+          AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }}
+          AWS_REGION: ${{ github.event.inputs.aws-region }}
+          OTEL_LAYER: ${{ github.event.inputs.otel-layer }}
+          ROTEL_LAYER: ${{ github.event.inputs.rotel-layer }}
+        run: |
+          ./scripts/benchmark-coldstart.sh

.gitignore

@@ -0,0 +1,6 @@
+function.zip
+rotel-layer.zip
+tmp/
+outputfile.txt
+package/
+lambda_benchmark_results.json

Makefile

@@ -0,0 +1,15 @@
+.PHONY: bundle deps
+
+SHELL := /bin/bash
+
+deps:
+	rm -rf package && \
+    mkdir -p package && \
+    pip install -q -r requirements.txt --target ./package
+
+function.zip: deps SimpleLambda.py collector.yaml rotel.env
+	rm -f function.zip && \
+    cd package && zip -q -r ../function.zip . && \
+    cd .. && zip -q function.zip SimpleLambda.py collector.yaml rotel.env
+
+bundle: function.zip

README.md

@@ -0,0 +1,11 @@
+# python-lambda-benchmark
+
+Small repo to benchmark Lambda runtime execution across different layers.
+
+Tests:
+* [AWS OpenTelemetry Collector layer](https://github.com/open-telemetry/opentelemetry-lambda)
+* [Rotel collector layer](https://github.com/streamfold/rotel-lambda-extension)
+
+## Usage
+
+Use Github to invoke a workflow to run the tests.

SimpleLambda.py

@@ -0,0 +1,64 @@
+import boto3
+import json
+import os
+
+from opentelemetry import trace
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource, DEPLOYMENT_ENVIRONMENT
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+# Set up the OpenTelemetry tracer provider with a resource name.
+resource = Resource(attributes={
+    SERVICE_NAME: "python-lambda-example",
+    DEPLOYMENT_ENVIRONMENT: "dev",
+})
+provider = TracerProvider(resource=resource)
+
+span_processor = BatchSpanProcessor(OTLPSpanExporter(endpoint="http://localhost:4318/v1/traces"))
+provider.add_span_processor(span_processor)
+
+trace.set_tracer_provider(provider)
+
+tracer = trace.get_tracer("python-lambda.tracer")
+
+def echo(payload):
+    return payload
+
+def list_buckets(payload):
+    client = boto3.client("s3")
+    buckets = client.list_buckets()
+    bucket_names = list(map(lambda b: b['Name'], buckets['Buckets']))
+
+    return {
+        "statusCode": 200,
+        "headers": {
+            "Content-Type": "application/json"
+        },
+        "body": json.dumps({
+            "Buckets": bucket_names,
+            "Region ": os.environ['AWS_REGION']
+        })
+    }
+
+operations = {
+    'echo': echo,
+    'list_buckets': list_buckets,
+}
+
+def lambda_handler(event, context):
+    '''Provide an event that contains the following keys:
+      - operation: one of the operations in the operations dict below
+      - payload: a JSON object containing parameters to pass to the
+        operation being performed
+    '''
+
+    with tracer.start_as_current_span(f"op-{event['operation']}") as span:
+        operation = event['operation']
+        payload = event['payload']
+
+        if operation in operations:
+            return operations[operation](payload)
+        else:
+            raise ValueError(f'Unrecognized operation "{operation}"')
+

benchmark/.python-version

@@ -0,0 +1 @@
+3.13

benchmark/README.md

@@ -0,0 +1,3 @@
+# benchmark
+
+Python benchmark to run a function multiple times and measure the coldstart time.

benchmark/main.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+import argparse
+from operator import contains
+
+import boto3
+import json
+import time
+import os
+import uuid
+from botocore.exceptions import ClientError
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Deploy a function as AWS Lambda multiple times and benchmark')
+    parser.add_argument('--path', required=True, help='Path to the function.zip file')
+    parser.add_argument('--count', type=int, default=10, help='Number of functions to deploy (default: 10)')
+    parser.add_argument('--function-name', default='test-function', help='Base name for the Lambda functions')
+    parser.add_argument('--role-arn', required=True, help='ARN of the IAM role for Lambda execution')
+    parser.add_argument('--region', default='us-east-1', help='AWS region (default: us-east-1)')
+    parser.add_argument('--handler', default='SimpleLambda.lambda_handler', help='Lambda function handler')
+    parser.add_argument('--runtime', default='python3.13', help='Lambda runtime (default: python3.13)')
+    parser.add_argument('--environment', help='Environment')
+    parser.add_argument('--layer', help="ARN of layer to include")
+    return parser.parse_args()
+
+def delete_lambda_function(lambda_client, function_name):
+    try:
+        lambda_client.delete_function(FunctionName=function_name)
+        print(f"Deleted Lambda function: {function_name}")
+    except ClientError as e:
+        if e.response['Error']['Code'] == 'ResourceNotFoundException':
+            print(f"Lambda function not found: {function_name}")
+        else:
+            raise
+
+def create_lambda_function(lambda_client, function_name, environment, zip_path, role_arn, handler, runtime, layer):
+    try:
+        with open(zip_path, 'rb') as zip_file:
+            zip_content = zip_file.read()
+
+        response = lambda_client.create_function(
+            FunctionName=function_name,
+            Environment={
+                'Variables': environment,
+            },
+            Runtime=runtime,
+            Role=role_arn,
+            Handler=handler,
+            Code={'ZipFile': zip_content},
+            Timeout=10,
+            MemorySize=128,
+            Layers=[layer] if layer else [],
+        )
+        print(f"Created Lambda function: {function_name}")
+        # Wait for function to be fully initialized
+        time.sleep(5)
+        return response['FunctionArn']
+    except ClientError as e:
+        print(f"Error creating Lambda function {function_name}: {e}")
+        return None
+
+def invoke_lambda_function(lambda_client, function_name, payload):
+    try:
+        start_time = time.time()
+        response = lambda_client.invoke(
+            FunctionName=function_name,
+            InvocationType='RequestResponse',
+            LogType='Tail',
+            Payload=json.dumps(payload)
+        )
+        duration = (time.time() - start_time) * 1000  # Convert to milliseconds
+
+        # Get the Lambda execution duration from the response headers
+        response_payload = json.loads(response['Payload'].read().decode('utf-8'))
+        status_code = response['StatusCode']
+        execution_log = None
+        if 'LogResult' in response:
+            import base64
+            execution_log = base64.b64decode(response['LogResult']).decode('utf-8')
+
+        # Try to extract the actual Lambda execution duration from logs
+        lambda_init_duration = None
+        if execution_log:
+            for line in execution_log.split('\n'):
+                if 'Init Duration:' in line:
+                    if 'Extension.Crash' in line:
+                        print(f"Exception crashed, full execution log: {execution_log}")
+                        raise Exception("Lambda extension has crashed")
+                    try:
+                        duration_part = line.split('Init Duration:')[1].split('ms')[0].strip()
+                        lambda_init_duration = float(duration_part)
+                        break
+                    except (IndexError, ValueError):
+                        pass
+        if not lambda_init_duration:
+            raise Exception("Could not extract the actual Lambda init duration from logs")
+
+        print(f"Invoked {function_name}:")
+        print(f"  HTTP Status: {status_code}")
+        print(f"  Client-side duration: {duration:.2f} ms")
+
+        if lambda_init_duration:
+            print(f"  Lambda-reported init duration: {lambda_init_duration:.2f} ms")
+
+        return {
+            'function_name': function_name,
+            'status_code': status_code,
+            'client_duration_ms': duration,
+            'init_duration_ms': lambda_init_duration,
+            'response': response_payload
+        }
+    except ClientError as e:
+        print(f"Error invoking Lambda function {function_name}: {e}")
+        return None
+
+def main():
+    args = parse_args()
+
+    environment = {}
+    if args.environment:
+        for val in args.environment.split(','):
+            key, value = val.split('=')
+            environment[key] = value
+
+    # Validate the ZIP file exists
+    if not os.path.isfile(args.path):
+        print(f"Error: ZIP file not found at {args.path}")
+        return
+
+    # Create AWS clients
+    lambda_client = boto3.client('lambda', region_name=args.region)
+
+    # Deploy multiple Lambda functions
+    functions = []
+    results = []
+
+    print(f"Deploying {args.count} Lambda functions...")
+    for i in range(1, args.count + 1):
+        function_name = f"{args.function_name}-{i}"
+        # Try to delete it first
+        delete_lambda_function(lambda_client, function_name)
+        function_arn = create_lambda_function(
+            lambda_client,
+            function_name,
+            environment,
+            args.path,
+            args.role_arn,
+            args.handler,
+            args.runtime,
+            args.layer,
+        )
+        if function_arn:
+            functions.append(function_name)
+
+    print(f"\nSuccessfully deployed {len(functions)} Lambda functions")
+
+    # Invoke each function with a simple payload
+    test_payload = {
+        'operation': "list_buckets",
+        'payload': {
+            'dog': "boxer",
+            'cat': "siamese",
+            'timestamp': int(time.time()),
+        }
+    }
+
+    print("\nInvoking each function and recording durations...")
+    for function_name in functions:
+        result = invoke_lambda_function(lambda_client, function_name, test_payload)
+        if result:
+            results.append(result)
+
+    # Write results to file
+    output_file = 'lambda_benchmark_results.json'
+    with open(output_file, 'w') as f:
+        json.dump(results, f, indent=2)
+
+    print(f"\nResults have been saved to {output_file}")
+
+    # Print summary
+    print(f"\nSummary of lambda init durations (cold start) for {args.function_name}:")
+
+    sorted_results = sorted(results, key=lambda x: x['init_duration_ms'])
+    sorted_durations = list(map(lambda x: str(x['init_duration_ms']), sorted_results))
+    print(f"Durations: {", ".join(sorted_durations)}")
+
+if __name__ == "__main__":
+    main()

benchmark/pyproject.toml

@@ -0,0 +1,9 @@
+[project]
+name = "benchmark"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "boto3"
+]