Test at different memory sizes

Author: mheffner

README.md

@@ -1,6 +1,6 @@
 # python-lambda-benchmark
 
-Small repo to benchmark Lambda runtime execution across different layers.
+Small repo to benchmark Lambda runtime execution across different layers and memory sizes.
 
 Tests:
 * [AWS OpenTelemetry Collector layer](https://github.com/open-telemetry/opentelemetry-lambda)

benchmark/main.py

@@ -18,21 +18,22 @@ def parse_args():
     parser.add_argument('--region', default='us-east-1', help='AWS region (default: us-east-1)')
     parser.add_argument('--handler', default='SimpleLambda.lambda_handler', help='Lambda function handler')
     parser.add_argument('--runtime', default='python3.13', help='Lambda runtime (default: python3.13)')
+    parser.add_argument('--memory', type=int, default=128, help='Memory in MB')
     parser.add_argument('--environment', help='Environment')
     parser.add_argument('--layer', help="ARN of layer to include")
+    parser.add_argument('--output', required=True, help="Output file")
     return parser.parse_args()
 
 def delete_lambda_function(lambda_client, function_name):
     try:
         lambda_client.delete_function(FunctionName=function_name)
-        print(f"Deleted Lambda function: {function_name}")
     except ClientError as e:
         if e.response['Error']['Code'] == 'ResourceNotFoundException':
             print(f"Lambda function not found: {function_name}")
         else:
             raise
 
-def create_lambda_function(lambda_client, function_name, environment, zip_path, role_arn, handler, runtime, layer):
+def create_lambda_function(lambda_client, function_name, environment, zip_path, role_arn, handler, runtime, memory, layer):
     try:
         with open(zip_path, 'rb') as zip_file:
             zip_content = zip_file.read()
@@ -47,18 +48,15 @@ def create_lambda_function(lambda_client, function_name, environment, zip_path,
             Handler=handler,
             Code={'ZipFile': zip_content},
             Timeout=10,
-            MemorySize=128,
+            MemorySize=memory,
             Layers=[layer] if layer else [],
         )
-        print(f"Created Lambda function: {function_name}")
-        # Wait for function to be fully initialized
-        time.sleep(5)
         return response['FunctionArn']
     except ClientError as e:
         print(f"Error creating Lambda function {function_name}: {e}")
         return None
 
-def invoke_lambda_function(lambda_client, function_name, payload):
+def invoke_lambda_function(lambda_client, base_name, function_name, memory, payload):
     try:
         start_time = time.time()
         response = lambda_client.invoke(
@@ -94,15 +92,12 @@ def invoke_lambda_function(lambda_client, function_name, payload):
         if not lambda_init_duration:
             raise Exception("Could not extract the actual Lambda init duration from logs")
 
-        print(f"Invoked {function_name}:")
-        print(f"  HTTP Status: {status_code}")
-        print(f"  Client-side duration: {duration:.2f} ms")
-
-        if lambda_init_duration:
-            print(f"  Lambda-reported init duration: {lambda_init_duration:.2f} ms")
+        print(f"Invoked {function_name}: Status: {status_code}, Duration: {duration:.2f} ms, Init Duration: {lambda_init_duration:.2f} ms")
 
         return {
+            'base_name': base_name,
             'function_name': function_name,
+            'memory': memory,
             'status_code': status_code,
             'client_duration_ms': duration,
             'init_duration_ms': lambda_init_duration,
@@ -135,7 +130,7 @@ def main():
 
     print(f"Deploying {args.count} Lambda functions...")
     for i in range(1, args.count + 1):
-        function_name = f"{args.function_name}-{i}"
+        function_name = f"{args.function_name}-{args.memory}-{i}"
         # Try to delete it first
         delete_lambda_function(lambda_client, function_name)
         function_arn = create_lambda_function(
@@ -146,12 +141,15 @@ def main():
             args.role_arn,
             args.handler,
             args.runtime,
+            args.memory,
             args.layer,
         )
         if function_arn:
             functions.append(function_name)
 
-    print(f"\nSuccessfully deployed {len(functions)} Lambda functions")
+    # Wait for all functions to be fully initialized
+    time.sleep(5)
+    print(f"Successfully deployed {len(functions)} Lambda functions")
 
     # Invoke each function with a simple payload
     test_payload = {
@@ -163,25 +161,25 @@ def main():
         }
     }
 
-    print("\nInvoking each function and recording durations...")
     for function_name in functions:
-        result = invoke_lambda_function(lambda_client, function_name, test_payload)
+        result = invoke_lambda_function(lambda_client, args.function_name, function_name, args.memory, test_payload)
         if result:
             results.append(result)
 
-    # Write results to file
-    output_file = 'lambda_benchmark_results.json'
-    with open(output_file, 'w') as f:
-        json.dump(results, f, indent=2)
-
-    print(f"\nResults have been saved to {output_file}")
-
     # Print summary
     print(f"\nSummary of lambda init durations (cold start) for {args.function_name}:")
 
     sorted_results = sorted(results, key=lambda x: x['init_duration_ms'])
     sorted_durations = list(map(lambda x: str(x['init_duration_ms']), sorted_results))
-    print(f"Durations: {", ".join(sorted_durations)}")
+    print(f"Init Durations: {", ".join(sorted_durations)}")
+
+    with open(args.output, 'a') as f:
+        f.write("base_name, memory, client_duration_ms, init_duration_ms\n")
+        for result in sorted_results:
+            f.write(f"{result['base_name']}, {result['memory']}, {result['client_duration_ms']}, {result['init_duration_ms']}\n")
+
+    for function_name in functions:
+        delete_lambda_function(lambda_client, function_name)
 
 if __name__ == "__main__":
     main()

scripts/benchmark-coldstart.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
-COUNT=${COUNT:-10}
+COUNT=${COUNT:-5}
+
+MEMORY_SIZES="128 1024 2048 3072 4096 5120"
 
 if [ $# -ne 1 ]; then
   echo "Usage: $0 <output-file>"
@@ -32,6 +34,7 @@ fi
 
 cat <<EOF
 Running benchmark with the following:
+ - Memory size: $MEMORY_SIZES
  - AWS Region: $AWS_REGION
  - OTEL Layer: $OTEL_LAYER
  - Rotel Layer: $ROTEL_LAYER
@@ -43,19 +46,29 @@ make bundle
 
 cd benchmark
 
-echo "Testing base case"
+for MEMORY_SIZE in ${MEMORY_SIZES}; do
+  echo "Testing base case at $MEMORY_SIZE MB"
+
+  uv run main.py --path ../function.zip --count $COUNT --function-name benchmark-coldstart \
+    --role-arn "$AWS_ROLE_ARN" --region "$AWS_REGION" --output "$OUTPUT" \
+    --memory "$MEMORY_SIZE"
+done
 
-uv run main.py --path ../function.zip --count $COUNT --function-name benchmark-coldstart \
-  --role-arn "$AWS_ROLE_ARN" --region "$AWS_REGION" --output "$OUTPUT"
 
-echo "Testing OpenTelemetry collector layer"
+for MEMORY_SIZE in ${MEMORY_SIZES}; do
+  echo "Testing OpenTelemetry collector layer at $MEMORY_SIZE MB"
 
-uv run main.py --path ../function.zip --count $COUNT --function-name benchmark-coldstart-otel \
-  --environment OPENTELEMETRY_COLLECTOR_CONFIG_URI=/var/task/collector.yaml \
-  --role-arn "$AWS_ROLE_ARN" --region "$AWS_REGION" --layer "$OTEL_LAYER" --output "$OUTPUT"
+  uv run main.py --path ../function.zip --count $COUNT --function-name benchmark-coldstart-otel \
+    --environment OPENTELEMETRY_COLLECTOR_CONFIG_URI=/var/task/collector.yaml \
+    --role-arn "$AWS_ROLE_ARN" --region "$AWS_REGION" --layer "$OTEL_LAYER" --output "$OUTPUT" \
+    --memory "$MEMORY_SIZE"
+done
 
-echo "Testing Rotel layer"
+for MEMORY_SIZE in ${MEMORY_SIZES}; do
+  echo "Testing Rotel layer at $MEMORY_SIZE MB"
 
-uv run main.py --path ../function.zip --count $COUNT --function-name benchmark-coldstart-rotel \
-  --environment ROTEL_ENV_FILE=/var/task/rotel.env \
-  --role-arn "$AWS_ROLE_ARN" --region "$AWS_REGION" --layer "$ROTEL_LAYER" --output "$OUTPUT"
+  uv run main.py --path ../function.zip --count $COUNT --function-name benchmark-coldstart-rotel \
+    --environment ROTEL_ENV_FILE=/var/task/rotel.env \
+    --role-arn "$AWS_ROLE_ARN" --region "$AWS_REGION" --layer "$ROTEL_LAYER" --output "$OUTPUT" \
+    --memory "$MEMORY_SIZE"
+done