Bun memory usage is high when uploading ReadableStream using S3 client #25375
Description
What version of Bun is running?
1.3.3+274e01c73
What platform is your computer?
Linux 6.12.42 x86_64 x86_64
What steps can reproduce the bug?
I have an app that fetches certain files and uploads them to S3 storage. For some reason Bun memory usage is pretty high compared to node, and in my app could grow far beyond the size of the file.
Here's a simple repro (you need to append S3 credentials to any storage of your choice):
import { env } from 'node:process'
import { S3Client } from '@aws-sdk/client-s3'
import { Upload } from '@aws-sdk/lib-storage'
const config = {
endpoint: env.S3_ENDPOINT! || '',
region: env.S3_REGION! || '',
accessKeyId: env.S3_ACCESS_KEY! || '',
secretAccessKey: env.S3_SECRET_KEY! || '',
bucket: env.S3_BUCKET! || '',
}
function getMemoryUsageInMB() {
const memory = process.memoryUsage()
return {
rss: Number.parseFloat((memory.rss / 1024 / 1024).toFixed(2)),
heapTotal: Number.parseFloat((memory.heapTotal / 1024 / 1024).toFixed(2)),
heapUsed: Number.parseFloat((memory.heapUsed / 1024 / 1024).toFixed(2)),
external: Number.parseFloat((memory.external / 1024 / 1024).toFixed(2)),
arrayBuffers: Number.parseFloat((memory.arrayBuffers / 1024 / 1024).toFixed(2)),
}
}
const s3Client = new S3Client({
endpoint: config.endpoint,
region: config.region,
credentials: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
},
})
const resp = await fetch('https://hel1-speed.hetzner.com/1GB.bin')
if (!resp.body) {
throw new Error('Body is null')
}
const upload = new Upload({
client: s3Client,
params: {
Bucket: config.bucket,
Key: '1GB.bin',
Body: resp.body,
},
})
upload.on('httpUploadProgress', (progress) => {
console.log(`loaded: ${progress.loaded / 1024 / 1024} MB, part: ${progress.part}, memoryUsage: `, getMemoryUsageInMB())
})
await upload.done()
console.log('upload completed')When using bun:
loaded: 5 MB, part: 2, memoryUsage: {
rss: 280.48,
heapTotal: 6.01,
heapUsed: 140.95,
external: 136.88,
arrayBuffers: 133.1,
}
loaded: 10 MB, part: 1, memoryUsage: {
rss: 386.39,
heapTotal: 6.03,
heapUsed: 198.55,
external: 194.49,
arrayBuffers: 190.19,
}
loaded: 15 MB, part: 3, memoryUsage: {
rss: 394.17,
heapTotal: 6.05,
heapUsed: 198.55,
external: 194.5,
arrayBuffers: 190.19,
}
loaded: 20 MB, part: 4, memoryUsage: {
rss: 403.63,
heapTotal: 6.07,
heapUsed: 198.67,
external: 194.53,
arrayBuffers: 190.19,
}
loaded: 25 MB, part: 5, memoryUsage: {
rss: 410.88,
heapTotal: 6.04,
heapUsed: 198.7,
external: 194.54,
arrayBuffers: 190.2,
}
loaded: 30 MB, part: 6, memoryUsage: {
rss: 434.23,
heapTotal: 6.05,
heapUsed: 198.74,
external: 194.55,
arrayBuffers: 190.2,
}
loaded: 35 MB, part: 7, memoryUsage: {
rss: 387.94,
heapTotal: 6.08,
heapUsed: 198.78,
external: 194.57,
arrayBuffers: 190.2,
}
loaded: 40 MB, part: 8, memoryUsage: {
rss: 401.44,
heapTotal: 6.1,
heapUsed: 198.82,
external: 194.58,
arrayBuffers: 190.2,
}
loaded: 45 MB, part: 9, memoryUsage: {
rss: 421.09,
heapTotal: 6.1,
heapUsed: 198.86,
external: 194.59,
arrayBuffers: 190.21,
}
loaded: 50 MB, part: 10, memoryUsage: {
rss: 456.93,
heapTotal: 6.15,
heapUsed: 198.9,
external: 194.61,
arrayBuffers: 190.21,
}
loaded: 55 MB, part: 13, memoryUsage: {
rss: 469.16,
heapTotal: 6.16,
heapUsed: 198.94,
external: 194.62,
arrayBuffers: 190.21,
}
loaded: 60 MB, part: 11, memoryUsage: {
rss: 469.64,
heapTotal: 6.16,
heapUsed: 198.97,
external: 194.63,
arrayBuffers: 190.21,
}
loaded: 65 MB, part: 12, memoryUsage: {
rss: 493.02,
heapTotal: 6.16,
heapUsed: 199.01,
external: 194.64,
arrayBuffers: 190.22,
}
loaded: 70 MB, part: 15, memoryUsage: {
rss: 523.77,
heapTotal: 6.22,
heapUsed: 199.04,
external: 194.64,
arrayBuffers: 190.22,
}
loaded: 75 MB, part: 14, memoryUsage: {
rss: 527.67,
heapTotal: 6.24,
heapUsed: 199.07,
external: 194.65,
arrayBuffers: 190.22,
}
loaded: 80 MB, part: 16, memoryUsage: {
rss: 559.67,
heapTotal: 6.24,
heapUsed: 199.1,
external: 194.66,
arrayBuffers: 190.22,
}
loaded: 85 MB, part: 17, memoryUsage: {
rss: 575.02,
heapTotal: 6.24,
heapUsed: 199.13,
external: 194.66,
arrayBuffers: 190.23,
}
loaded: 90 MB, part: 18, memoryUsage: {
rss: 581.65,
heapTotal: 6.29,
heapUsed: 199.17,
external: 194.67,
arrayBuffers: 190.23,
}
loaded: 95 MB, part: 19, memoryUsage: {
rss: 514.39,
heapTotal: 6.25,
heapUsed: 105.28,
external: 101.18,
arrayBuffers: 92.85,
}
loaded: 100 MB, part: 20, memoryUsage: {
rss: 542.64,
heapTotal: 6.1,
heapUsed: 115.75,
external: 111.62,
arrayBuffers: 98.27,
}
loaded: 105 MB, part: 21, memoryUsage: {
rss: 575.38,
heapTotal: 6.12,
heapUsed: 126.21,
external: 122.05,
arrayBuffers: 103.69,
}
loaded: 110 MB, part: 22, memoryUsage: {
rss: 591.13,
heapTotal: 6.12,
heapUsed: 136.67,
external: 132.48,
arrayBuffers: 109.12,
}
loaded: 115 MB, part: 23, memoryUsage: {
rss: 612.16,
heapTotal: 6.12,
heapUsed: 147.12,
external: 142.91,
arrayBuffers: 114.54,
}
loaded: 120 MB, part: 24, memoryUsage: {
rss: 668.13,
heapTotal: 5.81,
heapUsed: 36.24,
external: 32.13,
arrayBuffers: 21.69,
}
loaded: 125 MB, part: 26, memoryUsage: {
rss: 669.05,
heapTotal: 5.82,
heapUsed: 46.7,
external: 42.56,
arrayBuffers: 27.11,
}
loaded: 130 MB, part: 27, memoryUsage: {
rss: 587.76,
heapTotal: 5.82,
heapUsed: 57.17,
external: 53.01,
arrayBuffers: 32.54,
}
loaded: 135 MB, part: 25, memoryUsage: {
rss: 600.72,
heapTotal: 5.84,
heapUsed: 67.63,
external: 63.44,
arrayBuffers: 37.96,
}
loaded: 140 MB, part: 29, memoryUsage: {
rss: 634.66,
heapTotal: 5.86,
heapUsed: 34.48,
external: 30.65,
arrayBuffers: 21.69,
}
loaded: 145 MB, part: 28, memoryUsage: {
rss: 657.65,
heapTotal: 5.85,
heapUsed: 45.57,
external: 41.64,
arrayBuffers: 27.11,
}
loaded: 150 MB, part: 31, memoryUsage: {
rss: 663.25,
heapTotal: 5.87,
heapUsed: 56.03,
external: 52.07,
arrayBuffers: 32.54,
}
loaded: 155 MB, part: 30, memoryUsage: {
rss: 675.65,
heapTotal: 5.92,
heapUsed: 56.03,
external: 57.5,
arrayBuffers: 37.96,
}
loaded: 160 MB, part: 32, memoryUsage: {
rss: 703.65,
heapTotal: 5.99,
heapUsed: 35.14,
external: 31.3,
arrayBuffers: 21.69,
}
loaded: 165 MB, part: 33, memoryUsage: {
rss: 729.34,
heapTotal: 5.97,
heapUsed: 45.75,
external: 41.86,
arrayBuffers: 27.11,
}
loaded: 170 MB, part: 34, memoryUsage: {
rss: 746.13,
heapTotal: 5.98,
heapUsed: 45.75,
external: 47.28,
arrayBuffers: 32.54,
}
loaded: 175 MB, part: 36, memoryUsage: {
rss: 761.63,
heapTotal: 6.03,
heapUsed: 45.75,
external: 52.71,
arrayBuffers: 37.96,
}
loaded: 180 MB, part: 35, memoryUsage: {
rss: 1407.04,
heapTotal: 6.08,
heapUsed: 35.44,
external: 31.58,
arrayBuffers: 21.69,
}
loaded: 185 MB, part: 38, memoryUsage: {
rss: 1422.41,
heapTotal: 6.09,
heapUsed: 35.44,
external: 37,
arrayBuffers: 27.11,
}
loaded: 190 MB, part: 39, memoryUsage: {
rss: 1455.65,
heapTotal: 6.09,
heapUsed: 51.85,
external: 47.93,
arrayBuffers: 32.54,
}
loaded: 195 MB, part: 37, memoryUsage: {
rss: 1495.9,
heapTotal: 6.09,
heapUsed: 62.31,
external: 58.37,
arrayBuffers: 37.96,
}
loaded: 200 MB, part: 40, memoryUsage: {
rss: 1522.73,
heapTotal: 6.08,
heapUsed: 35.45,
external: 31.58,
arrayBuffers: 21.69,
}
loaded: 205 MB, part: 41, memoryUsage: {
rss: 1527.36,
heapTotal: 6.08,
heapUsed: 45.91,
external: 42.02,
arrayBuffers: 27.11,
}
loaded: 210 MB, part: 42, memoryUsage: {
rss: 1539.21,
heapTotal: 5.91,
heapUsed: 56.37,
external: 52.45,
arrayBuffers: 32.54,
}
loaded: 215 MB, part: 45, memoryUsage: {
rss: 951.09,
heapTotal: 5.85,
heapUsed: 35.48,
external: 31.61,
arrayBuffers: 21.69,
}
loaded: 220 MB, part: 43, memoryUsage: {
rss: 966.42,
heapTotal: 5.85,
heapUsed: 45.93,
external: 42.04,
arrayBuffers: 27.11,
}
loaded: 225 MB, part: 44, memoryUsage: {
rss: 968.53,
heapTotal: 5.85,
heapUsed: 56.39,
external: 52.47,
arrayBuffers: 32.54,
}
loaded: 230 MB, part: 47, memoryUsage: {
rss: 1004.94,
heapTotal: 5.87,
heapUsed: 66.86,
external: 62.91,
arrayBuffers: 37.96,
}
loaded: 235 MB, part: 46, memoryUsage: {
rss: 1030.66,
heapTotal: 5.88,
heapUsed: 35.54,
external: 31.67,
arrayBuffers: 21.69,
}
loaded: 240 MB, part: 49, memoryUsage: {
rss: 1041.79,
heapTotal: 5.88,
heapUsed: 46,
external: 42.1,
arrayBuffers: 27.11,
}
loaded: 245 MB, part: 50, memoryUsage: {
rss: 1057.29,
heapTotal: 5.88,
heapUsed: 56.46,
external: 52.53,
arrayBuffers: 32.54,
}
loaded: 250 MB, part: 48, memoryUsage: {
rss: 1072.07,
heapTotal: 5.93,
heapUsed: 56.46,
external: 57.96,
arrayBuffers: 37.96,
}
loaded: 255 MB, part: 51, memoryUsage: {
rss: 1079.39,
heapTotal: 5.99,
heapUsed: 35.57,
external: 31.69,
arrayBuffers: 21.69,
}
loaded: 260 MB, part: 53, memoryUsage: {
rss: 1081.33,
heapTotal: 5.99,
heapUsed: 46.03,
external: 42.12,
arrayBuffers: 27.11,
}
loaded: 265 MB, part: 54, memoryUsage: {
rss: 1079.5,
heapTotal: 5.99,
heapUsed: 56.48,
external: 52.55,
arrayBuffers: 32.54,
}
loaded: 270 MB, part: 52, memoryUsage: {
rss: 1085.01,
heapTotal: 6.01,
heapUsed: 66.94,
external: 62.98,
arrayBuffers: 37.96,
}
loaded: 275 MB, part: 56, memoryUsage: {
rss: 1094.98,
heapTotal: 6,
heapUsed: 77.4,
external: 31.72,
arrayBuffers: 21.69,
}
loaded: 280 MB, part: 55, memoryUsage: {
rss: 1099.84,
heapTotal: 5.89,
heapUsed: 46.06,
external: 42.15,
arrayBuffers: 27.11,
}
When using node it's fine:
loaded: 5 MB, part: 1, memoryUsage: {
rss: 182.34,
heapTotal: 34.82,
heapUsed: 17.25,
external: 34.61,
arrayBuffers: 25.23
}
loaded: 10 MB, part: 4, memoryUsage: {
rss: 183.39,
heapTotal: 34.82,
heapUsed: 17.37,
external: 34.66,
arrayBuffers: 25.28
}
loaded: 15 MB, part: 3, memoryUsage: {
rss: 183.52,
heapTotal: 34.82,
heapUsed: 20.15,
external: 49.88,
arrayBuffers: 40.44
}
loaded: 20 MB, part: 2, memoryUsage: {
rss: 179.79,
heapTotal: 32.25,
heapUsed: 12.31,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 25 MB, part: 6, memoryUsage: {
rss: 179.33,
heapTotal: 31.55,
heapUsed: 12.35,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 30 MB, part: 5, memoryUsage: {
rss: 180.2,
heapTotal: 31.55,
heapUsed: 12.41,
external: 34.72,
arrayBuffers: 25.28
}
loaded: 35 MB, part: 7, memoryUsage: {
rss: 180.38,
heapTotal: 31.55,
heapUsed: 12.47,
external: 34.67,
arrayBuffers: 25.23
}
loaded: 40 MB, part: 8, memoryUsage: {
rss: 179.88,
heapTotal: 31.55,
heapUsed: 12.55,
external: 34.68,
arrayBuffers: 25.24
}
loaded: 45 MB, part: 9, memoryUsage: {
rss: 180.62,
heapTotal: 31.55,
heapUsed: 12.64,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 50 MB, part: 10, memoryUsage: {
rss: 181,
heapTotal: 31.55,
heapUsed: 13.12,
external: 40.45,
arrayBuffers: 31.01
}
loaded: 55 MB, part: 11, memoryUsage: {
rss: 181,
heapTotal: 31.8,
heapUsed: 12.66,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 60 MB, part: 13, memoryUsage: {
rss: 179.94,
heapTotal: 31.8,
heapUsed: 13.16,
external: 40.43,
arrayBuffers: 30.99
}
loaded: 65 MB, part: 12, memoryUsage: {
rss: 180.76,
heapTotal: 31.8,
heapUsed: 15.46,
external: 55.53,
arrayBuffers: 46.08
}
loaded: 70 MB, part: 14, memoryUsage: {
rss: 181.01,
heapTotal: 32.05,
heapUsed: 12.66,
external: 34.67,
arrayBuffers: 25.23
}
loaded: 75 MB, part: 15, memoryUsage: {
rss: 181.01,
heapTotal: 31.8,
heapUsed: 12.78,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 80 MB, part: 16, memoryUsage: {
rss: 180.73,
heapTotal: 31.8,
heapUsed: 12.83,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 85 MB, part: 17, memoryUsage: {
rss: 180.25,
heapTotal: 32.05,
heapUsed: 12.7,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 90 MB, part: 18, memoryUsage: {
rss: 181.46,
heapTotal: 32.05,
heapUsed: 12.68,
external: 34.67,
arrayBuffers: 25.23
}
loaded: 95 MB, part: 19, memoryUsage: {
rss: 181.46,
heapTotal: 32.05,
heapUsed: 12.71,
external: 34.68,
arrayBuffers: 25.24
}
loaded: 100 MB, part: 20, memoryUsage: {
rss: 180.67,
heapTotal: 32.05,
heapUsed: 12.67,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 105 MB, part: 22, memoryUsage: {
rss: 180.58,
heapTotal: 32.05,
heapUsed: 12.72,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 110 MB, part: 21, memoryUsage: {
rss: 181.39,
heapTotal: 32.05,
heapUsed: 12.68,
external: 34.67,
arrayBuffers: 25.23
}
loaded: 115 MB, part: 24, memoryUsage: {
rss: 180.51,
heapTotal: 32.05,
heapUsed: 12.7,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 120 MB, part: 23, memoryUsage: {
rss: 180.7,
heapTotal: 32.05,
heapUsed: 13.71,
external: 39.49,
arrayBuffers: 30.05
}
loaded: 125 MB, part: 26, memoryUsage: {
rss: 180.39,
heapTotal: 32.05,
heapUsed: 12.7,
external: 34.67,
arrayBuffers: 25.23
}
loaded: 130 MB, part: 25, memoryUsage: {
rss: 181.56,
heapTotal: 32.3,
heapUsed: 12.72,
external: 34.67,
arrayBuffers: 25.23
}
loaded: 135 MB, part: 27, memoryUsage: {
rss: 181.56,
heapTotal: 31.8,
heapUsed: 12.69,
external: 34.67,
arrayBuffers: 25.24
}
loaded: 140 MB, part: 28, memoryUsage: {
rss: 181.56,
heapTotal: 31.8,
heapUsed: 12.81,
external: 34.7,
arrayBuffers: 25.27
}
loaded: 145 MB, part: 29, memoryUsage: {
rss: 181.68,
heapTotal: 31.8,
heapUsed: 12.73,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 150 MB, part: 31, memoryUsage: {
rss: 181.68,
heapTotal: 32.05,
heapUsed: 12.73,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 155 MB, part: 32, memoryUsage: {
rss: 181.68,
heapTotal: 32.3,
heapUsed: 12.75,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 160 MB, part: 30, memoryUsage: {
rss: 181.68,
heapTotal: 32.3,
heapUsed: 13.78,
external: 40.18,
arrayBuffers: 30.73
}
loaded: 165 MB, part: 33, memoryUsage: {
rss: 181.68,
heapTotal: 31.8,
heapUsed: 12.77,
external: 34.68,
arrayBuffers: 25.24
}
loaded: 170 MB, part: 34, memoryUsage: {
rss: 181.68,
heapTotal: 32.05,
heapUsed: 12.77,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 175 MB, part: 35, memoryUsage: {
rss: 180.71,
heapTotal: 32.05,
heapUsed: 13.2,
external: 40.32,
arrayBuffers: 30.87
}
loaded: 180 MB, part: 36, memoryUsage: {
rss: 181.69,
heapTotal: 32.55,
heapUsed: 12.77,
external: 34.67,
arrayBuffers: 25.23
}
loaded: 185 MB, part: 38, memoryUsage: {
rss: 181.69,
heapTotal: 31.8,
heapUsed: 12.92,
external: 34.71,
arrayBuffers: 25.26
}
loaded: 190 MB, part: 39, memoryUsage: {
rss: 180.76,
heapTotal: 31.8,
heapUsed: 12.78,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 195 MB, part: 37, memoryUsage: {
rss: 180.94,
heapTotal: 31.8,
heapUsed: 14.14,
external: 42.36,
arrayBuffers: 32.92
}
loaded: 200 MB, part: 40, memoryUsage: {
rss: 181.68,
heapTotal: 32.05,
heapUsed: 12.92,
external: 34.68,
arrayBuffers: 25.25
}
loaded: 205 MB, part: 41, memoryUsage: {
rss: 180.79,
heapTotal: 32.3,
heapUsed: 12.81,
external: 34.68,
arrayBuffers: 25.24
}
loaded: 210 MB, part: 44, memoryUsage: {
rss: 181.08,
heapTotal: 32.05,
heapUsed: 12.98,
external: 34.69,
arrayBuffers: 25.24
}
loaded: 215 MB, part: 43, memoryUsage: {
rss: 181.21,
heapTotal: 32.05,
heapUsed: 14.98,
external: 49.79,
arrayBuffers: 40.34
}
loaded: 220 MB, part: 42, memoryUsage: {
rss: 181.77,
heapTotal: 32.55,
heapUsed: 12.78,
external: 34.67,
arrayBuffers: 25.23
}
loaded: 225 MB, part: 45, memoryUsage: {
rss: 181.77,
heapTotal: 32.05,
heapUsed: 12.78,
external: 34.67,
arrayBuffers: 25.24
}
loaded: 230 MB, part: 47, memoryUsage: {
rss: 181.82,
heapTotal: 32.05,
heapUsed: 12.95,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 235 MB, part: 46, memoryUsage: {
rss: 181.82,
heapTotal: 32.3,
heapUsed: 12.79,
external: 34.72,
arrayBuffers: 25.28
}
loaded: 240 MB, part: 48, memoryUsage: {
rss: 181.82,
heapTotal: 31.8,
heapUsed: 12.81,
external: 34.68,
arrayBuffers: 25.24
}
loaded: 245 MB, part: 49, memoryUsage: {
rss: 181.89,
heapTotal: 48.3,
heapUsed: 12.82,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 250 MB, part: 51, memoryUsage: {
rss: 182.02,
heapTotal: 48.05,
heapUsed: 12.82,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 255 MB, part: 50, memoryUsage: {
rss: 182.15,
heapTotal: 48.05,
heapUsed: 12.81,
external: 34.67,
arrayBuffers: 25.23
}
loaded: 260 MB, part: 53, memoryUsage: {
rss: 182.27,
heapTotal: 48.05,
heapUsed: 12.82,
external: 34.68,
arrayBuffers: 25.23
}
loaded: 265 MB, part: 52, memoryUsage: {
rss: 182.27,
heapTotal: 48.05,
heapUsed: 12.8,
external: 34.67,
arrayBuffers: 25.23
}
loaded: 270 MB, part: 54, memoryUsage: {
rss: 181.19,
heapTotal: 48.05,
heapUsed: 12.83,
external: 34.68,
arrayBuffers: 25.24
}
loaded: 275 MB, part: 55, memoryUsage: {
rss: 181.18,
heapTotal: 48.05,
heapUsed: 12.8,
external: 34.67,
arrayBuffers: 25.23
}
loaded: 280 MB, part: 56, memoryUsage: {
rss: 181.93,
heapTotal: 48.05,
heapUsed: 12.84,
external: 34.68,
arrayBuffers: 25.24
}
Node version:
$ node --version
v24.11.1
What is the expected behavior?
Memory usage doesn't grow as much and stays at node level
What do you see instead?
Memory usage grows far beyond expected levels. Didn't test that with Bun native S3 client though.
Additional information
No response