From 70a8c3a2fa80d659b638deb1c3ec205e3608e7c9 Mon Sep 17 00:00:00 2001 From: Vanssh-k Date: Tue, 15 Jul 2025 22:47:07 +0530 Subject: [PATCH 1/8] Added direct file data stream in upload endpoint --- src/Lighthouse/upload/files/node.ts | 92 +++++++++++------------ src/Lighthouse/utils/util.ts | 109 ++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 50 deletions(-) diff --git a/src/Lighthouse/upload/files/node.ts b/src/Lighthouse/upload/files/node.ts index 3dfc731..4ddaae6 100644 --- a/src/Lighthouse/upload/files/node.ts +++ b/src/Lighthouse/upload/files/node.ts @@ -1,6 +1,6 @@ import basePathConvert from '../../utils/basePathConvert' import { lighthouseConfig } from '../../../lighthouse.config' -import { fetchWithTimeout } from '../../utils/util' +import { fetchWithDirectStream } from '../../utils/util' import { IFileUploadedResponse } from '../../../types' export async function walk(dir: string) { const { readdir, stat } = eval(`require`)('fs-extra') @@ -31,72 +31,64 @@ export default async ( const token = 'Bearer ' + apiKey const stats = lstatSync(sourcePath) + try { const endpoint = lighthouseConfig.lighthouseNode + `/api/v0/add?wrap-with-directory=false&cid-version=${cidVersion}` + const boundary = + '----WebKitFormBoundary' + Math.random().toString(16).substr(2) + + const headers = { + Authorization: token, + } + if (stats.isFile()) { - const data = new FormData() const stream = createReadStream(sourcePath) - const buffers: Buffer[] = [] - for await (const chunk of stream) { - buffers.push(chunk) + const streamData = { + boundary, + files: [ + { + stream, + filename: path.basename(sourcePath), + }, + ], } - const blob = new Blob(buffers) - - data.append('file', blob, path.basename(sourcePath)) - const response = await fetchWithTimeout(endpoint, { - method: 'POST', - body: data, - timeout: 7200000, - headers: { - Authorization: token + const response = await fetchWithDirectStream( + endpoint, + { + method: 'POST', + headers, + timeout: 7200000, }, - }) + streamData + ) - if (!response.ok) { - const res = (await response.json()) - throw new Error(res.error) - } - - const responseData = (await response.json()) - return { data: responseData } + return response } else { + // Handle directory upload const files = await walk(sourcePath) - const data = new FormData() - - for (const file of files) { - const stream = createReadStream(file) - const buffers: Buffer[] = [] - for await (const chunk of stream) { - buffers.push(chunk) - } - const blob = new Blob(buffers) - data.append( - 'file', - blob, - basePathConvert(sourcePath, file) - ) + const streamData = { + boundary, + files: files.map((file) => ({ + stream: createReadStream(file), + filename: basePathConvert(sourcePath, file), + })), } - const response = await fetchWithTimeout(endpoint, { - method: 'POST', - body: data, - timeout: 7200000, - headers: { - Authorization: token + const response = await fetchWithDirectStream( + endpoint, + { + method: 'POST', + headers, + timeout: 7200000, }, - }) - - if (!response.ok) { - const res = (await response.json()) - throw new Error(res.error) - } + streamData + ) - const responseData = (await response.json()) - return { data: responseData } + return response } } catch (error: any) { throw new Error(error) diff --git a/src/Lighthouse/utils/util.ts b/src/Lighthouse/utils/util.ts index 0b39a35..3e8d750 100644 --- a/src/Lighthouse/utils/util.ts +++ b/src/Lighthouse/utils/util.ts @@ -5,6 +5,13 @@ interface FetchOptions extends RequestInit { onProgress?: (progress: number) => void } +interface DirectStreamOptions { + method?: string + headers?: Record + timeout?: number + onProgress?: (progress: number) => void +} + const isCID = (cid: string) => { return /^(Qm[1-9A-HJ-NP-Za-km-z]{44}|b[A-Za-z2-7]{58}|B[A-Z2-7]{58}|z[1-9A-HJ-NP-Za-km-z]{48}|F[0-9A-F]{50})*$/.test( cid @@ -127,10 +134,112 @@ async function fetchWithTimeout( } } +async function fetchWithDirectStream( + endpointURL: string, + options: DirectStreamOptions, + streamData: { + boundary: string + files: Array<{ + stream: any + filename: string + }> + } +): Promise<{ data: any }> { + const { method = 'POST', headers = {}, timeout = 7200000 } = options + + const http = eval(`require`)('http') + const https = eval(`require`)('https') + const url = eval(`require`)('url') + + const parsedUrl = url.parse(endpointURL) + const isHttps = parsedUrl.protocol === 'https:' + const client = isHttps ? https : http + + return new Promise((resolve, reject) => { + const requestOptions = { + hostname: parsedUrl.hostname, + port: parsedUrl.port || (isHttps ? 443 : 80), + path: parsedUrl.path, + method, + headers: { + ...headers, + 'Content-Type': `multipart/form-data; boundary=${streamData.boundary}`, + }, + } + + const req = client.request(requestOptions, (res: any) => { + let data = '' + res.on('data', (chunk: any) => { + data += chunk + }) + res.on('end', () => { + if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) { + try { + const responseData = JSON.parse(data) + resolve({ data: responseData }) + } catch (error) { + reject(new Error('Invalid JSON response')) + } + } else { + reject(new Error(`Request failed with status code ${res.statusCode}`)) + } + }) + }) + + req.on('error', (error: any) => { + reject(new Error(error.message)) + }) + + // Handle timeout + const timeoutId = setTimeout(() => { + req.destroy() + reject(new Error('Request timed out')) + }, timeout) + + req.on('close', () => { + clearTimeout(timeoutId) + }) + + // Stream files + let fileIndex = 0 + + const processNextFile = () => { + if (fileIndex >= streamData.files.length) { + req.write(`\r\n--${streamData.boundary}--\r\n`) + req.end() + return + } + + const file = streamData.files[fileIndex] + const stream = file.stream + + req.write(`--${streamData.boundary}\r\n`) + req.write( + `Content-Disposition: form-data; name="file"; filename="${file.filename}"\r\n` + ) + req.write(`Content-Type: application/octet-stream\r\n\r\n`) + + stream.pipe(req, { end: false }) + + stream.on('end', () => { + fileIndex++ + processNextFile() + }) + + stream.on('error', (error: any) => { + reject(new Error(`File stream error: ${error.message}`)) + }) + } + + processNextFile() + }) +} + export { isCID, isPrivateKey, addressValidator, checkDuplicateFileNames, fetchWithTimeout, + fetchWithDirectStream, } From 43b429f1410845597d17bb173a0208f126c64250 Mon Sep 17 00:00:00 2001 From: Aakash Taneja Date: Wed, 23 Jul 2025 15:21:11 +0530 Subject: [PATCH 2/8] removed console errors --- .../uploadEncrypted/encryptionBrowser.ts | 102 ++++++-------- .../uploadEncrypted/encryptionNode.ts | 130 ++++++++---------- 2 files changed, 104 insertions(+), 128 deletions(-) diff --git a/src/Lighthouse/uploadEncrypted/encryptionBrowser.ts b/src/Lighthouse/uploadEncrypted/encryptionBrowser.ts index 97d9afa..d830c3b 100644 --- a/src/Lighthouse/uploadEncrypted/encryptionBrowser.ts +++ b/src/Lighthouse/uploadEncrypted/encryptionBrowser.ts @@ -19,72 +19,60 @@ const deriveKey = async ( ) export const encryptFile = async (fileArrayBuffer: any, password: any) => { - try { - const plainTextBytes = new Uint8Array(fileArrayBuffer) - const passwordBytes = new TextEncoder().encode(password) + const plainTextBytes = new Uint8Array(fileArrayBuffer) + const passwordBytes = new TextEncoder().encode(password) - const salt = window.crypto.getRandomValues(new Uint8Array(16)) - const iv = window.crypto.getRandomValues(new Uint8Array(12)) + const salt = window.crypto.getRandomValues(new Uint8Array(16)) + const iv = window.crypto.getRandomValues(new Uint8Array(12)) - const passwordKey = await importKeyFromBytes(passwordBytes) + const passwordKey = await importKeyFromBytes(passwordBytes) - const aesKey = await deriveKey(passwordKey, ['encrypt'], { - name: 'PBKDF2', - salt: salt, - iterations: 250000, - hash: 'SHA-256', - }) - const cipherBytes = await window.crypto.subtle.encrypt( - { name: 'AES-GCM', iv: iv }, - aesKey, - plainTextBytes - ) + const aesKey = await deriveKey(passwordKey, ['encrypt'], { + name: 'PBKDF2', + salt: salt, + iterations: 250000, + hash: 'SHA-256', + }) + const cipherBytes = await window.crypto.subtle.encrypt( + { name: 'AES-GCM', iv: iv }, + aesKey, + plainTextBytes + ) - const cipherBytesArray = new Uint8Array(cipherBytes) - const resultBytes = new Uint8Array( - cipherBytesArray.byteLength + salt.byteLength + iv.byteLength - ) - resultBytes.set(salt, 0) - resultBytes.set(iv, salt.byteLength) - resultBytes.set(cipherBytesArray, salt.byteLength + iv.byteLength) + const cipherBytesArray = new Uint8Array(cipherBytes) + const resultBytes = new Uint8Array( + cipherBytesArray.byteLength + salt.byteLength + iv.byteLength + ) + resultBytes.set(salt, 0) + resultBytes.set(iv, salt.byteLength) + resultBytes.set(cipherBytesArray, salt.byteLength + iv.byteLength) - return resultBytes - } catch (error) { - console.error('Error encrypting file') - console.error(error) - throw error - } + return resultBytes } export const decryptFile = async (cipher: any, password: any) => { - try { - const cipherBytes = new Uint8Array(cipher) - const passwordBytes = new TextEncoder().encode(password) + const cipherBytes = new Uint8Array(cipher) + const passwordBytes = new TextEncoder().encode(password) - const salt = cipherBytes.slice(0, 16) - const iv = cipherBytes.slice(16, 16 + 12) - const data = cipherBytes.slice(16 + 12) - const passwordKey = await importKeyFromBytes(passwordBytes) - const aesKey = await deriveKey(passwordKey, ['decrypt'], { - name: 'PBKDF2', - salt: salt, - iterations: 250000, - hash: 'SHA-256', - }) + const salt = cipherBytes.slice(0, 16) + const iv = cipherBytes.slice(16, 16 + 12) + const data = cipherBytes.slice(16 + 12) + const passwordKey = await importKeyFromBytes(passwordBytes) + const aesKey = await deriveKey(passwordKey, ['decrypt'], { + name: 'PBKDF2', + salt: salt, + iterations: 250000, + hash: 'SHA-256', + }) - const decryptedContent = await window.crypto.subtle.decrypt( - { - name: 'AES-GCM', - iv: iv, - }, - aesKey, - data - ) + const decryptedContent = await window.crypto.subtle.decrypt( + { + name: 'AES-GCM', + iv: iv, + }, + aesKey, + data + ) - return decryptedContent - } catch (error) { - console.error('Error decrypting file') - console.error(error) - return - } + return decryptedContent } diff --git a/src/Lighthouse/uploadEncrypted/encryptionNode.ts b/src/Lighthouse/uploadEncrypted/encryptionNode.ts index c90aab2..674960e 100644 --- a/src/Lighthouse/uploadEncrypted/encryptionNode.ts +++ b/src/Lighthouse/uploadEncrypted/encryptionNode.ts @@ -16,90 +16,78 @@ const deriveKey = async ( ) const encryptFile = async (fileArrayBuffer: any, password: any) => { - try { - const { Crypto } = eval('require')('@peculiar/webcrypto') - const crypto = new Crypto() + const { Crypto } = eval('require')('@peculiar/webcrypto') + const crypto = new Crypto() - const plainTextBytes = new Uint8Array(fileArrayBuffer) - const passwordBytes = new TextEncoder().encode(password) + const plainTextBytes = new Uint8Array(fileArrayBuffer) + const passwordBytes = new TextEncoder().encode(password) - const salt = crypto.getRandomValues(new Uint8Array(16)) - const iv = crypto.getRandomValues(new Uint8Array(12)) + const salt = crypto.getRandomValues(new Uint8Array(16)) + const iv = crypto.getRandomValues(new Uint8Array(12)) - const passwordKey = await importKeyFromBytes(passwordBytes, crypto) + const passwordKey = await importKeyFromBytes(passwordBytes, crypto) - const aesKey = await deriveKey( - passwordKey, - ['encrypt'], - { - name: 'PBKDF2', - salt: salt, - iterations: 250000, - hash: 'SHA-256', - }, - crypto - ) - const cipherBytes = await crypto.subtle.encrypt( - { name: 'AES-GCM', iv: iv }, - aesKey, - plainTextBytes - ) + const aesKey = await deriveKey( + passwordKey, + ['encrypt'], + { + name: 'PBKDF2', + salt: salt, + iterations: 250000, + hash: 'SHA-256', + }, + crypto + ) + const cipherBytes = await crypto.subtle.encrypt( + { name: 'AES-GCM', iv: iv }, + aesKey, + plainTextBytes + ) - const cipherBytesArray = new Uint8Array(cipherBytes) - const resultBytes = new Uint8Array( - cipherBytesArray.byteLength + salt.byteLength + iv.byteLength - ) - resultBytes.set(salt, 0) - resultBytes.set(iv, salt.byteLength) - resultBytes.set(cipherBytesArray, salt.byteLength + iv.byteLength) + const cipherBytesArray = new Uint8Array(cipherBytes) + const resultBytes = new Uint8Array( + cipherBytesArray.byteLength + salt.byteLength + iv.byteLength + ) + resultBytes.set(salt, 0) + resultBytes.set(iv, salt.byteLength) + resultBytes.set(cipherBytesArray, salt.byteLength + iv.byteLength) - return resultBytes - } catch (error) { - console.error('Error encrypting file') - console.error(error) - throw error - } + return resultBytes } const decryptFile = async (cipher: any, password: any) => { - try { - const { Crypto } = eval('require')('@peculiar/webcrypto') - const crypto = new Crypto() + const { Crypto } = eval('require')('@peculiar/webcrypto') + const crypto = new Crypto() - const cipherBytes = new Uint8Array(cipher) - const passwordBytes = new TextEncoder().encode(password) + const cipherBytes = new Uint8Array(cipher) + const passwordBytes = new TextEncoder().encode(password) - const salt = cipherBytes.slice(0, 16) - const iv = cipherBytes.slice(16, 16 + 12) - const data = cipherBytes.slice(16 + 12) - const passwordKey = await importKeyFromBytes(passwordBytes, crypto) - const aesKey = await deriveKey( - passwordKey, - ['decrypt'], - { - name: 'PBKDF2', - salt: salt, - iterations: 250000, - hash: 'SHA-256', - }, - crypto - ) + const salt = cipherBytes.slice(0, 16) + const iv = cipherBytes.slice(16, 16 + 12) + const data = cipherBytes.slice(16 + 12) + const passwordKey = await importKeyFromBytes(passwordBytes, crypto) + const aesKey = await deriveKey( + passwordKey, + ['decrypt'], + { + name: 'PBKDF2', + salt: salt, + iterations: 250000, + hash: 'SHA-256', + }, + crypto + ) - const decryptedContent = await crypto.subtle.decrypt( - { - name: 'AES-GCM', - iv: iv, - }, - aesKey, - data - ) + const decryptedContent = await crypto.subtle.decrypt( + { + name: 'AES-GCM', + iv: iv, + }, + aesKey, + data + ) - return decryptedContent - } catch (error) { - console.error('Error decrypting file') - console.error(error) - return - } + return decryptedContent } export { encryptFile, decryptFile } From ceae60b644b09409201b21c439a9419d49279893 Mon Sep 17 00:00:00 2001 From: Aakash Taneja Date: Wed, 23 Jul 2025 18:29:13 +0530 Subject: [PATCH 3/8] added proper error handling in applyaccesscondition Fixes #130 --- src/Lighthouse/encryption/applyAccessCondition.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Lighthouse/encryption/applyAccessCondition.ts b/src/Lighthouse/encryption/applyAccessCondition.ts index 75d94e7..ee828df 100644 --- a/src/Lighthouse/encryption/applyAccessCondition.ts +++ b/src/Lighthouse/encryption/applyAccessCondition.ts @@ -26,8 +26,14 @@ export default async ( chainType ) - if (error) { - throw error + if (!isSuccess || error) { + const errorMessage = + typeof error === 'string' + ? error + : error instanceof Error + ? error.message + : JSON.stringify(error) + throw new Error(errorMessage) } return { data: { cid: cid, status: 'Success' } } } From f8fdce1a5d434707be56482d5cf82090a3112d17 Mon Sep 17 00:00:00 2001 From: Vanssh-k Date: Sat, 9 Aug 2025 02:55:31 +0530 Subject: [PATCH 4/8] Fixed the streaming entruption bug --- src/Lighthouse/upload/files/node.ts | 6 +- src/Lighthouse/utils/util.ts | 85 ++++++++++++++++++++--------- 2 files changed, 62 insertions(+), 29 deletions(-) diff --git a/src/Lighthouse/upload/files/node.ts b/src/Lighthouse/upload/files/node.ts index 4ddaae6..c45f771 100644 --- a/src/Lighthouse/upload/files/node.ts +++ b/src/Lighthouse/upload/files/node.ts @@ -70,13 +70,15 @@ export default async ( // Handle directory upload const files = await walk(sourcePath) - const streamData = { + const createStreamData = () => ({ boundary, files: files.map((file) => ({ stream: createReadStream(file), filename: basePathConvert(sourcePath, file), })), - } + }) + + const streamData = createStreamData() const response = await fetchWithDirectStream( endpoint, diff --git a/src/Lighthouse/utils/util.ts b/src/Lighthouse/utils/util.ts index 3e8d750..f1c308f 100644 --- a/src/Lighthouse/utils/util.ts +++ b/src/Lighthouse/utils/util.ts @@ -200,38 +200,69 @@ async function fetchWithDirectStream( clearTimeout(timeoutId) }) - // Stream files - let fileIndex = 0 - - const processNextFile = () => { - if (fileIndex >= streamData.files.length) { - req.write(`\r\n--${streamData.boundary}--\r\n`) - req.end() - return - } - - const file = streamData.files[fileIndex] - const stream = file.stream - - req.write(`--${streamData.boundary}\r\n`) - req.write( - `Content-Disposition: form-data; name="file"; filename="${file.filename}"\r\n` - ) - req.write(`Content-Type: application/octet-stream\r\n\r\n`) - - stream.pipe(req, { end: false }) - - stream.on('end', () => { - fileIndex++ - processNextFile() + // Stream files sequentially with backpressure handling and proper part delimiters + const writeAsync = (data: string | Buffer): Promise => { + return new Promise((resolve) => { + const canWrite = req.write(data) + if (canWrite) { + resolve() + } else { + req.once('drain', () => resolve()) + } }) + } - stream.on('error', (error: any) => { - reject(new Error(`File stream error: ${error.message}`)) + const pumpStream = (stream: any): Promise => { + return new Promise((resolve, rejectPump) => { + const onData = (chunk: any) => { + const canWrite = req.write(chunk) + if (!canWrite) { + stream.pause() + req.once('drain', () => stream.resume()) + } + } + const onEnd = () => { + cleanup() + resolve() + } + const onError = (err: any) => { + cleanup() + rejectPump(new Error(`File stream error: ${err?.message || err}`)) + } + const cleanup = () => { + stream.off('data', onData) + stream.off('end', onEnd) + stream.off('error', onError) + } + stream.on('data', onData) + stream.on('end', onEnd) + stream.on('error', onError) }) } - processNextFile() + ;(async () => { + try { + for (let idx = 0; idx < streamData.files.length; idx++) { + const file = streamData.files[idx] + const headersPart = + `--${streamData.boundary}\r\n` + + `Content-Disposition: form-data; name="file"; filename="${file.filename}"\r\n` + + `Content-Type: application/octet-stream\r\n\r\n` + + await writeAsync(headersPart) + await pumpStream(file.stream) + await writeAsync(`\r\n`) + } + + await writeAsync(`--${streamData.boundary}--\r\n`) + req.end() + } catch (err: any) { + if (req && !req.destroyed) { + req.destroy() + } + reject(new Error(err?.message || String(err))) + } + })() }) } From dab6a6e03e7a6b506813e5c7d8bd62a4a40b1831 Mon Sep 17 00:00:00 2001 From: Vanssh-k Date: Tue, 12 Aug 2025 01:16:30 +0530 Subject: [PATCH 5/8] Added progress bar on package upload function --- src/Lighthouse/upload/files/index.ts | 2 +- src/Lighthouse/upload/files/node.ts | 24 +++++++++++++++------ src/Lighthouse/utils/util.ts | 31 ++++++++++++++++++++++++++-- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/src/Lighthouse/upload/files/index.ts b/src/Lighthouse/upload/files/index.ts index d55de93..82bc6e4 100644 --- a/src/Lighthouse/upload/files/index.ts +++ b/src/Lighthouse/upload/files/index.ts @@ -14,7 +14,7 @@ async function uploadFiles( // Upload File to IPFS //@ts-ignore if (typeof window === 'undefined') { - return await uploadFile(path, apiKey, cidVersion) + return await uploadFile(path, apiKey, cidVersion, uploadProgressCallback) } else { return await uploadFileBrowser( path, diff --git a/src/Lighthouse/upload/files/node.ts b/src/Lighthouse/upload/files/node.ts index c45f771..80bc1f0 100644 --- a/src/Lighthouse/upload/files/node.ts +++ b/src/Lighthouse/upload/files/node.ts @@ -1,7 +1,7 @@ import basePathConvert from '../../utils/basePathConvert' import { lighthouseConfig } from '../../../lighthouse.config' import { fetchWithDirectStream } from '../../utils/util' -import { IFileUploadedResponse } from '../../../types' +import { IFileUploadedResponse, IUploadProgressCallback } from '../../../types' export async function walk(dir: string) { const { readdir, stat } = eval(`require`)('fs-extra') let results: string[] = [] @@ -24,7 +24,8 @@ export async function walk(dir: string) { export default async ( sourcePath: string, apiKey: string, - cidVersion: number + cidVersion: number, + uploadProgressCallback?: (data: IUploadProgressCallback) => void ): Promise<{ data: IFileUploadedResponse }> => { const { createReadStream, lstatSync } = eval(`require`)('fs-extra') const path = eval(`require`)('path') @@ -51,6 +52,7 @@ export default async ( { stream, filename: path.basename(sourcePath), + size: stats.size, }, ], } @@ -61,6 +63,9 @@ export default async ( method: 'POST', headers, timeout: 7200000, + onProgress: uploadProgressCallback + ? (data: { progress: number }) => uploadProgressCallback(data) + : undefined, }, streamData ) @@ -72,10 +77,14 @@ export default async ( const createStreamData = () => ({ boundary, - files: files.map((file) => ({ - stream: createReadStream(file), - filename: basePathConvert(sourcePath, file), - })), + files: files.map((file) => { + const fileStats = lstatSync(file) + return { + stream: createReadStream(file), + filename: basePathConvert(sourcePath, file), + size: fileStats.size, + } + }), }) const streamData = createStreamData() @@ -86,6 +95,9 @@ export default async ( method: 'POST', headers, timeout: 7200000, + onProgress: uploadProgressCallback + ? (data: { progress: number }) => uploadProgressCallback(data) + : undefined, }, streamData ) diff --git a/src/Lighthouse/utils/util.ts b/src/Lighthouse/utils/util.ts index f1c308f..8aae099 100644 --- a/src/Lighthouse/utils/util.ts +++ b/src/Lighthouse/utils/util.ts @@ -9,7 +9,7 @@ interface DirectStreamOptions { method?: string headers?: Record timeout?: number - onProgress?: (progress: number) => void + onProgress?: (data: { progress: number }) => void } const isCID = (cid: string) => { @@ -142,10 +142,16 @@ async function fetchWithDirectStream( files: Array<{ stream: any filename: string + size: number }> } ): Promise<{ data: any }> { - const { method = 'POST', headers = {}, timeout = 7200000 } = options + const { + method = 'POST', + headers = {}, + timeout = 7200000, + onProgress, + } = options const http = eval(`require`)('http') const https = eval(`require`)('https') @@ -200,6 +206,17 @@ async function fetchWithDirectStream( clearTimeout(timeoutId) }) + // Track total bytes for progress calculation + let totalBytesUploaded = 0 + let totalBytesToUpload = 0 + + // Calculate total size for progress tracking + if (onProgress) { + for (const file of streamData.files) { + totalBytesToUpload += file.size + } + } + // Stream files sequentially with backpressure handling and proper part delimiters const writeAsync = (data: string | Buffer): Promise => { return new Promise((resolve) => { @@ -215,6 +232,16 @@ async function fetchWithDirectStream( const pumpStream = (stream: any): Promise => { return new Promise((resolve, rejectPump) => { const onData = (chunk: any) => { + // Update progress if callback is provided + if (onProgress && totalBytesToUpload > 0) { + totalBytesUploaded += chunk.length + const progress = Math.min( + (totalBytesUploaded / totalBytesToUpload) * 100, + 100 + ) + onProgress({ progress }) + } + const canWrite = req.write(chunk) if (!canWrite) { stream.pause() From bba9fa4350482ea7569e44993386cfd903570441 Mon Sep 17 00:00:00 2001 From: ravish1729 Date: Wed, 13 Aug 2025 18:55:01 +0530 Subject: [PATCH 6/8] version --- README.md | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1c0aee2..e60ebbf 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Lighthouse +# Lighthouse Lighthouse is a permanent decentralized file storage protocol that allows the ability to pay once and store forever. While traditionally, users need to repeatedly keep track and pay for their storage after every fixed amount of time, Lighthouse manages this for them and makes sure that user files are stored forever. The aim is to move users from a rent-based cost model where they are renting their own files on cloud storage to a permanent ownership model. It is built on top of IPFS, Filecoin, and Polygon. It uses the existing miner network and storage capacity of the filecoin network. diff --git a/package-lock.json b/package-lock.json index 39d1d53..f51e238 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lighthouse-web3/sdk", - "version": "0.4.0", + "version": "0.4.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lighthouse-web3/sdk", - "version": "0.4.0", + "version": "0.4.1", "license": "MIT", "dependencies": { "@lighthouse-web3/kavach": "^0.1.9", diff --git a/package.json b/package.json index fa7870a..1a28d5a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@lighthouse-web3/sdk", - "version": "0.4.0", + "version": "0.4.1", "description": "NPM package and CLI tool to interact with lighthouse protocol", "main": "./dist/Lighthouse/index.js", "types": "./dist/Lighthouse/index.d.ts", From 88cab7e0315112e92ef8b0d678676c86b1439cff Mon Sep 17 00:00:00 2001 From: ravish1729 Date: Wed, 13 Aug 2025 18:55:08 +0530 Subject: [PATCH 7/8] version --- src/Commands/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Commands/index.ts b/src/Commands/index.ts index 40014e0..34acd0a 100644 --- a/src/Commands/index.ts +++ b/src/Commands/index.ts @@ -71,7 +71,7 @@ Command.prototype.helpInformation = function (context: any) { } widgets.addHelpText('before', 'Welcome to lighthouse-web3') -widgets.version('0.4.0') +widgets.version('0.4.1') widgets .command('wallet') From 47ed424ad8cbe114b364212603b4c722c02a12db Mon Sep 17 00:00:00 2001 From: ravish1729 Date: Wed, 13 Aug 2025 18:56:29 +0530 Subject: [PATCH 8/8] regular update --- src/Lighthouse/tests/encryption.test.ts | 6 +----- src/Lighthouse/tests/upload.test.ts | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/Lighthouse/tests/encryption.test.ts b/src/Lighthouse/tests/encryption.test.ts index 9f3660b..27dae8a 100644 --- a/src/Lighthouse/tests/encryption.test.ts +++ b/src/Lighthouse/tests/encryption.test.ts @@ -13,7 +13,6 @@ const signAuthMessage = async (privateKey: string) => { describe('encryption', () => { describe('getAuthMessage', () => { it('should get auth message when valid public key is provided', async () => { - console.log('testing getAuthMessage') const response = await lighthouse.getAuthMessage( '0x1Ec09D4B3Cb565b7CCe2eEAf71CC90c9b46c5c26' ) @@ -24,7 +23,6 @@ describe('encryption', () => { it('should not get auth message when invalid public key is provided', async () => { try { - console.log('testing getAuthMessage with invalid public key') const response = await lighthouse.getAuthMessage('invalidPublicKey') } catch (error) { expect(error.message).toBe('Invalid public Key') @@ -38,7 +36,6 @@ describe('encryption', () => { const cid = 'QmVkHgHnYVUfvTXsaJisHRgc89zsrgVL6ATh9mSiegRYrX' it('should fetch encryption key when correct public-private key pair is provided', async () => { - console.log('testing fetchEncryptionKey') const signed_message = await signAuthMessage(privateKey) const response = await lighthouse.fetchEncryptionKey( cid, @@ -50,7 +47,6 @@ describe('encryption', () => { it('should not fetch encryption key when incorrect public-private key pair is provided', async () => { try { - console.log('testing fetchEncryptionKey with incorrect key pair') const randomPublicKey = '0x1ccEF158Dcbe6643F1cC577F236af79993F4D066' const signed_message = await signAuthMessage(privateKey) const response = await lighthouse.fetchEncryptionKey( @@ -122,7 +118,7 @@ describe('encryption', () => { signed_message ) } catch (error) { - expect(error.message.message.message).toEqual('access denied') + expect(error.message[0].message.message).toEqual('access denied') } }, 10000) }) diff --git a/src/Lighthouse/tests/upload.test.ts b/src/Lighthouse/tests/upload.test.ts index 4ac22f5..caeef03 100644 --- a/src/Lighthouse/tests/upload.test.ts +++ b/src/Lighthouse/tests/upload.test.ts @@ -53,7 +53,7 @@ describe('uploadFiles', () => { ) await lighthouse.upload(path, 'random apiKey') } catch (error) { - expect(error.message).toBe('Error: Authentication failed') + expect(error.message).toBe('Error: Request failed with status code 401') } }, 60000) })