diff --git a/README.md b/README.md index 1c0aee26..e60ebbfe 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Lighthouse +# Lighthouse Lighthouse is a permanent decentralized file storage protocol that allows the ability to pay once and store forever. While traditionally, users need to repeatedly keep track and pay for their storage after every fixed amount of time, Lighthouse manages this for them and makes sure that user files are stored forever. The aim is to move users from a rent-based cost model where they are renting their own files on cloud storage to a permanent ownership model. It is built on top of IPFS, Filecoin, and Polygon. It uses the existing miner network and storage capacity of the filecoin network. diff --git a/package-lock.json b/package-lock.json index 39d1d533..f51e2383 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lighthouse-web3/sdk", - "version": "0.4.0", + "version": "0.4.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lighthouse-web3/sdk", - "version": "0.4.0", + "version": "0.4.1", "license": "MIT", "dependencies": { "@lighthouse-web3/kavach": "^0.1.9", diff --git a/package.json b/package.json index fa7870a9..1a28d5a5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@lighthouse-web3/sdk", - "version": "0.4.0", + "version": "0.4.1", "description": "NPM package and CLI tool to interact with lighthouse protocol", "main": "./dist/Lighthouse/index.js", "types": "./dist/Lighthouse/index.d.ts", diff --git a/src/Commands/index.ts b/src/Commands/index.ts index 40014e00..34acd0a0 100644 --- a/src/Commands/index.ts +++ b/src/Commands/index.ts @@ -71,7 +71,7 @@ Command.prototype.helpInformation = function (context: any) { } widgets.addHelpText('before', 'Welcome to lighthouse-web3') -widgets.version('0.4.0') +widgets.version('0.4.1') widgets .command('wallet') diff --git a/src/Lighthouse/encryption/applyAccessCondition.ts b/src/Lighthouse/encryption/applyAccessCondition.ts index 75d94e75..ee828df3 100644 --- a/src/Lighthouse/encryption/applyAccessCondition.ts +++ b/src/Lighthouse/encryption/applyAccessCondition.ts @@ -26,8 +26,14 @@ export default async ( chainType ) - if (error) { - throw error + if (!isSuccess || error) { + const errorMessage = + typeof error === 'string' + ? error + : error instanceof Error + ? error.message + : JSON.stringify(error) + throw new Error(errorMessage) } return { data: { cid: cid, status: 'Success' } } } diff --git a/src/Lighthouse/tests/encryption.test.ts b/src/Lighthouse/tests/encryption.test.ts index 9f3660b5..27dae8af 100644 --- a/src/Lighthouse/tests/encryption.test.ts +++ b/src/Lighthouse/tests/encryption.test.ts @@ -13,7 +13,6 @@ const signAuthMessage = async (privateKey: string) => { describe('encryption', () => { describe('getAuthMessage', () => { it('should get auth message when valid public key is provided', async () => { - console.log('testing getAuthMessage') const response = await lighthouse.getAuthMessage( '0x1Ec09D4B3Cb565b7CCe2eEAf71CC90c9b46c5c26' ) @@ -24,7 +23,6 @@ describe('encryption', () => { it('should not get auth message when invalid public key is provided', async () => { try { - console.log('testing getAuthMessage with invalid public key') const response = await lighthouse.getAuthMessage('invalidPublicKey') } catch (error) { expect(error.message).toBe('Invalid public Key') @@ -38,7 +36,6 @@ describe('encryption', () => { const cid = 'QmVkHgHnYVUfvTXsaJisHRgc89zsrgVL6ATh9mSiegRYrX' it('should fetch encryption key when correct public-private key pair is provided', async () => { - console.log('testing fetchEncryptionKey') const signed_message = await signAuthMessage(privateKey) const response = await lighthouse.fetchEncryptionKey( cid, @@ -50,7 +47,6 @@ describe('encryption', () => { it('should not fetch encryption key when incorrect public-private key pair is provided', async () => { try { - console.log('testing fetchEncryptionKey with incorrect key pair') const randomPublicKey = '0x1ccEF158Dcbe6643F1cC577F236af79993F4D066' const signed_message = await signAuthMessage(privateKey) const response = await lighthouse.fetchEncryptionKey( @@ -122,7 +118,7 @@ describe('encryption', () => { signed_message ) } catch (error) { - expect(error.message.message.message).toEqual('access denied') + expect(error.message[0].message.message).toEqual('access denied') } }, 10000) }) diff --git a/src/Lighthouse/tests/upload.test.ts b/src/Lighthouse/tests/upload.test.ts index 4ac22f5a..caeef03e 100644 --- a/src/Lighthouse/tests/upload.test.ts +++ b/src/Lighthouse/tests/upload.test.ts @@ -53,7 +53,7 @@ describe('uploadFiles', () => { ) await lighthouse.upload(path, 'random apiKey') } catch (error) { - expect(error.message).toBe('Error: Authentication failed') + expect(error.message).toBe('Error: Request failed with status code 401') } }, 60000) }) diff --git a/src/Lighthouse/upload/files/index.ts b/src/Lighthouse/upload/files/index.ts index d55de937..82bc6e45 100644 --- a/src/Lighthouse/upload/files/index.ts +++ b/src/Lighthouse/upload/files/index.ts @@ -14,7 +14,7 @@ async function uploadFiles( // Upload File to IPFS //@ts-ignore if (typeof window === 'undefined') { - return await uploadFile(path, apiKey, cidVersion) + return await uploadFile(path, apiKey, cidVersion, uploadProgressCallback) } else { return await uploadFileBrowser( path, diff --git a/src/Lighthouse/upload/files/node.ts b/src/Lighthouse/upload/files/node.ts index 3dfc7319..80bc1f02 100644 --- a/src/Lighthouse/upload/files/node.ts +++ b/src/Lighthouse/upload/files/node.ts @@ -1,7 +1,7 @@ import basePathConvert from '../../utils/basePathConvert' import { lighthouseConfig } from '../../../lighthouse.config' -import { fetchWithTimeout } from '../../utils/util' -import { IFileUploadedResponse } from '../../../types' +import { fetchWithDirectStream } from '../../utils/util' +import { IFileUploadedResponse, IUploadProgressCallback } from '../../../types' export async function walk(dir: string) { const { readdir, stat } = eval(`require`)('fs-extra') let results: string[] = [] @@ -24,79 +24,85 @@ export async function walk(dir: string) { export default async ( sourcePath: string, apiKey: string, - cidVersion: number + cidVersion: number, + uploadProgressCallback?: (data: IUploadProgressCallback) => void ): Promise<{ data: IFileUploadedResponse }> => { const { createReadStream, lstatSync } = eval(`require`)('fs-extra') const path = eval(`require`)('path') const token = 'Bearer ' + apiKey const stats = lstatSync(sourcePath) + try { const endpoint = lighthouseConfig.lighthouseNode + `/api/v0/add?wrap-with-directory=false&cid-version=${cidVersion}` + const boundary = + '----WebKitFormBoundary' + Math.random().toString(16).substr(2) + + const headers = { + Authorization: token, + } + if (stats.isFile()) { - const data = new FormData() const stream = createReadStream(sourcePath) - const buffers: Buffer[] = [] - for await (const chunk of stream) { - buffers.push(chunk) + const streamData = { + boundary, + files: [ + { + stream, + filename: path.basename(sourcePath), + size: stats.size, + }, + ], } - const blob = new Blob(buffers) - data.append('file', blob, path.basename(sourcePath)) - - const response = await fetchWithTimeout(endpoint, { - method: 'POST', - body: data, - timeout: 7200000, - headers: { - Authorization: token + const response = await fetchWithDirectStream( + endpoint, + { + method: 'POST', + headers, + timeout: 7200000, + onProgress: uploadProgressCallback + ? (data: { progress: number }) => uploadProgressCallback(data) + : undefined, }, - }) + streamData + ) - if (!response.ok) { - const res = (await response.json()) - throw new Error(res.error) - } - - const responseData = (await response.json()) - return { data: responseData } + return response } else { + // Handle directory upload const files = await walk(sourcePath) - const data = new FormData() - for (const file of files) { - const stream = createReadStream(file) - const buffers: Buffer[] = [] - for await (const chunk of stream) { - buffers.push(chunk) - } - const blob = new Blob(buffers) + const createStreamData = () => ({ + boundary, + files: files.map((file) => { + const fileStats = lstatSync(file) + return { + stream: createReadStream(file), + filename: basePathConvert(sourcePath, file), + size: fileStats.size, + } + }), + }) - data.append( - 'file', - blob, - basePathConvert(sourcePath, file) - ) - } + const streamData = createStreamData() - const response = await fetchWithTimeout(endpoint, { - method: 'POST', - body: data, - timeout: 7200000, - headers: { - Authorization: token + const response = await fetchWithDirectStream( + endpoint, + { + method: 'POST', + headers, + timeout: 7200000, + onProgress: uploadProgressCallback + ? (data: { progress: number }) => uploadProgressCallback(data) + : undefined, }, - }) - - if (!response.ok) { - const res = (await response.json()) - throw new Error(res.error) - } + streamData + ) - const responseData = (await response.json()) - return { data: responseData } + return response } } catch (error: any) { throw new Error(error) diff --git a/src/Lighthouse/uploadEncrypted/encryptionBrowser.ts b/src/Lighthouse/uploadEncrypted/encryptionBrowser.ts index 97d9afab..d830c3b3 100644 --- a/src/Lighthouse/uploadEncrypted/encryptionBrowser.ts +++ b/src/Lighthouse/uploadEncrypted/encryptionBrowser.ts @@ -19,72 +19,60 @@ const deriveKey = async ( ) export const encryptFile = async (fileArrayBuffer: any, password: any) => { - try { - const plainTextBytes = new Uint8Array(fileArrayBuffer) - const passwordBytes = new TextEncoder().encode(password) + const plainTextBytes = new Uint8Array(fileArrayBuffer) + const passwordBytes = new TextEncoder().encode(password) - const salt = window.crypto.getRandomValues(new Uint8Array(16)) - const iv = window.crypto.getRandomValues(new Uint8Array(12)) + const salt = window.crypto.getRandomValues(new Uint8Array(16)) + const iv = window.crypto.getRandomValues(new Uint8Array(12)) - const passwordKey = await importKeyFromBytes(passwordBytes) + const passwordKey = await importKeyFromBytes(passwordBytes) - const aesKey = await deriveKey(passwordKey, ['encrypt'], { - name: 'PBKDF2', - salt: salt, - iterations: 250000, - hash: 'SHA-256', - }) - const cipherBytes = await window.crypto.subtle.encrypt( - { name: 'AES-GCM', iv: iv }, - aesKey, - plainTextBytes - ) + const aesKey = await deriveKey(passwordKey, ['encrypt'], { + name: 'PBKDF2', + salt: salt, + iterations: 250000, + hash: 'SHA-256', + }) + const cipherBytes = await window.crypto.subtle.encrypt( + { name: 'AES-GCM', iv: iv }, + aesKey, + plainTextBytes + ) - const cipherBytesArray = new Uint8Array(cipherBytes) - const resultBytes = new Uint8Array( - cipherBytesArray.byteLength + salt.byteLength + iv.byteLength - ) - resultBytes.set(salt, 0) - resultBytes.set(iv, salt.byteLength) - resultBytes.set(cipherBytesArray, salt.byteLength + iv.byteLength) + const cipherBytesArray = new Uint8Array(cipherBytes) + const resultBytes = new Uint8Array( + cipherBytesArray.byteLength + salt.byteLength + iv.byteLength + ) + resultBytes.set(salt, 0) + resultBytes.set(iv, salt.byteLength) + resultBytes.set(cipherBytesArray, salt.byteLength + iv.byteLength) - return resultBytes - } catch (error) { - console.error('Error encrypting file') - console.error(error) - throw error - } + return resultBytes } export const decryptFile = async (cipher: any, password: any) => { - try { - const cipherBytes = new Uint8Array(cipher) - const passwordBytes = new TextEncoder().encode(password) + const cipherBytes = new Uint8Array(cipher) + const passwordBytes = new TextEncoder().encode(password) - const salt = cipherBytes.slice(0, 16) - const iv = cipherBytes.slice(16, 16 + 12) - const data = cipherBytes.slice(16 + 12) - const passwordKey = await importKeyFromBytes(passwordBytes) - const aesKey = await deriveKey(passwordKey, ['decrypt'], { - name: 'PBKDF2', - salt: salt, - iterations: 250000, - hash: 'SHA-256', - }) + const salt = cipherBytes.slice(0, 16) + const iv = cipherBytes.slice(16, 16 + 12) + const data = cipherBytes.slice(16 + 12) + const passwordKey = await importKeyFromBytes(passwordBytes) + const aesKey = await deriveKey(passwordKey, ['decrypt'], { + name: 'PBKDF2', + salt: salt, + iterations: 250000, + hash: 'SHA-256', + }) - const decryptedContent = await window.crypto.subtle.decrypt( - { - name: 'AES-GCM', - iv: iv, - }, - aesKey, - data - ) + const decryptedContent = await window.crypto.subtle.decrypt( + { + name: 'AES-GCM', + iv: iv, + }, + aesKey, + data + ) - return decryptedContent - } catch (error) { - console.error('Error decrypting file') - console.error(error) - return - } + return decryptedContent } diff --git a/src/Lighthouse/uploadEncrypted/encryptionNode.ts b/src/Lighthouse/uploadEncrypted/encryptionNode.ts index c90aab29..674960eb 100644 --- a/src/Lighthouse/uploadEncrypted/encryptionNode.ts +++ b/src/Lighthouse/uploadEncrypted/encryptionNode.ts @@ -16,90 +16,78 @@ const deriveKey = async ( ) const encryptFile = async (fileArrayBuffer: any, password: any) => { - try { - const { Crypto } = eval('require')('@peculiar/webcrypto') - const crypto = new Crypto() + const { Crypto } = eval('require')('@peculiar/webcrypto') + const crypto = new Crypto() - const plainTextBytes = new Uint8Array(fileArrayBuffer) - const passwordBytes = new TextEncoder().encode(password) + const plainTextBytes = new Uint8Array(fileArrayBuffer) + const passwordBytes = new TextEncoder().encode(password) - const salt = crypto.getRandomValues(new Uint8Array(16)) - const iv = crypto.getRandomValues(new Uint8Array(12)) + const salt = crypto.getRandomValues(new Uint8Array(16)) + const iv = crypto.getRandomValues(new Uint8Array(12)) - const passwordKey = await importKeyFromBytes(passwordBytes, crypto) + const passwordKey = await importKeyFromBytes(passwordBytes, crypto) - const aesKey = await deriveKey( - passwordKey, - ['encrypt'], - { - name: 'PBKDF2', - salt: salt, - iterations: 250000, - hash: 'SHA-256', - }, - crypto - ) - const cipherBytes = await crypto.subtle.encrypt( - { name: 'AES-GCM', iv: iv }, - aesKey, - plainTextBytes - ) + const aesKey = await deriveKey( + passwordKey, + ['encrypt'], + { + name: 'PBKDF2', + salt: salt, + iterations: 250000, + hash: 'SHA-256', + }, + crypto + ) + const cipherBytes = await crypto.subtle.encrypt( + { name: 'AES-GCM', iv: iv }, + aesKey, + plainTextBytes + ) - const cipherBytesArray = new Uint8Array(cipherBytes) - const resultBytes = new Uint8Array( - cipherBytesArray.byteLength + salt.byteLength + iv.byteLength - ) - resultBytes.set(salt, 0) - resultBytes.set(iv, salt.byteLength) - resultBytes.set(cipherBytesArray, salt.byteLength + iv.byteLength) + const cipherBytesArray = new Uint8Array(cipherBytes) + const resultBytes = new Uint8Array( + cipherBytesArray.byteLength + salt.byteLength + iv.byteLength + ) + resultBytes.set(salt, 0) + resultBytes.set(iv, salt.byteLength) + resultBytes.set(cipherBytesArray, salt.byteLength + iv.byteLength) - return resultBytes - } catch (error) { - console.error('Error encrypting file') - console.error(error) - throw error - } + return resultBytes } const decryptFile = async (cipher: any, password: any) => { - try { - const { Crypto } = eval('require')('@peculiar/webcrypto') - const crypto = new Crypto() + const { Crypto } = eval('require')('@peculiar/webcrypto') + const crypto = new Crypto() - const cipherBytes = new Uint8Array(cipher) - const passwordBytes = new TextEncoder().encode(password) + const cipherBytes = new Uint8Array(cipher) + const passwordBytes = new TextEncoder().encode(password) - const salt = cipherBytes.slice(0, 16) - const iv = cipherBytes.slice(16, 16 + 12) - const data = cipherBytes.slice(16 + 12) - const passwordKey = await importKeyFromBytes(passwordBytes, crypto) - const aesKey = await deriveKey( - passwordKey, - ['decrypt'], - { - name: 'PBKDF2', - salt: salt, - iterations: 250000, - hash: 'SHA-256', - }, - crypto - ) + const salt = cipherBytes.slice(0, 16) + const iv = cipherBytes.slice(16, 16 + 12) + const data = cipherBytes.slice(16 + 12) + const passwordKey = await importKeyFromBytes(passwordBytes, crypto) + const aesKey = await deriveKey( + passwordKey, + ['decrypt'], + { + name: 'PBKDF2', + salt: salt, + iterations: 250000, + hash: 'SHA-256', + }, + crypto + ) - const decryptedContent = await crypto.subtle.decrypt( - { - name: 'AES-GCM', - iv: iv, - }, - aesKey, - data - ) + const decryptedContent = await crypto.subtle.decrypt( + { + name: 'AES-GCM', + iv: iv, + }, + aesKey, + data + ) - return decryptedContent - } catch (error) { - console.error('Error decrypting file') - console.error(error) - return - } + return decryptedContent } export { encryptFile, decryptFile } diff --git a/src/Lighthouse/utils/util.ts b/src/Lighthouse/utils/util.ts index 0b39a359..8aae099a 100644 --- a/src/Lighthouse/utils/util.ts +++ b/src/Lighthouse/utils/util.ts @@ -5,6 +5,13 @@ interface FetchOptions extends RequestInit { onProgress?: (progress: number) => void } +interface DirectStreamOptions { + method?: string + headers?: Record + timeout?: number + onProgress?: (data: { progress: number }) => void +} + const isCID = (cid: string) => { return /^(Qm[1-9A-HJ-NP-Za-km-z]{44}|b[A-Za-z2-7]{58}|B[A-Z2-7]{58}|z[1-9A-HJ-NP-Za-km-z]{48}|F[0-9A-F]{50})*$/.test( cid @@ -127,10 +134,170 @@ async function fetchWithTimeout( } } +async function fetchWithDirectStream( + endpointURL: string, + options: DirectStreamOptions, + streamData: { + boundary: string + files: Array<{ + stream: any + filename: string + size: number + }> + } +): Promise<{ data: any }> { + const { + method = 'POST', + headers = {}, + timeout = 7200000, + onProgress, + } = options + + const http = eval(`require`)('http') + const https = eval(`require`)('https') + const url = eval(`require`)('url') + + const parsedUrl = url.parse(endpointURL) + const isHttps = parsedUrl.protocol === 'https:' + const client = isHttps ? https : http + + return new Promise((resolve, reject) => { + const requestOptions = { + hostname: parsedUrl.hostname, + port: parsedUrl.port || (isHttps ? 443 : 80), + path: parsedUrl.path, + method, + headers: { + ...headers, + 'Content-Type': `multipart/form-data; boundary=${streamData.boundary}`, + }, + } + + const req = client.request(requestOptions, (res: any) => { + let data = '' + res.on('data', (chunk: any) => { + data += chunk + }) + res.on('end', () => { + if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) { + try { + const responseData = JSON.parse(data) + resolve({ data: responseData }) + } catch (error) { + reject(new Error('Invalid JSON response')) + } + } else { + reject(new Error(`Request failed with status code ${res.statusCode}`)) + } + }) + }) + + req.on('error', (error: any) => { + reject(new Error(error.message)) + }) + + // Handle timeout + const timeoutId = setTimeout(() => { + req.destroy() + reject(new Error('Request timed out')) + }, timeout) + + req.on('close', () => { + clearTimeout(timeoutId) + }) + + // Track total bytes for progress calculation + let totalBytesUploaded = 0 + let totalBytesToUpload = 0 + + // Calculate total size for progress tracking + if (onProgress) { + for (const file of streamData.files) { + totalBytesToUpload += file.size + } + } + + // Stream files sequentially with backpressure handling and proper part delimiters + const writeAsync = (data: string | Buffer): Promise => { + return new Promise((resolve) => { + const canWrite = req.write(data) + if (canWrite) { + resolve() + } else { + req.once('drain', () => resolve()) + } + }) + } + + const pumpStream = (stream: any): Promise => { + return new Promise((resolve, rejectPump) => { + const onData = (chunk: any) => { + // Update progress if callback is provided + if (onProgress && totalBytesToUpload > 0) { + totalBytesUploaded += chunk.length + const progress = Math.min( + (totalBytesUploaded / totalBytesToUpload) * 100, + 100 + ) + onProgress({ progress }) + } + + const canWrite = req.write(chunk) + if (!canWrite) { + stream.pause() + req.once('drain', () => stream.resume()) + } + } + const onEnd = () => { + cleanup() + resolve() + } + const onError = (err: any) => { + cleanup() + rejectPump(new Error(`File stream error: ${err?.message || err}`)) + } + const cleanup = () => { + stream.off('data', onData) + stream.off('end', onEnd) + stream.off('error', onError) + } + stream.on('data', onData) + stream.on('end', onEnd) + stream.on('error', onError) + }) + } + + ;(async () => { + try { + for (let idx = 0; idx < streamData.files.length; idx++) { + const file = streamData.files[idx] + const headersPart = + `--${streamData.boundary}\r\n` + + `Content-Disposition: form-data; name="file"; filename="${file.filename}"\r\n` + + `Content-Type: application/octet-stream\r\n\r\n` + + await writeAsync(headersPart) + await pumpStream(file.stream) + await writeAsync(`\r\n`) + } + + await writeAsync(`--${streamData.boundary}--\r\n`) + req.end() + } catch (err: any) { + if (req && !req.destroyed) { + req.destroy() + } + reject(new Error(err?.message || String(err))) + } + })() + }) +} + export { isCID, isPrivateKey, addressValidator, checkDuplicateFileNames, fetchWithTimeout, + fetchWithDirectStream, }