diff --git a/test-runner/package.json b/test-runner/package.json index 19583002c..6ad797d33 100644 --- a/test-runner/package.json +++ b/test-runner/package.json @@ -17,7 +17,7 @@ "test-link": "node build/cli.js --test ./example/link.test.js --bundle \"example/languages/social-context.js\" --meta '{\"name\":\"social-context\",\"description\":\"Shortform expression for flux application\",\"sourceCodeLink\":\"https://github.com/juntofoundation/ad4m-languages\",\"possibleTemplateParams\":[\"uid\",\"name\"]}'", "test-ui": "node build/cli.js --ui --bundle \"example/languages/note-ipfs.js\" --meta '{\"name\":\"note-ipfs\",\"description\":\"Shortform expression for flux application\",\"sourceCodeLink\":\"https://github.com/juntofoundation/ad4m-languages\",\"possibleTemplateParams\":[\"uid\",\"name\"]}'", "ad4m-test": "./build/cli.js", - "postinstall": "node ./scripts/get-builtin-test-langs.js" + "prepublishOnly": "cd ../tests/js && pnpm run prepare-test && cd ../../test-runner && tsc && cp ../tests/js/bootstrapSeed.json ./bootstrapSeed.json" }, "preferGlobal": true, "dependencies": { @@ -26,7 +26,6 @@ "@types/fs-extra": "^9.0.13", "@types/node": "^18.0.0", "@types/node-fetch": "^2.6.1", - "@types/unzipper": "^0.10.5", "@types/ws": "8.5.4", "@types/yargs": "^17.0.8", "appdata-path": "perspect3vism/appdata-path", @@ -36,16 +35,15 @@ "express": "4.18.2", "find-process": "^1.4.7", "fs-extra": "^10.0.1", + "get-port": "^5.1.1", "glob": "^7.2.0", "graphql": "15.7.2", "graphql-ws": "5.12.0", "node-fetch": "2", - "node-wget-js": "^1.0.1", "subscriptions-transport-ws": "^0.11.0", "tree-kill": "^1.2.2", "ts-node": "^10.5.0", "typescript": "^4.6.2", - "unzipper": "^0.10.11", "uuid": "^8.3.2", "wget-improved": "^3.3.0", "ws": "8.13.0", diff --git a/test-runner/scripts/get-builtin-test-langs.js b/test-runner/scripts/get-builtin-test-langs.js index 6e1e7db43..fc4fab1f1 100644 --- a/test-runner/scripts/get-builtin-test-langs.js +++ b/test-runner/scripts/get-builtin-test-langs.js @@ -1,89 +1,15 @@ -const fs = require("fs-extra"); -const wget = require("node-wget-js"); -const { Extract } = require("unzipper"); -const { join } = require("path"); - -const languages = { - "agent-expression-store": { - bundle: "https://github.com/perspect3vism/agent-language/releases/download/0.2.0/bundle.js", - }, - languages: { - targetDnaName: "languages", - bundle: "https://github.com/perspect3vism/local-language-persistence/releases/download/0.0.1/bundle.js", - }, - "neighbourhood-store": { - targetDnaName: "neighbourhood-store", - //dna: "https://github.com/perspect3vism/neighbourhood-language/releases/download/0.0.2/neighbourhood-store.dna", - bundle: "https://github.com/perspect3vism/local-neighbourhood-persistence/releases/download/0.0.1/bundle.js", - }, - "perspective-diff-sync": { - bundle: "https://github.com/perspect3vism/perspective-diff-sync/releases/download/v0.2.2-test/bundle.js", - }, - "note-ipfs": { - bundle: "https://github.com/perspect3vism/lang-note-ipfs/releases/download/0.0.4/bundle.js", - }, - "direct-message-language": { - bundle: "https://github.com/perspect3vism/direct-message-language/releases/download/0.1.0/bundle.js" - }, - "perspective-language": { - bundle: "https://github.com/perspect3vism/perspective-language/releases/download/0.0.1/bundle.js" - } -}; - -async function main() { - for (const lang in languages) { - // const targetDir = fs.readFileSync('./scripts/download-languages-path').toString() - const dir = join('build/languages', lang) - await fs.ensureDir(dir + "/build"); - - let url = ""; - let dest = ""; - - // bundle - if (languages[lang].bundle) { - url = languages[lang].bundle; - dest = dir + "/build/bundle.js"; - if (url.slice(0, 8) != "https://" && url.slice(0, 7) != "http://") { - fs.copyFileSync(url, dest); - } else { - wget({ url, dest }); - } - } - - // dna - if (languages[lang].dna) { - console.log(languages[lang]) - url = languages[lang].dna; - dest = dir + `/${languages[lang].targetDnaName}.dna`; - wget({ url, dest }); - } - - if (languages[lang].zipped) { - await wget( - { - url: languages[lang].resource, - dest: `${dir}/lang.zip`, - }, - async () => { - //Read the zip file into a temp directory - await fs.createReadStream(`${dir}/lang.zip`) - .pipe(Extract({ path: `${dir}` })) - .promise(); - - // if (!fs.pathExistsSync(`${dir}/bundle.js`)) { - // throw Error("Did not find bundle file in unzipped path"); - // } - - fs.copyFileSync( - join(`${dir}/bundle.js`), - join(`${dir}/build/bundle.js`) - ); - fs.rmSync(`${dir}/lang.zip`); - fs.rmSync(`${dir}/bundle.js`); - } - ); - } - } -} - -main(); \ No newline at end of file +/** + * Post-install script for @coasys/ad4m-test + * + * Previously, this script downloaded pre-built system language bundles from + * perspect3vism repos. These bundles were CJS and needed conversion to ESM + * for the executor's Deno runtime. + * + * Now, the test runner uses the bootstrap seed (tests/js/bootstrapSeed.json) + * which contains the language-language bundle inline. The language-language + * fetches other system languages by hash from the bootstrap store (Cloudflare) + * at runtime. No local language bundles are needed. + */ + +// No-op — system languages are fetched at runtime via the bootstrap seed. +console.log('@coasys/ad4m-test: System languages will be fetched at runtime via bootstrap seed.'); diff --git a/test-runner/src/cli.ts b/test-runner/src/cli.ts index bdb4c90e0..fd098419d 100644 --- a/test-runner/src/cli.ts +++ b/test-runner/src/cli.ts @@ -152,13 +152,14 @@ export function startServer(relativePath: string, bundle: string, meta: string, let child: ChildProcessWithoutNullStreams; - const languageLanguageOnly = defaultLangPath ? 'false' : 'true'; + // The bootstrap seed contains hashes for all system languages. + // The language-language fetches them from the bootstrap store at runtime. child = spawn(`${binaryPath}`, [ 'run', '--admin-credential', global.ad4mToken, '--app-data-path', relativePath, '--gql-port', port.toString(), - '--language-language-only', languageLanguageOnly, + '--language-language-only', 'false', ]) const logFile = fs.createWriteStream(path.join(process.cwd(), 'ad4m-test.log')) @@ -282,7 +283,8 @@ async function run() { if (args.ui) { - await startServer(relativePath, args.bundle!, args.meta!, 'expression', 4000, args.defaultLangPath, () => { + await installSystemLanguages(relativePath); + await startServer(relativePath, args.bundle!, args.meta!, 'expression', 4000, undefined, () => { const app = express(); console.log(process.env.IP) diff --git a/test-runner/src/installSystemLanguages.ts b/test-runner/src/installSystemLanguages.ts index 04268e088..800c1d477 100644 --- a/test-runner/src/installSystemLanguages.ts +++ b/test-runner/src/installSystemLanguages.ts @@ -1,147 +1,74 @@ -import { LanguageMetaInput } from "@coasys/ad4m" import path from "path"; import fs from 'fs-extra'; -import { ChildProcessWithoutNullStreams, execFileSync, spawn } from "child_process"; import { ad4mDataDirectory, deleteAllAd4mData, findAndKillProcess, getAd4mHostBinary, logger } from "./utils"; -import kill from 'tree-kill' -import { buildAd4mClient } from "./client"; - -let seed = { - trustedAgents: [], - knownLinkLanguages: [], - directMessageLanguage: "", - agentLanguage: "", - perspectiveLanguage: "", - neighbourhoodLanguage: "", - languageLanguageBundle: "", - languageLanguageSettings : { - storagePath: "" - }, - neighbourhoodLanguageSettings: { - storagePath: "" - } -} - -const languagesToPublish = { - "agent-expression-store": {name: "agent-expression-store", description: "", possibleTemplateParams: ["id", "name", "description"], sourceCodeLink: ""} as LanguageMetaInput, - "direct-message-language": {name: "direct-message-language", description: "", possibleTemplateParams: ["recipient_did", "recipient_hc_agent_pubkey"], sourceCodeLink: ""} as LanguageMetaInput, - "neighbourhood-store": {name: "neighbourhood-store", description: "", possibleTemplateParams: ["id", "name", "description"], sourceCodeLink: ""} as LanguageMetaInput, - "perspective-language": {name: "perspective-language", description: "", possibleTemplateParams: ["id", "name", "description"], sourceCodeLink: ""} as LanguageMetaInput, -} +/** + * Prepare the test environment by setting up the bootstrap seed and initializing + * the AD4M executor data directory. + * + * The bootstrap seed contains the language-language bundle inline (ESM) and + * hashes for all system languages. At runtime, the language-language fetches + * other languages by hash from the bootstrap store (Cloudflare). + * + * The seed is located at: + * - In the AD4M monorepo: ../../tests/js/bootstrapSeed.json (relative to build/) + * - When installed as a package: ../bootstrapSeed.json (placed by consumer's setup) + * + * This replaces the previous approach of downloading individual language bundles + * from perspect3vism repos and converting them from CJS to ESM. + */ export async function installSystemLanguages(relativePath = '') { - return new Promise(async (resolve, reject) => { - deleteAllAd4mData(relativePath); - fs.removeSync(path.join(__dirname, 'publishedLanguages')) - fs.removeSync(path.join(__dirname, 'publishedNeighbourhood')) - fs.mkdirSync(path.join(__dirname, 'publishedLanguages')) - fs.mkdirSync(path.join(__dirname, 'publishedNeighbourhood')) - - let binaryPath = path.join(ad4mDataDirectory(relativePath), 'binary', `ad4m`); - - if (!fs.existsSync(binaryPath)) { - await getAd4mHostBinary(relativePath); - binaryPath = path.join(ad4mDataDirectory(relativePath), 'binary', `ad4m`); - } - - await findAndKillProcess('holochain') - await findAndKillProcess('lair-keystore') - - const seedFile = path.join(__dirname, '../bootstrapSeed.json') - - execFileSync(binaryPath, ['init', '--data-path', relativePath, '--network-bootstrap-seed', seedFile], { encoding: 'utf-8' }); - - logger.info('ad4m-test initialized') - - let child: ChildProcessWithoutNullStreams; - - const languageLanguageBundlePath = path.join(__dirname, 'languages', "languages", "build", "bundle.js"); - - seed['languageLanguageBundle'] = fs.readFileSync(languageLanguageBundlePath).toString(); - seed['languageLanguageSettings'] = { storagePath: path.join(__dirname, 'publishedLanguages') } - seed['neighbourhoodLanguageSettings'] = { storagePath: path.join(__dirname, 'publishedNeighbourhood') } + deleteAllAd4mData(relativePath); - fs.writeFileSync(path.join(__dirname, '../bootstrapSeed.json'), JSON.stringify(seed)); + let binaryPath = path.join(ad4mDataDirectory(relativePath), 'binary', `ad4m`); - child = spawn(`${binaryPath}`, [ - 'run', - '--admin-credential', global.ad4mToken, - '--app-data-path', relativePath, - '--gql-port', '4000', - '--language-language-only', 'true', - ]) - - - const logFile = fs.createWriteStream(path.join(process.cwd(), 'ad4m-test.log')) - - child.stdout.on('data', async (data) => { - logFile.write(data) - }); - child.stderr.on('data', async (data) => { - // Re-emit stderr on stdout so detection logic below catches Rust log output - // (stdout handler already writes to logFile, so no duplicate write needed here) - child.stdout.emit('data', data) - }) - - child.stdout.on('data', async (data) => { - if (data.toString().includes('GraphQL server started, Unlock the agent to start holohchain') || data.toString().includes('listening on http://127.0.0.1')) { - const client = await buildAd4mClient(4000); - - await client.agent.generate('123456789') - } - - if (data.toString().includes('AD4M init complete')) { - for (const [lang, languageMeta] of Object.entries(languagesToPublish)) { - const client = await buildAd4mClient(4000); - - const bundlePath = path.join(__dirname, 'languages', lang, 'build', 'bundle.js') - - const language = await client.languages.publish(bundlePath, languageMeta); - - if (lang === "agent-expression-store") { - seed["agentLanguage"] = language.address; - } - if (lang === "neighbourhood-store") { - seed["neighbourhoodLanguage"] = language.address; - } - if (lang === "direct-message-language") { - seed["directMessageLanguage"] = language.address; - } - if (lang === "perspective-language") { - seed["perspectiveLanguage"] = language.address; - } - } - fs.writeFileSync(path.join(__dirname, '../bootstrapSeed.json'), JSON.stringify(seed)); - - logger.info('bootstrapSeed file populated with system language hashes') + if (!fs.existsSync(binaryPath)) { + await getAd4mHostBinary(relativePath); + binaryPath = path.join(ad4mDataDirectory(relativePath), 'binary', `ad4m`); + } - kill(child.pid!, async () => { - await findAndKillProcess('holochain') - await findAndKillProcess('lair-keystore') - resolve(null); - }) - } - }); + await findAndKillProcess('holochain') + await findAndKillProcess('lair-keystore') + + // Look for bootstrap seed in order of preference: + // 1. Adjacent to package root (../bootstrapSeed.json from build/) — installed package + // 2. In the monorepo (../../tests/js/bootstrapSeed.json from build/) — development + const candidates = [ + path.join(__dirname, '../bootstrapSeed.json'), + path.join(__dirname, '../../tests/js/bootstrapSeed.json'), + ]; + + let seedPath: string | null = null; + for (const candidate of candidates) { + if (fs.existsSync(candidate)) { + seedPath = candidate; + break; + } + } - child.on('exit', (code) => { - logger.info(`exit is called ${code}`); - resolve(null); - }) + if (!seedPath) { + throw new Error( + `Bootstrap seed not found. Looked in:\n` + + candidates.map(c => ` - ${c}`).join('\n') + + `\nEnsure bootstrapSeed.json exists (copy from tests/js/ or download from the AD4M repo).` + ); + } - child.on('error', () => { - logger.error(`process error: ${child.pid}`) - findAndKillProcess('holochain') - findAndKillProcess('lair-keystore') - findAndKillProcess('ad4m') - reject() - }); - }); + // If the seed is in the monorepo, copy it to the package root for startServer to find + const targetSeedPath = path.join(__dirname, '../bootstrapSeed.json'); + if (seedPath !== targetSeedPath) { + fs.copySync(seedPath, targetSeedPath); + logger.info(`Bootstrap seed copied from ${seedPath}`); + } else { + logger.info('Bootstrap seed found at package root'); + } } if (require.main === module) { installSystemLanguages().then(() => { process.exit(0); }).catch(e => { + console.error(e); process.exit(1); }); }