-
Notifications
You must be signed in to change notification settings - Fork 21
fix(test-runner): use bootstrap seed instead of downloading stale language bundles #678
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,89 +1,15 @@ | ||
| const fs = require("fs-extra"); | ||
| const wget = require("node-wget-js"); | ||
| const { Extract } = require("unzipper"); | ||
| const { join } = require("path"); | ||
|
|
||
| const languages = { | ||
| "agent-expression-store": { | ||
| bundle: "https://github.com/perspect3vism/agent-language/releases/download/0.2.0/bundle.js", | ||
| }, | ||
| languages: { | ||
| targetDnaName: "languages", | ||
| bundle: "https://github.com/perspect3vism/local-language-persistence/releases/download/0.0.1/bundle.js", | ||
| }, | ||
| "neighbourhood-store": { | ||
| targetDnaName: "neighbourhood-store", | ||
| //dna: "https://github.com/perspect3vism/neighbourhood-language/releases/download/0.0.2/neighbourhood-store.dna", | ||
| bundle: "https://github.com/perspect3vism/local-neighbourhood-persistence/releases/download/0.0.1/bundle.js", | ||
| }, | ||
| "perspective-diff-sync": { | ||
| bundle: "https://github.com/perspect3vism/perspective-diff-sync/releases/download/v0.2.2-test/bundle.js", | ||
| }, | ||
| "note-ipfs": { | ||
| bundle: "https://github.com/perspect3vism/lang-note-ipfs/releases/download/0.0.4/bundle.js", | ||
| }, | ||
| "direct-message-language": { | ||
| bundle: "https://github.com/perspect3vism/direct-message-language/releases/download/0.1.0/bundle.js" | ||
| }, | ||
| "perspective-language": { | ||
| bundle: "https://github.com/perspect3vism/perspective-language/releases/download/0.0.1/bundle.js" | ||
| } | ||
| }; | ||
|
|
||
| async function main() { | ||
| for (const lang in languages) { | ||
| // const targetDir = fs.readFileSync('./scripts/download-languages-path').toString() | ||
| const dir = join('build/languages', lang) | ||
| await fs.ensureDir(dir + "/build"); | ||
|
|
||
| let url = ""; | ||
| let dest = ""; | ||
|
|
||
| // bundle | ||
| if (languages[lang].bundle) { | ||
| url = languages[lang].bundle; | ||
| dest = dir + "/build/bundle.js"; | ||
| if (url.slice(0, 8) != "https://" && url.slice(0, 7) != "http://") { | ||
| fs.copyFileSync(url, dest); | ||
| } else { | ||
| wget({ url, dest }); | ||
| } | ||
| } | ||
|
|
||
| // dna | ||
| if (languages[lang].dna) { | ||
| console.log(languages[lang]) | ||
| url = languages[lang].dna; | ||
| dest = dir + `/${languages[lang].targetDnaName}.dna`; | ||
| wget({ url, dest }); | ||
| } | ||
|
|
||
| if (languages[lang].zipped) { | ||
| await wget( | ||
| { | ||
| url: languages[lang].resource, | ||
| dest: `${dir}/lang.zip`, | ||
| }, | ||
| async () => { | ||
| //Read the zip file into a temp directory | ||
| await fs.createReadStream(`${dir}/lang.zip`) | ||
| .pipe(Extract({ path: `${dir}` })) | ||
| .promise(); | ||
|
|
||
| // if (!fs.pathExistsSync(`${dir}/bundle.js`)) { | ||
| // throw Error("Did not find bundle file in unzipped path"); | ||
| // } | ||
|
|
||
| fs.copyFileSync( | ||
| join(`${dir}/bundle.js`), | ||
| join(`${dir}/build/bundle.js`) | ||
| ); | ||
| fs.rmSync(`${dir}/lang.zip`); | ||
| fs.rmSync(`${dir}/bundle.js`); | ||
| } | ||
| ); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| main(); | ||
| /** | ||
| * Post-install script for @coasys/ad4m-test | ||
| * | ||
| * Previously, this script downloaded pre-built system language bundles from | ||
| * perspect3vism repos. These bundles were CJS and needed conversion to ESM | ||
| * for the executor's Deno runtime. | ||
| * | ||
| * Now, the test runner uses the bootstrap seed (tests/js/bootstrapSeed.json) | ||
| * which contains the language-language bundle inline. The language-language | ||
| * fetches other system languages by hash from the bootstrap store (Cloudflare) | ||
| * at runtime. No local language bundles are needed. | ||
| */ | ||
|
|
||
| // No-op — system languages are fetched at runtime via the bootstrap seed. | ||
| console.log('@coasys/ad4m-test: System languages will be fetched at runtime via bootstrap seed.'); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,147 +1,74 @@ | ||
| import { LanguageMetaInput } from "@coasys/ad4m" | ||
| import path from "path"; | ||
| import fs from 'fs-extra'; | ||
| import { ChildProcessWithoutNullStreams, execFileSync, spawn } from "child_process"; | ||
| import { ad4mDataDirectory, deleteAllAd4mData, findAndKillProcess, getAd4mHostBinary, logger } from "./utils"; | ||
| import kill from 'tree-kill' | ||
| import { buildAd4mClient } from "./client"; | ||
|
|
||
| let seed = { | ||
| trustedAgents: [], | ||
| knownLinkLanguages: [], | ||
| directMessageLanguage: "", | ||
| agentLanguage: "", | ||
| perspectiveLanguage: "", | ||
| neighbourhoodLanguage: "", | ||
| languageLanguageBundle: "", | ||
| languageLanguageSettings : { | ||
| storagePath: "" | ||
| }, | ||
| neighbourhoodLanguageSettings: { | ||
| storagePath: "" | ||
| } | ||
| } | ||
|
|
||
| const languagesToPublish = { | ||
| "agent-expression-store": {name: "agent-expression-store", description: "", possibleTemplateParams: ["id", "name", "description"], sourceCodeLink: ""} as LanguageMetaInput, | ||
| "direct-message-language": {name: "direct-message-language", description: "", possibleTemplateParams: ["recipient_did", "recipient_hc_agent_pubkey"], sourceCodeLink: ""} as LanguageMetaInput, | ||
| "neighbourhood-store": {name: "neighbourhood-store", description: "", possibleTemplateParams: ["id", "name", "description"], sourceCodeLink: ""} as LanguageMetaInput, | ||
| "perspective-language": {name: "perspective-language", description: "", possibleTemplateParams: ["id", "name", "description"], sourceCodeLink: ""} as LanguageMetaInput, | ||
| } | ||
|
|
||
| /** | ||
| * Prepare the test environment by setting up the bootstrap seed and initializing | ||
| * the AD4M executor data directory. | ||
| * | ||
| * The bootstrap seed contains the language-language bundle inline (ESM) and | ||
| * hashes for all system languages. At runtime, the language-language fetches | ||
| * other languages by hash from the bootstrap store (Cloudflare). | ||
| * | ||
| * The seed is located at: | ||
| * - In the AD4M monorepo: ../../tests/js/bootstrapSeed.json (relative to build/) | ||
| * - When installed as a package: ../bootstrapSeed.json (placed by consumer's setup) | ||
| * | ||
| * This replaces the previous approach of downloading individual language bundles | ||
| * from perspect3vism repos and converting them from CJS to ESM. | ||
| */ | ||
| export async function installSystemLanguages(relativePath = '') { | ||
| return new Promise(async (resolve, reject) => { | ||
| deleteAllAd4mData(relativePath); | ||
| fs.removeSync(path.join(__dirname, 'publishedLanguages')) | ||
| fs.removeSync(path.join(__dirname, 'publishedNeighbourhood')) | ||
| fs.mkdirSync(path.join(__dirname, 'publishedLanguages')) | ||
| fs.mkdirSync(path.join(__dirname, 'publishedNeighbourhood')) | ||
|
|
||
| let binaryPath = path.join(ad4mDataDirectory(relativePath), 'binary', `ad4m`); | ||
|
|
||
| if (!fs.existsSync(binaryPath)) { | ||
| await getAd4mHostBinary(relativePath); | ||
| binaryPath = path.join(ad4mDataDirectory(relativePath), 'binary', `ad4m`); | ||
| } | ||
|
|
||
| await findAndKillProcess('holochain') | ||
| await findAndKillProcess('lair-keystore') | ||
|
|
||
| const seedFile = path.join(__dirname, '../bootstrapSeed.json') | ||
|
|
||
| execFileSync(binaryPath, ['init', '--data-path', relativePath, '--network-bootstrap-seed', seedFile], { encoding: 'utf-8' }); | ||
|
|
||
| logger.info('ad4m-test initialized') | ||
|
|
||
| let child: ChildProcessWithoutNullStreams; | ||
|
|
||
| const languageLanguageBundlePath = path.join(__dirname, 'languages', "languages", "build", "bundle.js"); | ||
|
|
||
| seed['languageLanguageBundle'] = fs.readFileSync(languageLanguageBundlePath).toString(); | ||
| seed['languageLanguageSettings'] = { storagePath: path.join(__dirname, 'publishedLanguages') } | ||
| seed['neighbourhoodLanguageSettings'] = { storagePath: path.join(__dirname, 'publishedNeighbourhood') } | ||
| deleteAllAd4mData(relativePath); | ||
coderabbitai[bot] marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| fs.writeFileSync(path.join(__dirname, '../bootstrapSeed.json'), JSON.stringify(seed)); | ||
| let binaryPath = path.join(ad4mDataDirectory(relativePath), 'binary', `ad4m`); | ||
|
|
||
| child = spawn(`${binaryPath}`, [ | ||
| 'run', | ||
| '--admin-credential', global.ad4mToken, | ||
| '--app-data-path', relativePath, | ||
| '--gql-port', '4000', | ||
| '--language-language-only', 'true', | ||
| ]) | ||
|
|
||
|
|
||
| const logFile = fs.createWriteStream(path.join(process.cwd(), 'ad4m-test.log')) | ||
|
|
||
| child.stdout.on('data', async (data) => { | ||
| logFile.write(data) | ||
| }); | ||
| child.stderr.on('data', async (data) => { | ||
| // Re-emit stderr on stdout so detection logic below catches Rust log output | ||
| // (stdout handler already writes to logFile, so no duplicate write needed here) | ||
| child.stdout.emit('data', data) | ||
| }) | ||
|
|
||
| child.stdout.on('data', async (data) => { | ||
| if (data.toString().includes('GraphQL server started, Unlock the agent to start holohchain') || data.toString().includes('listening on http://127.0.0.1')) { | ||
| const client = await buildAd4mClient(4000); | ||
|
|
||
| await client.agent.generate('123456789') | ||
| } | ||
|
|
||
| if (data.toString().includes('AD4M init complete')) { | ||
| for (const [lang, languageMeta] of Object.entries(languagesToPublish)) { | ||
| const client = await buildAd4mClient(4000); | ||
|
|
||
| const bundlePath = path.join(__dirname, 'languages', lang, 'build', 'bundle.js') | ||
|
|
||
| const language = await client.languages.publish(bundlePath, languageMeta); | ||
|
|
||
| if (lang === "agent-expression-store") { | ||
| seed["agentLanguage"] = language.address; | ||
| } | ||
| if (lang === "neighbourhood-store") { | ||
| seed["neighbourhoodLanguage"] = language.address; | ||
| } | ||
| if (lang === "direct-message-language") { | ||
| seed["directMessageLanguage"] = language.address; | ||
| } | ||
| if (lang === "perspective-language") { | ||
| seed["perspectiveLanguage"] = language.address; | ||
| } | ||
| } | ||
| fs.writeFileSync(path.join(__dirname, '../bootstrapSeed.json'), JSON.stringify(seed)); | ||
|
|
||
| logger.info('bootstrapSeed file populated with system language hashes') | ||
| if (!fs.existsSync(binaryPath)) { | ||
| await getAd4mHostBinary(relativePath); | ||
| binaryPath = path.join(ad4mDataDirectory(relativePath), 'binary', `ad4m`); | ||
| } | ||
HexaField marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| kill(child.pid!, async () => { | ||
| await findAndKillProcess('holochain') | ||
| await findAndKillProcess('lair-keystore') | ||
| resolve(null); | ||
| }) | ||
| } | ||
| }); | ||
| await findAndKillProcess('holochain') | ||
| await findAndKillProcess('lair-keystore') | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. that is also super old. we are not spawning holochain (no lair-keystore) as external processes. all inside the ad4m-executor. these lines should go away.
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @HexaField I will merge this now, but please note my comments above for future work on the test-runner. |
||
|
|
||
| // Look for bootstrap seed in order of preference: | ||
| // 1. Adjacent to package root (../bootstrapSeed.json from build/) — installed package | ||
| // 2. In the monorepo (../../tests/js/bootstrapSeed.json from build/) — development | ||
| const candidates = [ | ||
| path.join(__dirname, '../bootstrapSeed.json'), | ||
| path.join(__dirname, '../../tests/js/bootstrapSeed.json'), | ||
| ]; | ||
|
|
||
| let seedPath: string | null = null; | ||
| for (const candidate of candidates) { | ||
| if (fs.existsSync(candidate)) { | ||
| seedPath = candidate; | ||
| break; | ||
| } | ||
| } | ||
|
|
||
| child.on('exit', (code) => { | ||
| logger.info(`exit is called ${code}`); | ||
| resolve(null); | ||
| }) | ||
| if (!seedPath) { | ||
| throw new Error( | ||
| `Bootstrap seed not found. Looked in:\n` + | ||
| candidates.map(c => ` - ${c}`).join('\n') + | ||
| `\nEnsure bootstrapSeed.json exists (copy from tests/js/ or download from the AD4M repo).` | ||
| ); | ||
| } | ||
coderabbitai[bot] marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| child.on('error', () => { | ||
| logger.error(`process error: ${child.pid}`) | ||
| findAndKillProcess('holochain') | ||
| findAndKillProcess('lair-keystore') | ||
| findAndKillProcess('ad4m') | ||
| reject() | ||
| }); | ||
| }); | ||
| // If the seed is in the monorepo, copy it to the package root for startServer to find | ||
| const targetSeedPath = path.join(__dirname, '../bootstrapSeed.json'); | ||
| if (seedPath !== targetSeedPath) { | ||
| fs.copySync(seedPath, targetSeedPath); | ||
| logger.info(`Bootstrap seed copied from ${seedPath}`); | ||
| } else { | ||
| logger.info('Bootstrap seed found at package root'); | ||
| } | ||
| } | ||
|
|
||
| if (require.main === module) { | ||
| installSystemLanguages().then(() => { | ||
| process.exit(0); | ||
| }).catch(e => { | ||
| console.error(e); | ||
| process.exit(1); | ||
| }); | ||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.