Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions test-runner/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"test-link": "node build/cli.js --test ./example/link.test.js --bundle \"example/languages/social-context.js\" --meta '{\"name\":\"social-context\",\"description\":\"Shortform expression for flux application\",\"sourceCodeLink\":\"https://github.com/juntofoundation/ad4m-languages\",\"possibleTemplateParams\":[\"uid\",\"name\"]}'",
"test-ui": "node build/cli.js --ui --bundle \"example/languages/note-ipfs.js\" --meta '{\"name\":\"note-ipfs\",\"description\":\"Shortform expression for flux application\",\"sourceCodeLink\":\"https://github.com/juntofoundation/ad4m-languages\",\"possibleTemplateParams\":[\"uid\",\"name\"]}'",
"ad4m-test": "./build/cli.js",
"postinstall": "node ./scripts/get-builtin-test-langs.js"
"prepublishOnly": "cd ../tests/js && pnpm run prepare-test && cd ../../test-runner && tsc && cp ../tests/js/bootstrapSeed.json ./bootstrapSeed.json"
},
"preferGlobal": true,
"dependencies": {
Expand All @@ -26,7 +26,6 @@
"@types/fs-extra": "^9.0.13",
"@types/node": "^18.0.0",
"@types/node-fetch": "^2.6.1",
"@types/unzipper": "^0.10.5",
"@types/ws": "8.5.4",
"@types/yargs": "^17.0.8",
"appdata-path": "perspect3vism/appdata-path",
Expand All @@ -36,16 +35,15 @@
"express": "4.18.2",
"find-process": "^1.4.7",
"fs-extra": "^10.0.1",
"get-port": "^5.1.1",
"glob": "^7.2.0",
"graphql": "15.7.2",
"graphql-ws": "5.12.0",
"node-fetch": "2",
"node-wget-js": "^1.0.1",
"subscriptions-transport-ws": "^0.11.0",
"tree-kill": "^1.2.2",
"ts-node": "^10.5.0",
"typescript": "^4.6.2",
"unzipper": "^0.10.11",
"uuid": "^8.3.2",
"wget-improved": "^3.3.0",
"ws": "8.13.0",
Expand Down
104 changes: 15 additions & 89 deletions test-runner/scripts/get-builtin-test-langs.js
Original file line number Diff line number Diff line change
@@ -1,89 +1,15 @@
const fs = require("fs-extra");
const wget = require("node-wget-js");
const { Extract } = require("unzipper");
const { join } = require("path");

const languages = {
"agent-expression-store": {
bundle: "https://github.com/perspect3vism/agent-language/releases/download/0.2.0/bundle.js",
},
languages: {
targetDnaName: "languages",
bundle: "https://github.com/perspect3vism/local-language-persistence/releases/download/0.0.1/bundle.js",
},
"neighbourhood-store": {
targetDnaName: "neighbourhood-store",
//dna: "https://github.com/perspect3vism/neighbourhood-language/releases/download/0.0.2/neighbourhood-store.dna",
bundle: "https://github.com/perspect3vism/local-neighbourhood-persistence/releases/download/0.0.1/bundle.js",
},
"perspective-diff-sync": {
bundle: "https://github.com/perspect3vism/perspective-diff-sync/releases/download/v0.2.2-test/bundle.js",
},
"note-ipfs": {
bundle: "https://github.com/perspect3vism/lang-note-ipfs/releases/download/0.0.4/bundle.js",
},
"direct-message-language": {
bundle: "https://github.com/perspect3vism/direct-message-language/releases/download/0.1.0/bundle.js"
},
"perspective-language": {
bundle: "https://github.com/perspect3vism/perspective-language/releases/download/0.0.1/bundle.js"
}
};

async function main() {
for (const lang in languages) {
// const targetDir = fs.readFileSync('./scripts/download-languages-path').toString()
const dir = join('build/languages', lang)
await fs.ensureDir(dir + "/build");

let url = "";
let dest = "";

// bundle
if (languages[lang].bundle) {
url = languages[lang].bundle;
dest = dir + "/build/bundle.js";
if (url.slice(0, 8) != "https://" && url.slice(0, 7) != "http://") {
fs.copyFileSync(url, dest);
} else {
wget({ url, dest });
}
}

// dna
if (languages[lang].dna) {
console.log(languages[lang])
url = languages[lang].dna;
dest = dir + `/${languages[lang].targetDnaName}.dna`;
wget({ url, dest });
}

if (languages[lang].zipped) {
await wget(
{
url: languages[lang].resource,
dest: `${dir}/lang.zip`,
},
async () => {
//Read the zip file into a temp directory
await fs.createReadStream(`${dir}/lang.zip`)
.pipe(Extract({ path: `${dir}` }))
.promise();

// if (!fs.pathExistsSync(`${dir}/bundle.js`)) {
// throw Error("Did not find bundle file in unzipped path");
// }

fs.copyFileSync(
join(`${dir}/bundle.js`),
join(`${dir}/build/bundle.js`)
);
fs.rmSync(`${dir}/lang.zip`);
fs.rmSync(`${dir}/bundle.js`);
}
);
}
}
}

main();
/**
* Post-install script for @coasys/ad4m-test
*
* Previously, this script downloaded pre-built system language bundles from
* perspect3vism repos. These bundles were CJS and needed conversion to ESM
* for the executor's Deno runtime.
*
* Now, the test runner uses the bootstrap seed (tests/js/bootstrapSeed.json)
* which contains the language-language bundle inline. The language-language
* fetches other system languages by hash from the bootstrap store (Cloudflare)
* at runtime. No local language bundles are needed.
*/

// No-op — system languages are fetched at runtime via the bootstrap seed.
console.log('@coasys/ad4m-test: System languages will be fetched at runtime via bootstrap seed.');
8 changes: 5 additions & 3 deletions test-runner/src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -152,13 +152,14 @@ export function startServer(relativePath: string, bundle: string, meta: string,

let child: ChildProcessWithoutNullStreams;

const languageLanguageOnly = defaultLangPath ? 'false' : 'true';
// The bootstrap seed contains hashes for all system languages.
// The language-language fetches them from the bootstrap store at runtime.
child = spawn(`${binaryPath}`, [
'run',
'--admin-credential', global.ad4mToken,
'--app-data-path', relativePath,
'--gql-port', port.toString(),
'--language-language-only', languageLanguageOnly,
'--language-language-only', 'false',
])

const logFile = fs.createWriteStream(path.join(process.cwd(), 'ad4m-test.log'))
Expand Down Expand Up @@ -282,7 +283,8 @@ async function run() {


if (args.ui) {
await startServer(relativePath, args.bundle!, args.meta!, 'expression', 4000, args.defaultLangPath, () => {
await installSystemLanguages(relativePath);
await startServer(relativePath, args.bundle!, args.meta!, 'expression', 4000, undefined, () => {
const app = express();

console.log(process.env.IP)
Expand Down
183 changes: 55 additions & 128 deletions test-runner/src/installSystemLanguages.ts
Original file line number Diff line number Diff line change
@@ -1,147 +1,74 @@
import { LanguageMetaInput } from "@coasys/ad4m"
import path from "path";
import fs from 'fs-extra';
import { ChildProcessWithoutNullStreams, execFileSync, spawn } from "child_process";
import { ad4mDataDirectory, deleteAllAd4mData, findAndKillProcess, getAd4mHostBinary, logger } from "./utils";
import kill from 'tree-kill'
import { buildAd4mClient } from "./client";

let seed = {
trustedAgents: [],
knownLinkLanguages: [],
directMessageLanguage: "",
agentLanguage: "",
perspectiveLanguage: "",
neighbourhoodLanguage: "",
languageLanguageBundle: "",
languageLanguageSettings : {
storagePath: ""
},
neighbourhoodLanguageSettings: {
storagePath: ""
}
}

const languagesToPublish = {
"agent-expression-store": {name: "agent-expression-store", description: "", possibleTemplateParams: ["id", "name", "description"], sourceCodeLink: ""} as LanguageMetaInput,
"direct-message-language": {name: "direct-message-language", description: "", possibleTemplateParams: ["recipient_did", "recipient_hc_agent_pubkey"], sourceCodeLink: ""} as LanguageMetaInput,
"neighbourhood-store": {name: "neighbourhood-store", description: "", possibleTemplateParams: ["id", "name", "description"], sourceCodeLink: ""} as LanguageMetaInput,
"perspective-language": {name: "perspective-language", description: "", possibleTemplateParams: ["id", "name", "description"], sourceCodeLink: ""} as LanguageMetaInput,
}

/**
* Prepare the test environment by setting up the bootstrap seed and initializing
* the AD4M executor data directory.
*
* The bootstrap seed contains the language-language bundle inline (ESM) and
* hashes for all system languages. At runtime, the language-language fetches
* other languages by hash from the bootstrap store (Cloudflare).
*
* The seed is located at:
* - In the AD4M monorepo: ../../tests/js/bootstrapSeed.json (relative to build/)
* - When installed as a package: ../bootstrapSeed.json (placed by consumer's setup)
*
* This replaces the previous approach of downloading individual language bundles
* from perspect3vism repos and converting them from CJS to ESM.
*/
export async function installSystemLanguages(relativePath = '') {
return new Promise(async (resolve, reject) => {
deleteAllAd4mData(relativePath);
fs.removeSync(path.join(__dirname, 'publishedLanguages'))
fs.removeSync(path.join(__dirname, 'publishedNeighbourhood'))
fs.mkdirSync(path.join(__dirname, 'publishedLanguages'))
fs.mkdirSync(path.join(__dirname, 'publishedNeighbourhood'))

let binaryPath = path.join(ad4mDataDirectory(relativePath), 'binary', `ad4m`);

if (!fs.existsSync(binaryPath)) {
await getAd4mHostBinary(relativePath);
binaryPath = path.join(ad4mDataDirectory(relativePath), 'binary', `ad4m`);
}

await findAndKillProcess('holochain')
await findAndKillProcess('lair-keystore')

const seedFile = path.join(__dirname, '../bootstrapSeed.json')

execFileSync(binaryPath, ['init', '--data-path', relativePath, '--network-bootstrap-seed', seedFile], { encoding: 'utf-8' });

logger.info('ad4m-test initialized')

let child: ChildProcessWithoutNullStreams;

const languageLanguageBundlePath = path.join(__dirname, 'languages', "languages", "build", "bundle.js");

seed['languageLanguageBundle'] = fs.readFileSync(languageLanguageBundlePath).toString();
seed['languageLanguageSettings'] = { storagePath: path.join(__dirname, 'publishedLanguages') }
seed['neighbourhoodLanguageSettings'] = { storagePath: path.join(__dirname, 'publishedNeighbourhood') }
deleteAllAd4mData(relativePath);

fs.writeFileSync(path.join(__dirname, '../bootstrapSeed.json'), JSON.stringify(seed));
let binaryPath = path.join(ad4mDataDirectory(relativePath), 'binary', `ad4m`);

child = spawn(`${binaryPath}`, [
'run',
'--admin-credential', global.ad4mToken,
'--app-data-path', relativePath,
'--gql-port', '4000',
'--language-language-only', 'true',
])


const logFile = fs.createWriteStream(path.join(process.cwd(), 'ad4m-test.log'))

child.stdout.on('data', async (data) => {
logFile.write(data)
});
child.stderr.on('data', async (data) => {
// Re-emit stderr on stdout so detection logic below catches Rust log output
// (stdout handler already writes to logFile, so no duplicate write needed here)
child.stdout.emit('data', data)
})

child.stdout.on('data', async (data) => {
if (data.toString().includes('GraphQL server started, Unlock the agent to start holohchain') || data.toString().includes('listening on http://127.0.0.1')) {
const client = await buildAd4mClient(4000);

await client.agent.generate('123456789')
}

if (data.toString().includes('AD4M init complete')) {
for (const [lang, languageMeta] of Object.entries(languagesToPublish)) {
const client = await buildAd4mClient(4000);

const bundlePath = path.join(__dirname, 'languages', lang, 'build', 'bundle.js')

const language = await client.languages.publish(bundlePath, languageMeta);

if (lang === "agent-expression-store") {
seed["agentLanguage"] = language.address;
}
if (lang === "neighbourhood-store") {
seed["neighbourhoodLanguage"] = language.address;
}
if (lang === "direct-message-language") {
seed["directMessageLanguage"] = language.address;
}
if (lang === "perspective-language") {
seed["perspectiveLanguage"] = language.address;
}
}
fs.writeFileSync(path.join(__dirname, '../bootstrapSeed.json'), JSON.stringify(seed));

logger.info('bootstrapSeed file populated with system language hashes')
if (!fs.existsSync(binaryPath)) {
await getAd4mHostBinary(relativePath);
binaryPath = path.join(ad4mDataDirectory(relativePath), 'binary', `ad4m`);
}

kill(child.pid!, async () => {
await findAndKillProcess('holochain')
await findAndKillProcess('lair-keystore')
resolve(null);
})
}
});
await findAndKillProcess('holochain')
await findAndKillProcess('lair-keystore')
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that is also super old. we are not spawning holochain (no lair-keystore) as external processes. all inside the ad4m-executor. these lines should go away.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@HexaField I will merge this now, but please note my comments above for future work on the test-runner.


// Look for bootstrap seed in order of preference:
// 1. Adjacent to package root (../bootstrapSeed.json from build/) — installed package
// 2. In the monorepo (../../tests/js/bootstrapSeed.json from build/) — development
const candidates = [
path.join(__dirname, '../bootstrapSeed.json'),
path.join(__dirname, '../../tests/js/bootstrapSeed.json'),
];

let seedPath: string | null = null;
for (const candidate of candidates) {
if (fs.existsSync(candidate)) {
seedPath = candidate;
break;
}
}

child.on('exit', (code) => {
logger.info(`exit is called ${code}`);
resolve(null);
})
if (!seedPath) {
throw new Error(
`Bootstrap seed not found. Looked in:\n` +
candidates.map(c => ` - ${c}`).join('\n') +
`\nEnsure bootstrapSeed.json exists (copy from tests/js/ or download from the AD4M repo).`
);
}

child.on('error', () => {
logger.error(`process error: ${child.pid}`)
findAndKillProcess('holochain')
findAndKillProcess('lair-keystore')
findAndKillProcess('ad4m')
reject()
});
});
// If the seed is in the monorepo, copy it to the package root for startServer to find
const targetSeedPath = path.join(__dirname, '../bootstrapSeed.json');
if (seedPath !== targetSeedPath) {
fs.copySync(seedPath, targetSeedPath);
logger.info(`Bootstrap seed copied from ${seedPath}`);
} else {
logger.info('Bootstrap seed found at package root');
}
}

if (require.main === module) {
installSystemLanguages().then(() => {
process.exit(0);
}).catch(e => {
console.error(e);
process.exit(1);
});
}