diff --git a/package-lock.json b/package-lock.json index 4dec453..7b436a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,6 +43,7 @@ }, "devDependencies": { "@biomejs/biome": "^1.9.4", + "@iarna/toml": "^2.2.5", "@tauri-apps/cli": "^2.8.4", "@types/node": "^22.10.7", "@types/react": "^18.3.1", @@ -54,7 +55,8 @@ "postcss": "^8.5.6", "tailwindcss": "^3.4.17", "typescript": "~5.9.2", - "vite": "^6.3.6" + "vite": "^6.3.6", + "yaml": "^2.8.1" } }, "node_modules/@alloc/quick-lru": { @@ -2518,6 +2520,13 @@ "react-dom": ">=18 || >=19.0.0-rc.0" } }, + "node_modules/@iarna/toml": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/@iarna/toml/-/toml-2.2.5.tgz", + "integrity": "sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==", + "dev": true, + "license": "ISC" + }, "node_modules/@internationalized/date": { "version": "3.8.2", "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.8.2.tgz", @@ -7886,14 +7895,15 @@ "license": "ISC" }, "node_modules/yaml": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.7.0.tgz", - "integrity": "sha512-+hSoy/QHluxmC9kCIJyL/uyFmLmc+e5CFR5Wa+bpIhIj85LVb9ZH2nVnqrHoSvKogwODv0ClqZkmiSSaIH5LTA==", + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.1.tgz", + "integrity": "sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==", + "license": "ISC", "bin": { "yaml": "bin.mjs" }, "engines": { - "node": ">= 14" + "node": ">= 14.6" } }, "node_modules/zustand": { diff --git a/package.json b/package.json index c6ebf1b..6abd3f4 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "below targets do not work": "echo 'you heard me'", "build:windows:arm": "tauri build --runner cargo-xwin --target aarch64-pc-windows-msvc", "build:linux:arm": "tauri build --target aarch64-unknown-linux-gnu", - "build:wasm": "tauri build --target wasm32-unknown-unknown" + "build:wasm": "tauri build --target wasm32-unknown-unknown", + "sources:cargo": "node scripts/flatpak-cargo-generator.js" }, "dependencies": { "@formkit/auto-animate": "^0.9.0", @@ -54,6 +55,7 @@ }, "devDependencies": { "@biomejs/biome": "^1.9.4", + "@iarna/toml": "^2.2.5", "@tauri-apps/cli": "^2.8.4", "@types/node": "^22.10.7", "@types/react": "^18.3.1", @@ -65,6 +67,7 @@ "postcss": "^8.5.6", "tailwindcss": "^3.4.17", "typescript": "~5.9.2", - "vite": "^6.3.6" + "vite": "^6.3.6", + "yaml": "^2.8.1" } } diff --git a/scripts/flatpak-cargo-generator.js b/scripts/flatpak-cargo-generator.js new file mode 100644 index 0000000..75f0932 --- /dev/null +++ b/scripts/flatpak-cargo-generator.js @@ -0,0 +1,517 @@ +/* +# Flatpak Cargo Generator + +This file contains the Flatpak Cargo generator tool. +The JS version has been created to eliminate the Python dependency. +Both versions provide the same functionality for generating Flatpak build sources from Cargo.lock files. + +### Usage +First, ensure the required dependencies are installed: +```bash +npm install @iarna/toml yaml -D +``` + +Then, run the script and point it to your Cargo.lock file: +```bash +node flatpak-cargo-generator.js path/to/Cargo.lock [options] +``` + +### Options +- `-o, --output ` - Specify output file (default: generated-sources.json) +- `--yaml` - Output as YAML instead of JSON (default: generated-sources.yml) +- `-t, --git-tarballs` - Download git repos as tarballs instead of cloning +- `-d, --debug` - Enable debug logging +- `-h, --help` - Show help message + +### Examples +```bash +# Generate JSON sources +node flatpak-cargo-generator.js ../Cargo.lock + +# Generate YAML sources with custom output +node flatpak-cargo-generator.js ../Cargo.lock --yaml -o my-sources.yml + +# Use git tarballs instead of git clones +node flatpak-cargo-generator.js ../Cargo.lock --git-tarballs +``` +*/ + +import { spawnSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import process from 'node:process' +import { parse as parseToml, stringify as stringifyToml } from '@iarna/toml' +import { stringify as stringifyYaml } from 'yaml' + +const CRATES_IO = 'https://static.crates.io/crates' +const CARGO_HOME = 'cargo' +const CARGO_CRATES = `${CARGO_HOME}/vendor` +const VENDORED_SOURCES = 'vendored-sources' +const GIT_CACHE = 'flatpak-cargo/git' +const COMMIT_LEN = 7 + +let DEBUG = false + +function logInfo(...args) { + console.log(...args) +} + +function logDebug(...args) { + if (DEBUG) console.log(...args) +} + +function canonicalUrl(urlStr) { + const uStr = urlStr.replace('git+https://', 'https://') + const u = new URL(uStr) + u.search = '' + u.hash = '' + if (u.pathname.endsWith('/')) { + u.pathname = u.pathname.replace(/\/+$/, '') + } + if (u.hostname === 'github.com') { + u.protocol = 'https:' + u.pathname = u.pathname.toLowerCase() + } + if (u.pathname.endsWith('.git')) { + u.pathname = u.pathname.slice(0, -4) + } + return u +} + +function getGitTarball(repoUrl, commit) { + const url = canonicalUrl(repoUrl) + const parts = url.pathname.split('/').filter(Boolean) + if (parts.length !== 2) throw new Error('Invalid repo URL: ' + url.toString()) + const owner = parts[0] + const repo = parts[1] + if (url.hostname === 'github.com') { + return `https://codeload.${url.hostname}/${owner}/${repo}/tar.gz/${commit}` + } else if (url.hostname.split('.')[0] === 'gitlab') { + return `https://${url.hostname}/${owner}/${repo}/-/archive/${commit}/${repo}-${commit}.tar.gz` + } else if (url.hostname === 'bitbucket.org') { + return `https://${url.hostname}/${owner}/${repo}/get/${commit}.tar.gz` + } + throw new Error(`Don't know how to get tarball for ${repoUrl}`) +} + +async function getRemoteSha256(url) { + logInfo(`started sha256(${url})`) + const sha256 = createHash('sha256') + const response = await fetch(url) + if (!response.ok) throw new Error(`HTTP ${response.status} for ${url}`) + const reader = response.body.getReader() + while (true) { + const { value, done } = await reader.read() + if (done) break + if (value) sha256.update(value) + } + const digest = sha256.digest('hex') + logInfo(`done sha256(${url})`) + return digest +} + +async function loadToml(tomlFile = 'Cargo.lock') { + const text = await fs.readFile(tomlFile, 'utf-8') + return parseToml(text) +} + +function gitRepoName(gitUrl, commit) { + const name = canonicalUrl(gitUrl).pathname.split('/').filter(Boolean).slice(-1)[0] + return `${name}-${commit.slice(0, COMMIT_LEN)}` +} + +function runGit(args, opts = {}) { + const res = spawnSync('git', args, { cwd: opts.cwd, stdio: 'pipe', encoding: 'utf-8' }) + if (res.status !== 0) { + const msg = res.stderr || res.stdout || `git ${args.join(' ')} failed` + throw new Error(msg) + } + return res.stdout +} + +function fetchGitRepo(gitUrl, commit) { + const repoDirKey = gitUrl.replace('://', '_').replace(/\//g, '_') + const cacheDir = process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache') + const cloneDir = path.join(cacheDir, 'flatpak-cargo', repoDirKey) + const gitDir = path.join(cloneDir, '.git') + + return (async () => { + try { + await fs.stat(gitDir) + } catch { + runGit(['clone', '--depth=1', gitUrl, cloneDir]) + } + + const head = runGit(['rev-parse', 'HEAD'], { cwd: cloneDir }).trim() + if (head.slice(0, COMMIT_LEN) !== commit.slice(0, COMMIT_LEN)) { + runGit(['fetch', 'origin', commit], { cwd: cloneDir }) + try { + runGit(['checkout', commit], { cwd: cloneDir }) + } catch (e) { + logInfo(`Checking out commit ${commit} failed for ${gitUrl}. Forcing checkout.`) + runGit(['checkout', '-f', commit], { cwd: cloneDir }) + } + } + + runGit(['submodule', 'update', '--init', '--recursive'], { cwd: cloneDir }) + return cloneDir + })() +} + +function deepClone(obj) { + return JSON.parse(JSON.stringify(obj)) +} + +function updateWorkspaceKeys(pkg, workspace) { + for (const [key, item] of Object.entries(pkg)) { + if (typeof item !== 'object' || item === null || Array.isArray(item)) continue + + if (key === 'target') { + for (const target of Object.values(item)) { + updateWorkspaceKeys(target, workspace) + } + continue + } else if (key === 'dev-dependencies' || key === 'build-dependencies') { + updateWorkspaceKeys( + item, + workspace && workspace.dependencies ? workspace.dependencies : undefined + ) + continue + } + + if (!workspace || !(key in workspace)) continue + const workspaceItem = workspace[key] + + if (Object.prototype.hasOwnProperty.call(item, 'workspace')) { + if ( + typeof workspaceItem === 'object' && + workspaceItem !== null && + !Array.isArray(workspaceItem) + ) { + delete item.workspace + for (const [depKey, workspaceValue] of Object.entries(workspaceItem)) { + if ( + depKey === 'features' && + Object.prototype.hasOwnProperty.call(item, 'features') + ) { + item.features = [...item.features, ...workspaceValue] + } else { + item[depKey] = workspaceValue + } + } + } else if (Object.keys(item).length > 1) { + delete item.workspace + item.version = workspaceItem + } else { + pkg[key] = workspaceItem + } + } else { + updateWorkspaceKeys(item, workspaceItem) + } + } +} + +async function getGitRepoPackages(gitUrl, commit) { + logInfo(`Loading packages from ${gitUrl}`) + const gitRepoDir = await fetchGitRepo(gitUrl, commit) + const packages = {} + + async function pathExists(p) { + try { + await fs.access(p) + return true + } catch { + return false + } + } + + async function getCargoTomlPackages(absRoot, relRoot = '.', workspace) { + const cargoTomlPath = path.join(absRoot, 'Cargo.toml') + if (await pathExists(cargoTomlPath)) { + const cargoToml = parseToml(await fs.readFile(cargoTomlPath, 'utf-8')) + const currentWorkspace = cargoToml.workspace || workspace + if (cargoToml.package) { + const posixRel = relRoot.split(path.sep).join('/') + packages[cargoToml.package.name] = { + path: posixRel, + package: cargoToml, + workspace: currentWorkspace, + } + } + workspace = currentWorkspace + } + + const dirents = await fs.readdir(absRoot, { withFileTypes: true }) + for (const d of dirents) { + if (d.isDirectory()) { + await getCargoTomlPackages( + path.join(absRoot, d.name), + path.join(relRoot, d.name), + workspace + ) + } + } + } + + await getCargoTomlPackages(gitRepoDir, '.') + if (Object.keys(packages).length === 0) { + throw new Error(`No packages found in ${gitRepoDir}`) + } + logDebug( + 'Packages in', + gitUrl, + JSON.stringify( + Object.fromEntries(Object.entries(packages).map(([k, v]) => [k, v.path])), + null, + 4 + ) + ) + return packages +} + +const repoPackagesCache = new Map() + +function getGitRepoPackagesCached(repoUrl, commit) { + const key = `${repoUrl}#${commit}` + if (!repoPackagesCache.has(key)) { + repoPackagesCache.set(key, getGitRepoPackages(repoUrl, commit)) + } + return repoPackagesCache.get(key) +} + +async function getGitRepoSources(url, commit, tarball = false) { + const name = gitRepoName(url, commit) + if (tarball) { + const tarballUrl = getGitTarball(url, commit) + return [ + { + type: 'archive', + 'archive-type': 'tar-gzip', + url: tarballUrl, + sha256: await getRemoteSha256(tarballUrl), + dest: `${GIT_CACHE}/${name}`, + }, + ] + } + return [ + { + type: 'git', + url, + commit, + dest: `${GIT_CACHE}/${name}`, + }, + ] +} + +async function getGitPackageSources(pkg, gitRepoCommits) { + const name = pkg.name + const source = pkg.source + const u = new URL(source.replace('git+https://', 'https://')) + const commit = u.hash ? u.hash.slice(1) : '' + if (!commit) throw new Error('The commit needs to be indicated in the fragment part') + const canonical = canonicalUrl(source) + const repoUrl = canonical.toString() + + if (!gitRepoCommits.has(repoUrl)) gitRepoCommits.set(repoUrl, new Set()) + gitRepoCommits.get(repoUrl).add(commit) + + const packages = await getGitRepoPackagesCached(repoUrl, commit) + const gitPkg = packages[name] + if (!gitPkg) throw new Error(`Package ${name} not found in repo ${repoUrl}@${commit}`) + + const normalizedPkg = (() => { + const cloned = deepClone(gitPkg.package) + if (gitPkg.workspace) updateWorkspaceKeys(cloned, gitPkg.workspace) + return cloned + })() + + const pkgRepoDir = path.posix.join(GIT_CACHE, gitRepoName(repoUrl, commit), gitPkg.path) + + const gitSources = [ + { + type: 'shell', + commands: [`cp -r --reflink=auto "${pkgRepoDir}" "${CARGO_CRATES}/${name}"`], + }, + { + type: 'inline', + contents: stringifyToml(normalizedPkg), + dest: `${CARGO_CRATES}/${name}`, + 'dest-filename': 'Cargo.toml', + }, + { + type: 'inline', + contents: JSON.stringify({ package: null, files: {} }), + dest: `${CARGO_CRATES}/${name}`, + 'dest-filename': '.cargo-checksum.json', + }, + ] + + const cargoVendoredEntry = { + [repoUrl]: { + git: repoUrl, + 'replace-with': VENDORED_SOURCES, + }, + } + const rev = u.searchParams.get('rev') + const tag = u.searchParams.get('tag') + const branch = u.searchParams.get('branch') + if (rev) cargoVendoredEntry[repoUrl].rev = rev + else if (tag) cargoVendoredEntry[repoUrl].tag = tag + else if (branch) cargoVendoredEntry[repoUrl].branch = branch + + logInfo(`Adding package ${name} from ${repoUrl}`) + return [gitSources, cargoVendoredEntry] +} + +async function getPackageSources(pkg, cargoLock, gitRepoCommits) { + const metadata = cargoLock.metadata + const name = pkg.name + const version = pkg.version + + if (!Object.prototype.hasOwnProperty.call(pkg, 'source')) { + logDebug(`${name} has no source`) + return null + } + const source = pkg.source + + if (source.startsWith('git+')) { + return await getGitPackageSources(pkg, gitRepoCommits) + } + + const key = `checksum ${name} ${version} (${source})` + let checksum + if (metadata && Object.prototype.hasOwnProperty.call(metadata, key)) { + checksum = metadata[key] + } else if (Object.prototype.hasOwnProperty.call(pkg, 'checksum')) { + checksum = pkg.checksum + } else { + console.warn(`${name} doesn't have checksum`) + return null + } + + const crateSources = [ + { + type: 'archive', + 'archive-type': 'tar-gzip', + url: `${CRATES_IO}/${name}/${name}-${version}.crate`, + sha256: checksum, + dest: `${CARGO_CRATES}/${name}-${version}`, + }, + { + type: 'inline', + contents: JSON.stringify({ package: checksum, files: {} }), + dest: `${CARGO_CRATES}/${name}-${version}`, + 'dest-filename': '.cargo-checksum.json', + }, + ] + return [crateSources, { 'crates-io': { 'replace-with': VENDORED_SOURCES } }] +} + +async function generateSources(cargoLock, gitTarballs = false) { + const gitRepoCommits = new Map() + const sources = [] + const packageSources = [] + const cargoVendoredSources = { + [VENDORED_SOURCES]: { directory: `${CARGO_CRATES}` }, + } + + const pkgPromises = cargoLock.package.map((p) => + getPackageSources(p, cargoLock, gitRepoCommits) + ) + const pkgResults = await Promise.all(pkgPromises) + for (const pkg of pkgResults) { + if (!pkg) continue + const [pkgSrc, cargoVendoredEntry] = pkg + packageSources.push(...pkgSrc) + Object.assign(cargoVendoredSources, cargoVendoredEntry) + } + + logDebug( + 'Adding collected git repos:\n' + JSON.stringify(Array.from(gitRepoCommits.keys()), null, 4) + ) + const gitRepoCoros = [] + for (const [gitUrl, commits] of gitRepoCommits.entries()) { + for (const commit of commits) { + gitRepoCoros.push(getGitRepoSources(gitUrl, commit, gitTarballs)) + } + } + const gitRepoSourcesNested = await Promise.all(gitRepoCoros) + for (const arr of gitRepoSourcesNested) { + sources.push(...arr) + } + + sources.push(...packageSources) + + logDebug('Vendored sources:\n' + JSON.stringify(cargoVendoredSources, null, 4)) + sources.push({ + type: 'inline', + contents: stringifyToml({ source: cargoVendoredSources }), + dest: CARGO_HOME, + 'dest-filename': 'config', + }) + return sources +} + +function printUsageAndExit() { + console.error( + 'Usage: node flatpak-cargo-generator.js [-o OUT] [--yaml] [-t|--git-tarballs] [-d|--debug]' + ) + process.exit(1) +} + +async function main() { + const argv = process.argv.slice(2) + let output = null + let yaml = false + let gitTarballs = false + let debug = false + let cargoLockPath = null + + for (let i = 0; i < argv.length; i++) { + const a = argv[i] + if (a === '-o' || a === '--output') { + i++ + output = argv[i] + } else if (a === '--yaml') { + yaml = true + } else if (a === '-t' || a === '--git-tarballs') { + gitTarballs = true + } else if (a === '-d' || a === '--debug') { + debug = true + } else if (!a.startsWith('-') && !cargoLockPath) { + cargoLockPath = a + } else { + printUsageAndExit() + } + } + + if (!cargoLockPath) printUsageAndExit() + + DEBUG = debug + const cargoLock = await loadToml(cargoLockPath) + + let outfile + if (output) outfile = output + else if (yaml) outfile = 'generated-sources.yml' + else outfile = 'generated-sources.json' + + const generatedSources = await generateSources(cargoLock, gitTarballs) + + if (yaml) { + await fs.writeFile(outfile, stringifyYaml(generatedSources)) + } else { + await fs.writeFile(outfile, JSON.stringify(generatedSources, null, 4)) + } +} + +if ( + import.meta.url === `file://${process.argv[1]}` || + process.argv[1].endsWith('flatpak-cargo-generator.js') +) { + main().catch((err) => { + console.error(err) + process.exit(1) + }) +}