From c7899b56c240e22cffdf2ab5b3b98183544d5b82 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 26 Nov 2025 15:49:10 +0000 Subject: [PATCH 001/108] Added using Dependency Review on branches --- README.md | 26 ++++++ package-lock.json | 7 ++ package.json | 4 +- src/cli.ts | 8 ++ src/sbomCollector.ts | 182 +++++++++++++++++++++++++++++++++++++- src/test-branch-search.ts | 88 ++++++++++++++++++ src/types.ts | 36 ++++++++ 7 files changed, 349 insertions(+), 2 deletions(-) create mode 100644 src/test-branch-search.ts diff --git a/README.md b/README.md index 4db1493..51de3cc 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ Supports human-readable, JSON, CSV and SARIF output. SARIF alerts can be uploade - JSON or CSV output (to stdout or file) with both search and malware matches - Optional SARIF 2.1.0 output per repository for malware matches with optional Code Scanning upload - Works with GitHub.com, GitHub Enterprise Server, GitHub Enterprise Managed Users and GitHub Enterprise Cloud with Data Residency (custom base URL) +- Optional branch scanning: fetch SBOMs for non-default branches (limited) and compute Dependency Review diffs vs the default (or chosen base) branch ## Usage @@ -55,6 +56,31 @@ Using GitHub Enterprise Server: npm run start -- --sync-sboms --enterprise ent --base-url https://github.internal/api/v3 --sbom-cache sboms --token $GHES_TOKEN ``` +### 🔀 Branch Scanning & Dependency Review + +Enable branch SBOM collection and dependency diffs with `--branch-scan`. + +Flags: + +```bash +--branch-scan # Fetch SBOMs for non-default branches +--branch-limit # Max number of non-default branches per repo (default 10) +--dependency-review # Fetch dependency review diffs (enabled by default) +--diff-base # Override base branch for diffs (default: repository default) +``` + +Example: scan first 5 feature branches and diff them against `main`: + +```bash +npm run start -- --sync-sboms --org my-org \ + --sbom-cache sboms --branch-scan --branch-limit 5 \ + --diff-base main --token $GITHUB_TOKEN +``` + +Search results will include branch matches: package PURLs annotated with `@branch` inside the match list (e.g. `pkg:npm/react@18.3.0@feature-x`). Dependency Review additions / updates are also searched; only added/updated head-side packages are considered. + +If a branch SBOM or diff retrieval fails, the error is recorded but does not stop collection for other branches or repositories. + ### 🔑 Authentication A GitHub token with appropriate scope is required when performing network operations such as `--sync-sboms`, `--sync-malware` and `--upload-sarif`. diff --git a/package-lock.json b/package-lock.json index d036a77..1fa3460 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1099,6 +1099,7 @@ "resolved": "https://registry.npmjs.org/@octokit/core/-/core-7.0.6.tgz", "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "license": "MIT", + "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -1381,6 +1382,7 @@ "integrity": "sha512-qzQZRBqkFsYyaSWXuEHc2WR9c0a0CXwiE5FWUvn7ZM+vdy1uZLfCunD38UzhuB7YN/J11ndbDBcTmOdxJo9Q7A==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -1465,6 +1467,7 @@ "integrity": "sha512-jCzKdm/QK0Kg4V4IK/oMlRZlY+QOcdjv89U2NgKHZk1CYTj82/RVSx1mV/0gqCVMJ/DA+Zf/S4NBWNF8GQ+eqQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.48.0", "@typescript-eslint/types": "8.48.0", @@ -1941,6 +1944,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2319,6 +2323,7 @@ "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -3193,6 +3198,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -3507,6 +3513,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/package.json b/package.json index e9f211c..218991b 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,9 @@ "start": "node dist/cli.js", "dev": "tsx src/cli.ts", "lint": "eslint . --ext .ts --max-warnings=0", - "test": "node dist/test-fixture-match.js" + "test": "node dist/test-fixture-match.js", + "test:branch-search": "node dist/test-branch-search.js" + }, "engines": { "node": ">=18.0.0" diff --git a/src/cli.ts b/src/cli.ts index d45450a..da7c161 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -39,6 +39,10 @@ async function main() { .option("csv", { type: "boolean", describe: "Emit results (search + malware matches) as CSV" }) .option("ignore-file", { type: "string", describe: "Path to YAML ignore file (advisories, purls, scoped ignores)" }) .option("ignore-unbounded-malware", { type: "boolean", default: false, describe: "Ignore malware advisories whose vulnerable range covers all versions (e.g. '*', '>=0')" }) + .option("branch-scan", { type: "boolean", default: false, describe: "Fetch SBOMs for non-default branches (limited by --branch-limit)" }) + .option("branch-limit", { type: "number", default: 10, describe: "Limit number of non-default branches to scan per repository" }) + .option("dependency-review", { type: "boolean", default: true, describe: "Fetch dependency review diffs for scanned branches" }) + .option("diff-base", { type: "string", describe: "Override base branch for dependency review diffs (defaults to default branch)" }) .check(args => { const syncing = !!args.syncSboms; if (syncing) { @@ -102,6 +106,10 @@ async function main() { suppressSecondaryRateLimitLogs: argv.suppressSecondaryRateLimitLogs as boolean, quiet, caBundlePath: argv["ca-bundle"] as string | undefined, + includeBranches: argv["branch-scan"] as boolean, + branchLimit: argv["branch-limit"] as number, + includeDependencyReviewDiffs: argv["dependency-review"] as boolean, + branchDiffBase: argv["diff-base"] as string | undefined, }); if (!quiet) process.stderr.write(chalk.cyan(offline ? "Loading SBOMs from cache..." : "Collecting SBOMs from cache & GitHub...") + "\n"); diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 13020a8..de576de 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -1,5 +1,5 @@ import { createOctokit } from "./octokit.js"; -import type { RepositorySbom, CollectionSummary, SbomPackage, Sbom } from "./types.js"; +import type { RepositorySbom, CollectionSummary, SbomPackage, Sbom, BranchSbom, BranchDependencyDiff, DependencyReviewPackageChange } from "./types.js"; import * as semver from "semver"; import { readAll, writeOne } from "./serialization.js"; // p-limit lacks bundled types in some versions; declare minimal shape @@ -24,6 +24,10 @@ export interface CollectorOptions { suppressSecondaryRateLimitLogs?: boolean; // suppress secondary rate limit warning logs (so they don't break the progress bar) quiet?: boolean; // suppress non-error logging (does not affect progress bar) caBundlePath?: string; // path to PEM CA bundle for self-signed/internal certs + includeBranches?: boolean; // when true, fetch SBOM for non-default branches + branchLimit?: number; // limit number of branches per repo (excluding default) + includeDependencyReviewDiffs?: boolean; // fetch dependency review diff base->branch + branchDiffBase?: string; // override base branch for diffs (defaults to default branch) } export class SbomCollector { @@ -57,6 +61,10 @@ export class SbomCollector { suppressSecondaryRateLimitLogs: o.suppressSecondaryRateLimitLogs ?? false, quiet: o.quiet ?? false, caBundlePath: o.caBundlePath + ,includeBranches: o.includeBranches ?? false + ,branchLimit: o.branchLimit ?? 20 + ,includeDependencyReviewDiffs: o.includeDependencyReviewDiffs ?? true + ,branchDiffBase: o.branchDiffBase } as Required; if (this.opts.token) { @@ -233,6 +241,32 @@ export class SbomCollector { res.defaultBranchCommitSha = pendingCommitMeta.sha; res.defaultBranchCommitDate = pendingCommitMeta.date; } + // Branch scanning (optional) + if (this.opts.includeBranches && res.defaultBranch) { + try { + const branches = await this.listBranches(org, repo.name); + const nonDefault = branches.filter(b => b.name !== res.defaultBranch); + const limited = this.opts.branchLimit && this.opts.branchLimit > 0 ? nonDefault.slice(0, this.opts.branchLimit) : nonDefault; + const branchSboms: BranchSbom[] = []; + const branchDiffs: BranchDependencyDiff[] = []; + for (const b of limited) { + if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); + const bSbom = await this.fetchBranchSbom(org, repo.name, b.name, b.commit?.sha); + branchSboms.push(bSbom); + const base = this.opts.branchDiffBase || res.defaultBranch; + if (this.opts.includeDependencyReviewDiffs) { + if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); + const diff = await this.fetchDependencyReviewDiff(org, repo.name, base, b.name); + branchDiffs.push(diff); + } + } + if (branchSboms.length) res.branchSboms = branchSboms; + if (branchDiffs.length) res.branchDiffs = branchDiffs; + } catch (e) { + // Non-fatal; annotate decision + this.decisions[fullName] = (this.decisions[fullName] || "") + ` (branch scan error: ${(e as Error).message})`; + } + } newSboms.push(res); if (res.error) this.summary.failedCount++; else this.summary.successCount++; // Write freshly fetched SBOM immediately if a cache directory is configured @@ -340,6 +374,69 @@ export class SbomCollector { } } + private async listBranches(org: string, repo: string): Promise<{ name: string; protected?: boolean; commit?: { sha?: string } }[]> { + if (!this.octokit) throw new Error("No Octokit instance"); + const branches: { name: string; protected?: boolean; commit?: { sha?: string } }[] = []; + const per_page = 100; let page = 1; let done = false; + while (!done) { + try { + const resp = await this.octokit.request("GET /repos/{owner}/{repo}/branches", { owner: org, repo, per_page, page }); + const data = resp.data as Array<{ name: string; protected?: boolean; commit?: { sha?: string } }>; + branches.push(...data); + if (data.length < per_page) done = true; else page++; + } catch (e) { + throw new Error(`Failed listing branches for ${org}/${repo}: ${(e as Error).message}`); + } + } + return branches; + } + + private async fetchBranchSbom(org: string, repo: string, branch: string, commitSha?: string): Promise { + if (!this.octokit) throw new Error("No Octokit instance"); + try { + const resp = await this.octokit.request("GET /repos/{owner}/{repo}/dependency-graph/sbom", { owner: org, repo, ref: branch, headers: { Accept: "application/vnd.github+json" } }); + const sbomWrapper = resp.data as { sbom?: Sbom }; + const packages: SbomPackage[] = sbomWrapper?.sbom?.packages ?? []; + return { branch, commitSha, retrievedAt: new Date().toISOString(), sbom: sbomWrapper?.sbom, packages }; + } catch (e) { + return { branch, commitSha, retrievedAt: new Date().toISOString(), packages: [], error: e instanceof Error ? e.message : String(e) }; + } + } + + private async fetchDependencyReviewDiff(org: string, repo: string, base: string, head: string): Promise { + if (!this.octokit) throw new Error("No Octokit instance"); + try { + const resp = await this.octokit.request("GET /repos/{owner}/{repo}/dependency-graph/dependency-review", { owner: org, repo, base, head, headers: { Accept: "application/vnd.github+json" } }); + // Response shape includes change_set array (per docs). We normalize to DependencyReviewPackageChange[] + const raw = resp.data as { change_set?: unknown[] }; + const changes: DependencyReviewPackageChange[] = []; + for (const c of raw.change_set ?? []) { + const obj = c as Record; + const change: DependencyReviewPackageChange = { + changeType: String(obj.change_type || obj.changeType || "unknown"), + name: obj.name as string | undefined, + ecosystem: obj.ecosystem as string | undefined, + packageURL: obj.package_url as string | undefined, + purl: obj.purl as string | undefined, + license: obj.license as string | undefined, + manifest: obj.manifest as string | undefined, + scope: obj.scope as string | undefined, + previousVersion: obj.previous_version as string | undefined, + newVersion: obj.version as string | undefined + }; + changes.push(change); + } + return { base, head, retrievedAt: new Date().toISOString(), changes }; + } catch (e) { + const status = (e as { status?: number })?.status; + let reason = e instanceof Error ? e.message : String(e); + if (status === 404) { + reason = "Dependency review unavailable (missing snapshot or feature disabled)"; + } + return { base, head, retrievedAt: new Date().toISOString(), changes: [], error: reason }; + } + } + // New method including the query that produced each match searchByPurlsWithReasons(purls: string[]): Map { purls = purls.map(q => q.startsWith("pkg:") ? q : `pkg:${q}`); @@ -421,6 +518,89 @@ export class SbomCollector { } } } + // Include branch SBOM packages + if (repoSbom.branchSboms) { + for (const b of repoSbom.branchSboms) { + if (b.error) continue; + for (const pkg of b.packages as Array) { + const refs = (pkg as { externalRefs?: ExtRef[] }).externalRefs; + const candidatePurls: string[] = []; + if (refs) for (const r of refs) if (r.referenceType === "purl" && r.referenceLocator) candidatePurls.push(r.referenceLocator); + if ((pkg as { purl?: string }).purl) candidatePurls.push((pkg as { purl?: string }).purl as string); + const unique = Array.from(new Set(candidatePurls)); + for (const p of unique) { + const pLower = p.toLowerCase(); + for (const q of queries) { + if (q.isPrefixWildcard) { + const prefix = q.lower.slice(0, -1); + if (pLower.startsWith(prefix)) { if (!found.has(`${p}@${b.branch}`)) found.set(`${p}@${b.branch}`, q.raw); } + continue; + } + if (q.versionConstraint && q.type && q.name) { + if (!pLower.startsWith("pkg:")) continue; + const body = p.slice(4); + const atIdx = body.indexOf("@"); + const main = atIdx >= 0 ? body.slice(0, atIdx) : body; + const ver = atIdx >= 0 ? body.slice(atIdx + 1) : (pkg.version as string | undefined) || undefined; + const slashIdx = main.indexOf("/"); + if (slashIdx < 0) continue; + const pType = main.slice(0, slashIdx).toLowerCase(); + const pName = main.slice(slashIdx + 1); + if (pType === q.type && pName.toLowerCase() === q.name.toLowerCase() && ver) { + try { + const coerced = semver.coerce(ver)?.version || ver; + if (semver.valid(coerced) && semver.satisfies(coerced, q.versionConstraint, { includePrerelease: true })) { + if (!found.has(`${p}@${b.branch}`)) found.set(`${p}@${b.branch}`, q.raw); + } + } catch { /* ignore */ } + } + } else if (q.exact) { + if (pLower === q.exact) { if (!found.has(`${p}@${b.branch}`)) found.set(`${p}@${b.branch}`, q.raw); } + } + } + } + } + } + } + // Include dependency review diff additions/updates (head packages only) + if (repoSbom.branchDiffs) { + for (const diff of repoSbom.branchDiffs) { + for (const change of diff.changes) { + if (change.changeType !== "added" && change.changeType !== "updated") continue; + const p = (change.purl || change.packageURL || (change.ecosystem ? `pkg:${change.ecosystem}/${change.name || ""}${change.newVersion ? "@" + change.newVersion : ""}` : undefined)); + if (!p) continue; + const pLower = p.toLowerCase(); + for (const q of queries) { + if (q.isPrefixWildcard) { + const prefix = q.lower.slice(0, -1); + if (pLower.startsWith(prefix)) { if (!found.has(`${p}@${diff.head}`)) found.set(`${p}@${diff.head}`, q.raw); } + continue; + } + if (q.versionConstraint && q.type && q.name) { + if (!pLower.startsWith("pkg:")) continue; + const body = p.slice(4); + const atIdx = body.indexOf("@"); + const main = atIdx >= 0 ? body.slice(0, atIdx) : body; + const ver = atIdx >= 0 ? body.slice(atIdx + 1) : change.newVersion; + const slashIdx = main.indexOf("/"); + if (slashIdx < 0) continue; + const pType = main.slice(0, slashIdx).toLowerCase(); + const pName = main.slice(slashIdx + 1); + if (pType === q.type && pName.toLowerCase() === q.name.toLowerCase() && ver) { + try { + const coerced = semver.coerce(ver)?.version || ver; + if (semver.valid(coerced) && semver.satisfies(coerced, q.versionConstraint, { includePrerelease: true })) { + if (!found.has(`${p}@${diff.head}`)) found.set(`${p}@${diff.head}`, q.raw); + } + } catch { /* ignore */ } + } + } else if (q.exact) { + if (pLower === q.exact) { if (!found.has(`${p}@${diff.head}`)) found.set(`${p}@${diff.head}`, q.raw); } + } + } + } + } + } if (found.size) results.set(repoSbom.repo, Array.from(found.entries()).map(([purl, reason]) => ({ purl, reason }))); } return results; diff --git a/src/test-branch-search.ts b/src/test-branch-search.ts new file mode 100644 index 0000000..446a738 --- /dev/null +++ b/src/test-branch-search.ts @@ -0,0 +1,88 @@ +import fs from 'fs'; +import path from 'path'; +import { SbomCollector } from './sbomCollector.js'; +import type { RepositorySbom } from './types.js'; + +// This test harness validates that branch SBOMs and dependency review diffs +// participate in search results. It constructs a synthetic repo SBOM object, +// writes it to a temp cache directory, then performs searches. + +async function main() { + const tempRoot = path.join(process.cwd(), 'tmp-branch-search-cache'); + const org = 'example-org'; + const repo = 'demo-repo'; + const repoDir = path.join(tempRoot, org, repo); + fs.rmSync(tempRoot, { recursive: true, force: true }); + fs.mkdirSync(repoDir, { recursive: true }); + + const basePackages = [ + { name: 'chalk', version: '5.6.1', purl: 'pkg:npm/chalk@5.6.1' }, + { name: 'react', version: '18.2.0', purl: 'pkg:npm/react@18.2.0' } + ]; + const featurePackages = [ + { name: 'react', version: '18.3.0-beta', purl: 'pkg:npm/react@18.3.0-beta' }, + { name: 'lodash', version: '4.17.21', purl: 'pkg:npm/lodash@4.17.21' } + ]; + const diffChanges = [ + { changeType: 'added', name: 'lodash', ecosystem: 'npm', purl: 'pkg:npm/lodash@4.17.21', newVersion: '4.17.21' }, + { changeType: 'updated', name: 'react', ecosystem: 'npm', purl: 'pkg:npm/react@18.3.0-beta', previousVersion: '18.2.0', newVersion: '18.3.0-beta' } + ]; + + const synthetic: RepositorySbom = { + repo: `${org}/${repo}`, + org, + retrievedAt: new Date().toISOString(), + packages: basePackages, + branchSboms: [ + { + branch: 'feature-x', + retrievedAt: new Date().toISOString(), + packages: featurePackages + } + ], + branchDiffs: [ + { + base: 'main', + head: 'feature-x', + retrievedAt: new Date().toISOString(), + changes: diffChanges + } + ] + }; + + fs.writeFileSync(path.join(repoDir, 'sbom.json'), JSON.stringify(synthetic, null, 2), 'utf8'); + + const collector = new SbomCollector({ + token: undefined, + org, + loadFromDir: tempRoot, + syncSboms: false, + quiet: true + }); + await collector.collect(); + + const queries = [ + 'pkg:npm/react@>=18.2.0 <19.0.0', // should match base & branch updated version + 'pkg:npm/lodash@4.17.21', // should match added in branch diff & branch SBOM + 'pkg:npm/chalk@5.6.1' // base only + ]; + const results = collector.searchByPurlsWithReasons(queries); + + if (!results.size) { + console.error('No search results found; expected matches from branch data'); + process.exit(1); + } + const entries = results.get(`${org}/${repo}`); + if (!entries || entries.length < 4) { + console.error(`Unexpected number of matches: ${(entries || []).length}`); + console.error(JSON.stringify(entries, null, 2)); + process.exit(1); + } + + process.stdout.write('Branch search test passed. Matches:\n'); + for (const e of entries) { + process.stdout.write(` ${e.purl} {query: ${e.reason}}\n`); + } +} + +main().catch(e => { console.error(e); process.exit(1); }); diff --git a/src/types.ts b/src/types.ts index 3d1a62b..16ffcdf 100644 --- a/src/types.ts +++ b/src/types.ts @@ -77,6 +77,9 @@ export interface RepositorySbom { etag?: string; // ETag from SBOM response (future: conditional requests) defaultBranchCommitSha?: string; // commit SHA of default branch at time of retrieval defaultBranchCommitDate?: string; // ISO date of that commit + // Branch-level SBOMs & diffs (optional when branch scanning enabled) + branchSboms?: BranchSbom[]; + branchDiffs?: BranchDependencyDiff[]; } export interface CollectionSummary { @@ -94,3 +97,36 @@ export interface SearchResultEntry { repository: string; matches: SbomPackage[]; } + +// Branch-specific SBOM capture +export interface BranchSbom { + branch: string; + commitSha?: string; + retrievedAt: string; + sbom?: Sbom; + packages: SbomPackage[]; + error?: string; +} + +// Dependency Review change format (subset; future-proof with index signature) +export interface DependencyReviewPackageChange { + changeType: string; // added | removed | updated + name?: string; // package name + ecosystem?: string; // e.g. npm, maven, pip + packageURL?: string; // raw package URL (may be purl-like) + purl?: string; // normalized purl (if derivable) + license?: string; + manifest?: string; // manifest path + scope?: string; // e.g. runtime, development + previousVersion?: string; // for updated/removed + newVersion?: string; // for added/updated + [k: string]: unknown; +} + +export interface BranchDependencyDiff { + base: string; // base branch + head: string; // head branch + retrievedAt: string; + changes: DependencyReviewPackageChange[]; + error?: string; +} From d2a6b9fd15ff9eecd806764416a44e2197ab5140 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 26 Nov 2025 17:03:14 +0000 Subject: [PATCH 002/108] Added Dependency Submission with Component Detection --- .gitmodules | 3 ++ README.md | 30 +++++++++++ ...ent-detection-dependency-submission-action | 1 + src/cli.ts | 2 + src/componentSubmission.ts | 52 +++++++++++++++++++ src/sbomCollector.ts | 28 ++++++++++ 6 files changed, 116 insertions(+) create mode 100644 .gitmodules create mode 160000 component-detection-dependency-submission-action create mode 100644 src/componentSubmission.ts diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..3ef4986 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "component-detection-dependency-submission-action"] + path = component-detection-dependency-submission-action + url = https://github.com/advanced-security/component-detection-dependency-submission-action.git diff --git a/README.md b/README.md index 51de3cc..819db7e 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,36 @@ Search results will include branch matches: package PURLs annotated with `@branc If a branch SBOM or diff retrieval fails, the error is recorded but does not stop collection for other branches or repositories. +#### Handling Missing Dependency Review Snapshots + +If the Dependency Review API returns a 404 for a branch diff (commonly due to a missing dependency snapshot on either the base or head commit), the toolkit can optionally attempt to generate and submit a snapshot using the Component Detection + Dependency Submission Action. + +Enable automatic submission + retry with: + +```bash +--submit-on-missing-snapshot +``` + +This requires the action repository to be present as a git submodule (or copied) at the path: + +``` +component-detection-dependency-submission-action/ +``` + +After cloning, initialize submodules: + +```bash +git submodule update --init --recursive +``` + +Build the action (if not already built) so its `dist/entrypoint.js` exists. The toolkit will then: + +1. Detect 404 from diff endpoint. +2. Invoke the action locally to produce a snapshot for the target branch. +3. Wait briefly then retry the dependency review diff once. + +If submission fails, the original 404 reason is retained and collection proceeds. + ### 🔑 Authentication A GitHub token with appropriate scope is required when performing network operations such as `--sync-sboms`, `--sync-malware` and `--upload-sarif`. diff --git a/component-detection-dependency-submission-action b/component-detection-dependency-submission-action new file mode 160000 index 0000000..51ff88a --- /dev/null +++ b/component-detection-dependency-submission-action @@ -0,0 +1 @@ +Subproject commit 51ff88ad105a2df32251ba5f361412b753f9a27d diff --git a/src/cli.ts b/src/cli.ts index da7c161..661c418 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -43,6 +43,7 @@ async function main() { .option("branch-limit", { type: "number", default: 10, describe: "Limit number of non-default branches to scan per repository" }) .option("dependency-review", { type: "boolean", default: true, describe: "Fetch dependency review diffs for scanned branches" }) .option("diff-base", { type: "string", describe: "Override base branch for dependency review diffs (defaults to default branch)" }) + .option("submit-on-missing-snapshot", { type: "boolean", default: false, describe: "When dependency review diff returns 404 (missing snapshot), run Component Detection to submit a snapshot, then retry." }) .check(args => { const syncing = !!args.syncSboms; if (syncing) { @@ -110,6 +111,7 @@ async function main() { branchLimit: argv["branch-limit"] as number, includeDependencyReviewDiffs: argv["dependency-review"] as boolean, branchDiffBase: argv["diff-base"] as string | undefined, + submitOnMissingSnapshot: argv["submit-on-missing-snapshot"] as boolean, }); if (!quiet) process.stderr.write(chalk.cyan(offline ? "Loading SBOMs from cache..." : "Collecting SBOMs from cache & GitHub...") + "\n"); diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts new file mode 100644 index 0000000..e328be9 --- /dev/null +++ b/src/componentSubmission.ts @@ -0,0 +1,52 @@ +import chalk from 'chalk'; +import { spawn } from 'child_process'; +import path from 'path'; + +export interface SubmitOpts { + owner: string; + repo: string; + branch: string; + token?: string; + baseUrl?: string; + caBundlePath?: string; + quiet?: boolean; +} + +// This helper attempts to run the Component Detection + Dependency Submission action +// as a local script, assuming the repository has the submodule checked out at +// `component-detection-dependency-submission-action`. +// It falls back to returning false if not available. +export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise { + const root = process.cwd(); + const actionDir = path.join(root, 'component-detection-dependency-submission-action'); + const entry = path.join(actionDir, 'dist', 'entrypoint.js'); + // Minimal validation: require entrypoint to exist + try { + const fs = await import('fs'); + if (!fs.existsSync(entry)) { + if (!opts.quiet) console.error(chalk.yellow('Component Detection action not found; ensure submodule initialized and built.')); + return false; + } + } catch { + return false; + } + // Run the entrypoint with necessary env vars + const env = { + ...process.env, + GITHUB_TOKEN: opts.token || process.env.GITHUB_TOKEN || '', + GITHUB_BASE_URL: opts.baseUrl || process.env.GITHUB_BASE_URL || '', + TARGET_OWNER: opts.owner, + TARGET_REPO: opts.repo, + TARGET_REF: opts.branch + }; + if (!env.GITHUB_TOKEN) { + if (!opts.quiet) console.error(chalk.red('GITHUB_TOKEN required to submit dependency snapshot')); + return false; + } + await new Promise((resolve, reject) => { + const child = spawn('node', [entry], { env, stdio: opts.quiet ? 'ignore' : 'inherit' }); + child.on('error', reject); + child.on('exit', code => code === 0 ? resolve() : reject(new Error(`entrypoint exit ${code}`))); + }); + return true; +} diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index de576de..132c772 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -1,5 +1,6 @@ import { createOctokit } from "./octokit.js"; import type { RepositorySbom, CollectionSummary, SbomPackage, Sbom, BranchSbom, BranchDependencyDiff, DependencyReviewPackageChange } from "./types.js"; +import { submitSnapshotIfPossible } from "./componentSubmission.js"; import * as semver from "semver"; import { readAll, writeOne } from "./serialization.js"; // p-limit lacks bundled types in some versions; declare minimal shape @@ -28,6 +29,7 @@ export interface CollectorOptions { branchLimit?: number; // limit number of branches per repo (excluding default) includeDependencyReviewDiffs?: boolean; // fetch dependency review diff base->branch branchDiffBase?: string; // override base branch for diffs (defaults to default branch) + submitOnMissingSnapshot?: boolean; // run component detection submission when diff 404 } export class SbomCollector { @@ -65,6 +67,7 @@ export class SbomCollector { ,branchLimit: o.branchLimit ?? 20 ,includeDependencyReviewDiffs: o.includeDependencyReviewDiffs ?? true ,branchDiffBase: o.branchDiffBase + ,submitOnMissingSnapshot: o.submitOnMissingSnapshot ?? false } as Required; if (this.opts.token) { @@ -432,11 +435,36 @@ export class SbomCollector { let reason = e instanceof Error ? e.message : String(e); if (status === 404) { reason = "Dependency review unavailable (missing snapshot or feature disabled)"; + // Optional retry path: submit snapshot then retry once + if (this.opts.submitOnMissingSnapshot) { + try { + const ok = await this.trySubmitSnapshot(org, repo, head); + if (ok) { + await new Promise(r => setTimeout(r, 3000)); + return await this.fetchDependencyReviewDiff(org, repo, base, head); + } + } catch (subErr) { + reason += ` (submission attempt failed: ${(subErr as Error).message})`; + } + } } return { base, head, retrievedAt: new Date().toISOString(), changes: [], error: reason }; } } + private async trySubmitSnapshot(org: string, repo: string, branch: string): Promise { + // Dynamically import to avoid hard dependency when submodule not present + try { + const mod = await import("./componentSubmission.js"); + if (typeof mod.submitSnapshotIfPossible === "function") { + return await mod.submitSnapshotIfPossible({ owner: org, repo, branch, token: this.opts.token, baseUrl: this.opts.baseUrl, caBundlePath: this.opts.caBundlePath, quiet: this.opts.quiet }); + } + return false; + } catch { + return false; + } + } + // New method including the query that produced each match searchByPurlsWithReasons(purls: string[]): Map { purls = purls.map(q => q.startsWith("pkg:") ? q : `pkg:${q}`); From ec32784b2a03655d0e65d6010d9dbbbae3980735 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 26 Nov 2025 17:29:22 +0000 Subject: [PATCH 003/108] Added language choice to submission --- src/cli.ts | 2 + src/componentSubmission.ts | 120 +++++++++++++++++++++++++++++++------ src/sbomCollector.ts | 4 +- 3 files changed, 106 insertions(+), 20 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 661c418..5b96ee7 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -44,6 +44,7 @@ async function main() { .option("dependency-review", { type: "boolean", default: true, describe: "Fetch dependency review diffs for scanned branches" }) .option("diff-base", { type: "string", describe: "Override base branch for dependency review diffs (defaults to default branch)" }) .option("submit-on-missing-snapshot", { type: "boolean", default: false, describe: "When dependency review diff returns 404 (missing snapshot), run Component Detection to submit a snapshot, then retry." }) + .option("submit-languages", { type: "array", describe: "Limit snapshot submission to these languages (e.g., JavaScript,TypeScript,Python,Maven)." }) .check(args => { const syncing = !!args.syncSboms; if (syncing) { @@ -112,6 +113,7 @@ async function main() { includeDependencyReviewDiffs: argv["dependency-review"] as boolean, branchDiffBase: argv["diff-base"] as string | undefined, submitOnMissingSnapshot: argv["submit-on-missing-snapshot"] as boolean, + submitLanguages: (argv["submit-languages"] as string[] | undefined) || undefined, }); if (!quiet) process.stderr.write(chalk.cyan(offline ? "Loading SBOMs from cache..." : "Collecting SBOMs from cache & GitHub...") + "\n"); diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index e328be9..4063e92 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -10,6 +10,7 @@ export interface SubmitOpts { baseUrl?: string; caBundlePath?: string; quiet?: boolean; + languages?: string[]; } // This helper attempts to run the Component Detection + Dependency Submission action @@ -20,33 +21,114 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise wanted.some(w => w.toLowerCase() === l.toLowerCase())); + if (!intersect.length) { + if (!opts.quiet) console.error(chalk.yellow(`Skipping submission: none of selected languages present in repo (${repoLangs.join(', ')})`)); + return false; + } + // Create temp dir and sparse checkout only manifest files according to selected languages + const os = await import('os'); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'cd-submission-')); + cwd = tmp; + const repoUrl = (opts.baseUrl && opts.baseUrl.includes('api/v3')) + ? opts.baseUrl.replace(/\/api\/v3$/, '') + `/${opts.owner}/${opts.repo}.git` + : `https://github.com/${opts.owner}/${opts.repo}.git`; + const patterns = buildSparsePatterns(intersect); + // init repo + await execGit(['init'], { cwd }); + await execGit(['remote', 'add', 'origin', repoUrl], { cwd }); + await execGit(['config', 'core.sparseCheckout', 'true'], { cwd }); + fs.mkdirSync(path.join(cwd, '.git', 'info'), { recursive: true }); + fs.writeFileSync(path.join(cwd, '.git', 'info', 'sparse-checkout'), patterns.join('\n') + '\n', 'utf8'); + await execGit(['fetch', '--depth=1', 'origin', opts.branch], { cwd }); + await execGit(['checkout', 'FETCH_HEAD'], { cwd }); + } catch (e) { + if (!opts.quiet) console.error(chalk.red(`Sparse checkout failed: ${(e as Error).message}`)); + return false; + } + } + + // Run the action entrypoint pointing at the sparse checkout dir (or root if none) await new Promise((resolve, reject) => { - const child = spawn('node', [entry], { env, stdio: opts.quiet ? 'ignore' : 'inherit' }); + const env = { + ...process.env, + GITHUB_TOKEN: token, + GITHUB_BASE_URL: opts.baseUrl || process.env.GITHUB_BASE_URL || '', + }; + const child = spawn('node', [entry], { env, stdio: opts.quiet ? 'ignore' : 'inherit', cwd }); child.on('error', reject); child.on('exit', code => code === 0 ? resolve() : reject(new Error(`entrypoint exit ${code}`))); }); return true; } + +function buildSparsePatterns(langs: string[]): string[] { + const set = new Set(); + const add = (p: string) => set.add(p); + for (const l of langs) { + const ll = l.toLowerCase(); + if (ll === 'javascript' || ll === 'typescript') { + add('**/package.json'); + add('**/package-lock.json'); + add('**/yarn.lock'); + add('**/pnpm-lock.yaml'); + } else if (ll === 'python') { + add('**/requirements.txt'); + add('**/Pipfile.lock'); + add('**/poetry.lock'); + add('**/pyproject.toml'); + } else if (ll === 'go') { + add('**/go.mod'); + add('**/go.sum'); + } else if (ll === 'ruby') { + add('**/Gemfile.lock'); + add('**/gems.locked'); + } else if (ll === 'rust') { + add('**/Cargo.toml'); + add('**/Cargo.lock'); + } else if (ll === 'java') { + // Maven & Gradle + add('**/pom.xml'); + add('**/build.gradle'); + add('**/build.gradle.kts'); + add('**/settings.gradle'); + add('**/settings.gradle.kts'); + add('**/gradle.lockfile'); + } else if (ll === 'c#' || ll === 'csharp') { + add('**/packages.lock.json'); + add('**/*.csproj'); + add('**/*.sln'); + } + } + // Always include root lockfiles just in case + add('package.json'); add('package-lock.json'); add('yarn.lock'); add('pnpm-lock.yaml'); + return Array.from(set); +} + +async function execGit(args: string[], opts: { cwd: string }): Promise { + await new Promise((resolve, reject) => { + const child = spawn('git', args, { cwd: opts.cwd, stdio: 'inherit' }); + child.on('error', reject); + child.on('exit', code => code === 0 ? resolve() : reject(new Error(`git ${args.join(' ')} exit ${code}`))); + }); +} diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 132c772..5f0a439 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -30,6 +30,7 @@ export interface CollectorOptions { includeDependencyReviewDiffs?: boolean; // fetch dependency review diff base->branch branchDiffBase?: string; // override base branch for diffs (defaults to default branch) submitOnMissingSnapshot?: boolean; // run component detection submission when diff 404 + submitLanguages?: string[]; // limit submission to these languages } export class SbomCollector { @@ -68,6 +69,7 @@ export class SbomCollector { ,includeDependencyReviewDiffs: o.includeDependencyReviewDiffs ?? true ,branchDiffBase: o.branchDiffBase ,submitOnMissingSnapshot: o.submitOnMissingSnapshot ?? false + ,submitLanguages: o.submitLanguages ?? undefined } as Required; if (this.opts.token) { @@ -457,7 +459,7 @@ export class SbomCollector { try { const mod = await import("./componentSubmission.js"); if (typeof mod.submitSnapshotIfPossible === "function") { - return await mod.submitSnapshotIfPossible({ owner: org, repo, branch, token: this.opts.token, baseUrl: this.opts.baseUrl, caBundlePath: this.opts.caBundlePath, quiet: this.opts.quiet }); + return await mod.submitSnapshotIfPossible({ owner: org, repo, branch, token: this.opts.token, baseUrl: this.opts.baseUrl, caBundlePath: this.opts.caBundlePath, quiet: this.opts.quiet, languages: this.opts.submitLanguages }); } return false; } catch { From 5043ccbc3dca700518a9e5413e85f1e15f245090 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Wed, 26 Nov 2025 17:47:52 +0000 Subject: [PATCH 004/108] Debug of submission, partial progress, but not working yet --- src/componentSubmission.ts | 2 +- src/sbomCollector.ts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index 4063e92..48b6551 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -20,7 +20,7 @@ export interface SubmitOpts { export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise { const root = process.cwd(); const actionDir = path.join(root, 'component-detection-dependency-submission-action'); - const entry = path.join(actionDir, 'dist', 'entrypoint.js'); + const entry = path.join(actionDir, 'dist/index.js'); const fs = await import('fs'); if (!fs.existsSync(entry)) { if (!opts.quiet) console.error(chalk.yellow('Component Detection action not found; ensure submodule initialized and built.')); diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 5f0a439..d68faf2 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -246,8 +246,11 @@ export class SbomCollector { res.defaultBranchCommitSha = pendingCommitMeta.sha; res.defaultBranchCommitDate = pendingCommitMeta.date; } + // Branch scanning (optional) if (this.opts.includeBranches && res.defaultBranch) { + console.log(chalk.blue(`Scanning branches for ${fullName}...`)); + try { const branches = await this.listBranches(org, repo.name); const nonDefault = branches.filter(b => b.name !== res.defaultBranch); From a729fb3f1316b176d5472a926a55adfff28f917d Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Thu, 27 Nov 2025 17:36:13 +0000 Subject: [PATCH 005/108] Single repo scanning added. More scaffolding added to enable Dep Review and submission --- ...ent-detection-dependency-submission-action | 1 - package-lock.json | 770 ++++++++++++++++++ package.json | 6 +- src/cli.ts | 5 +- src/componentDetection.ts | 340 ++++++++ src/componentSubmission.ts | 259 +++--- src/sbomCollector.ts | 88 +- .../example-org/demo-repo/sbom.json | 59 ++ 8 files changed, 1370 insertions(+), 158 deletions(-) delete mode 160000 component-detection-dependency-submission-action create mode 100644 src/componentDetection.ts create mode 100644 tmp-branch-search-cache/example-org/demo-repo/sbom.json diff --git a/component-detection-dependency-submission-action b/component-detection-dependency-submission-action deleted file mode 160000 index 51ff88a..0000000 --- a/component-detection-dependency-submission-action +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 51ff88ad105a2df32251ba5f361412b753f9a27d diff --git a/package-lock.json b/package-lock.json index 1fa3460..5b5fb56 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "github-sbom-toolkit", "version": "0.1.0", "dependencies": { + "@github/dependency-submission-toolkit": "^2.0.5", "@octokit/core": "^7.0.6", "@octokit/graphql": "^9.0.1", "@octokit/plugin-paginate-rest": "^13.2.1", @@ -15,7 +16,9 @@ "@octokit/plugin-retry": "^8.0.3", "@octokit/plugin-throttling": "^11.0.3", "chalk": "^5.6.2", + "cross-fetch": "^4.1.0", "inquirer": "^12.11.0", + "octokit": "^5.0.5", "p-limit": "^7.2.0", "packageurl-js": "^2.0.1", "semver": "^7.7.3", @@ -40,6 +43,227 @@ "node": ">=18.0.0" } }, + "node_modules/@actions/core": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.11.1.tgz", + "integrity": "sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A==", + "license": "MIT", + "dependencies": { + "@actions/exec": "^1.1.1", + "@actions/http-client": "^2.0.1" + } + }, + "node_modules/@actions/exec": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.1.tgz", + "integrity": "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==", + "license": "MIT", + "dependencies": { + "@actions/io": "^1.0.1" + } + }, + "node_modules/@actions/github": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.1.tgz", + "integrity": "sha512-xbZVcaqD4XnQAe35qSQqskb3SqIAfRyLBrHMd/8TuL7hJSz2QtbDwnNM8zWx4zO5l2fnGtseNE3MbEvD7BxVMw==", + "license": "MIT", + "dependencies": { + "@actions/http-client": "^2.2.0", + "@octokit/core": "^5.0.1", + "@octokit/plugin-paginate-rest": "^9.2.2", + "@octokit/plugin-rest-endpoint-methods": "^10.4.0", + "@octokit/request": "^8.4.1", + "@octokit/request-error": "^5.1.1", + "undici": "^5.28.5" + } + }, + "node_modules/@actions/github/node_modules/@octokit/auth-token": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", + "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==", + "license": "MIT", + "engines": { + "node": ">= 18" + } + }, + "node_modules/@actions/github/node_modules/@octokit/core": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.2.tgz", + "integrity": "sha512-/g2d4sW9nUDJOMz3mabVQvOGhVa4e/BN/Um7yca9Bb2XTzPPnfTWHWQg+IsEYO7M3Vx+EXvaM/I2pJWIMun1bg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@octokit/auth-token": "^4.0.0", + "@octokit/graphql": "^7.1.0", + "@octokit/request": "^8.4.1", + "@octokit/request-error": "^5.1.1", + "@octokit/types": "^13.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@actions/github/node_modules/@octokit/endpoint": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.6.tgz", + "integrity": "sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.1.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@actions/github/node_modules/@octokit/graphql": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.1.1.tgz", + "integrity": "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==", + "license": "MIT", + "dependencies": { + "@octokit/request": "^8.4.1", + "@octokit/types": "^13.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@actions/github/node_modules/@octokit/openapi-types": { + "version": "24.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz", + "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==", + "license": "MIT" + }, + "node_modules/@actions/github/node_modules/@octokit/plugin-paginate-rest": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.2.2.tgz", + "integrity": "sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^12.6.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": "5" + } + }, + "node_modules/@actions/github/node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/openapi-types": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz", + "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==", + "license": "MIT" + }, + "node_modules/@actions/github/node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/types": { + "version": "12.6.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz", + "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^20.0.0" + } + }, + "node_modules/@actions/github/node_modules/@octokit/plugin-rest-endpoint-methods": { + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.4.1.tgz", + "integrity": "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^12.6.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": "5" + } + }, + "node_modules/@actions/github/node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/openapi-types": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz", + "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==", + "license": "MIT" + }, + "node_modules/@actions/github/node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/types": { + "version": "12.6.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz", + "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^20.0.0" + } + }, + "node_modules/@actions/github/node_modules/@octokit/request": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.1.tgz", + "integrity": "sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw==", + "license": "MIT", + "dependencies": { + "@octokit/endpoint": "^9.0.6", + "@octokit/request-error": "^5.1.1", + "@octokit/types": "^13.1.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@actions/github/node_modules/@octokit/request-error": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.1.1.tgz", + "integrity": "sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.1.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@actions/github/node_modules/@octokit/types": { + "version": "13.10.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz", + "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^24.2.0" + } + }, + "node_modules/@actions/github/node_modules/before-after-hook": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", + "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==", + "license": "Apache-2.0" + }, + "node_modules/@actions/github/node_modules/universal-user-agent": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz", + "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==", + "license": "ISC" + }, + "node_modules/@actions/http-client": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.3.tgz", + "integrity": "sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA==", + "license": "MIT", + "dependencies": { + "tunnel": "^0.0.6", + "undici": "^5.25.4" + } + }, + "node_modules/@actions/io": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz", + "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==", + "license": "MIT" + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.25.10", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.10.tgz", @@ -661,6 +885,64 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@fastify/busboy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@github/dependency-submission-toolkit": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@github/dependency-submission-toolkit/-/dependency-submission-toolkit-2.0.5.tgz", + "integrity": "sha512-bCgbNa1WZZuexw5B3DVlIrkiMLf4kDtdPIdvAh7SibtvOM+lMCcLZXsEz3ukGY3QYay0+FPuMiiINw0LCRJJ5w==", + "license": "MIT", + "workspaces": [ + "example" + ], + "dependencies": { + "@actions/core": "^1.10.1", + "@actions/github": "^6.0.0", + "@octokit/request-error": "^6.1.1", + "@octokit/webhooks-types": "^7.5.0", + "packageurl-js": "^1.2.1" + } + }, + "node_modules/@github/dependency-submission-toolkit/node_modules/@octokit/openapi-types": { + "version": "25.1.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-25.1.0.tgz", + "integrity": "sha512-idsIggNXUKkk0+BExUn1dQ92sfysJrje03Q0bv0e+KPLrvyqZF8MnBpFz8UNfYDwB3Ie7Z0TByjWfzxt7vseaA==", + "license": "MIT" + }, + "node_modules/@github/dependency-submission-toolkit/node_modules/@octokit/request-error": { + "version": "6.1.8", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-6.1.8.tgz", + "integrity": "sha512-WEi/R0Jmq+IJKydWlKDmryPcmdYSVjL3ekaiEL1L9eo1sUnqMJ+grqmC9cjk7CA7+b2/T397tO5d8YLOH3qYpQ==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^14.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@github/dependency-submission-toolkit/node_modules/@octokit/types": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-14.1.0.tgz", + "integrity": "sha512-1y6DgTy8Jomcpu33N+p5w58l6xyt55Ar2I91RPiIA0xCJBXyUAhXCcmZaDWSANiha7R9a6qJJ2CRomGPZ6f46g==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^25.1.0" + } + }, + "node_modules/@github/dependency-submission-toolkit/node_modules/packageurl-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/packageurl-js/-/packageurl-js-1.2.1.tgz", + "integrity": "sha512-cZ6/MzuXaoFd16/k0WnwtI298UCaDHe/XlSh85SeOKbGZ1hq0xvNbx3ILyCMyk7uFQxl6scF3Aucj6/EO9NwcA==", + "license": "MIT" + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -1085,6 +1367,180 @@ "node": ">= 8" } }, + "node_modules/@octokit/app": { + "version": "16.1.2", + "resolved": "https://registry.npmjs.org/@octokit/app/-/app-16.1.2.tgz", + "integrity": "sha512-8j7sEpUYVj18dxvh0KWj6W/l6uAiVRBl1JBDVRqH1VHKAO/G5eRVl4yEoYACjakWers1DjUkcCHyJNQK47JqyQ==", + "license": "MIT", + "dependencies": { + "@octokit/auth-app": "^8.1.2", + "@octokit/auth-unauthenticated": "^7.0.3", + "@octokit/core": "^7.0.6", + "@octokit/oauth-app": "^8.0.3", + "@octokit/plugin-paginate-rest": "^14.0.0", + "@octokit/types": "^16.0.0", + "@octokit/webhooks": "^14.0.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/app/node_modules/@octokit/openapi-types": { + "version": "27.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", + "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", + "license": "MIT" + }, + "node_modules/@octokit/app/node_modules/@octokit/plugin-paginate-rest": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-14.0.0.tgz", + "integrity": "sha512-fNVRE7ufJiAA3XUrha2omTA39M6IXIc6GIZLvlbsm8QOQCYvpq/LkMNGyFlB1d8hTDzsAXa3OKtybdMAYsV/fw==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^16.0.0" + }, + "engines": { + "node": ">= 20" + }, + "peerDependencies": { + "@octokit/core": ">=6" + } + }, + "node_modules/@octokit/app/node_modules/@octokit/types": { + "version": "16.0.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", + "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^27.0.0" + } + }, + "node_modules/@octokit/auth-app": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/@octokit/auth-app/-/auth-app-8.1.2.tgz", + "integrity": "sha512-db8VO0PqXxfzI6GdjtgEFHY9tzqUql5xMFXYA12juq8TeTgPAuiiP3zid4h50lwlIP457p5+56PnJOgd2GGBuw==", + "license": "MIT", + "dependencies": { + "@octokit/auth-oauth-app": "^9.0.3", + "@octokit/auth-oauth-user": "^6.0.2", + "@octokit/request": "^10.0.6", + "@octokit/request-error": "^7.0.2", + "@octokit/types": "^16.0.0", + "toad-cache": "^3.7.0", + "universal-github-app-jwt": "^2.2.0", + "universal-user-agent": "^7.0.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/auth-app/node_modules/@octokit/openapi-types": { + "version": "27.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", + "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", + "license": "MIT" + }, + "node_modules/@octokit/auth-app/node_modules/@octokit/types": { + "version": "16.0.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", + "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^27.0.0" + } + }, + "node_modules/@octokit/auth-oauth-app": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-app/-/auth-oauth-app-9.0.3.tgz", + "integrity": "sha512-+yoFQquaF8OxJSxTb7rnytBIC2ZLbLqA/yb71I4ZXT9+Slw4TziV9j/kyGhUFRRTF2+7WlnIWsePZCWHs+OGjg==", + "license": "MIT", + "dependencies": { + "@octokit/auth-oauth-device": "^8.0.3", + "@octokit/auth-oauth-user": "^6.0.2", + "@octokit/request": "^10.0.6", + "@octokit/types": "^16.0.0", + "universal-user-agent": "^7.0.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/auth-oauth-app/node_modules/@octokit/openapi-types": { + "version": "27.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", + "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", + "license": "MIT" + }, + "node_modules/@octokit/auth-oauth-app/node_modules/@octokit/types": { + "version": "16.0.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", + "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^27.0.0" + } + }, + "node_modules/@octokit/auth-oauth-device": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-device/-/auth-oauth-device-8.0.3.tgz", + "integrity": "sha512-zh2W0mKKMh/VWZhSqlaCzY7qFyrgd9oTWmTmHaXnHNeQRCZr/CXy2jCgHo4e4dJVTiuxP5dLa0YM5p5QVhJHbw==", + "license": "MIT", + "dependencies": { + "@octokit/oauth-methods": "^6.0.2", + "@octokit/request": "^10.0.6", + "@octokit/types": "^16.0.0", + "universal-user-agent": "^7.0.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/auth-oauth-device/node_modules/@octokit/openapi-types": { + "version": "27.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", + "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", + "license": "MIT" + }, + "node_modules/@octokit/auth-oauth-device/node_modules/@octokit/types": { + "version": "16.0.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", + "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^27.0.0" + } + }, + "node_modules/@octokit/auth-oauth-user": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-user/-/auth-oauth-user-6.0.2.tgz", + "integrity": "sha512-qLoPPc6E6GJoz3XeDG/pnDhJpTkODTGG4kY0/Py154i/I003O9NazkrwJwRuzgCalhzyIeWQ+6MDvkUmKXjg/A==", + "license": "MIT", + "dependencies": { + "@octokit/auth-oauth-device": "^8.0.3", + "@octokit/oauth-methods": "^6.0.2", + "@octokit/request": "^10.0.6", + "@octokit/types": "^16.0.0", + "universal-user-agent": "^7.0.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/auth-oauth-user/node_modules/@octokit/openapi-types": { + "version": "27.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", + "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", + "license": "MIT" + }, + "node_modules/@octokit/auth-oauth-user/node_modules/@octokit/types": { + "version": "16.0.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", + "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^27.0.0" + } + }, "node_modules/@octokit/auth-token": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-6.0.0.tgz", @@ -1094,6 +1550,34 @@ "node": ">= 20" } }, + "node_modules/@octokit/auth-unauthenticated": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/@octokit/auth-unauthenticated/-/auth-unauthenticated-7.0.3.tgz", + "integrity": "sha512-8Jb1mtUdmBHL7lGmop9mU9ArMRUTRhg8vp0T1VtZ4yd9vEm3zcLwmjQkhNEduKawOOORie61xhtYIhTDN+ZQ3g==", + "license": "MIT", + "dependencies": { + "@octokit/request-error": "^7.0.2", + "@octokit/types": "^16.0.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/auth-unauthenticated/node_modules/@octokit/openapi-types": { + "version": "27.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", + "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", + "license": "MIT" + }, + "node_modules/@octokit/auth-unauthenticated/node_modules/@octokit/types": { + "version": "16.0.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", + "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^27.0.0" + } + }, "node_modules/@octokit/core": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/@octokit/core/-/core-7.0.6.tgz", @@ -1185,12 +1669,88 @@ "@octokit/openapi-types": "^27.0.0" } }, + "node_modules/@octokit/oauth-app": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/@octokit/oauth-app/-/oauth-app-8.0.3.tgz", + "integrity": "sha512-jnAjvTsPepyUaMu9e69hYBuozEPgYqP4Z3UnpmvoIzHDpf8EXDGvTY1l1jK0RsZ194oRd+k6Hm13oRU8EoDFwg==", + "license": "MIT", + "dependencies": { + "@octokit/auth-oauth-app": "^9.0.2", + "@octokit/auth-oauth-user": "^6.0.1", + "@octokit/auth-unauthenticated": "^7.0.2", + "@octokit/core": "^7.0.5", + "@octokit/oauth-authorization-url": "^8.0.0", + "@octokit/oauth-methods": "^6.0.1", + "@types/aws-lambda": "^8.10.83", + "universal-user-agent": "^7.0.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/oauth-authorization-url": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@octokit/oauth-authorization-url/-/oauth-authorization-url-8.0.0.tgz", + "integrity": "sha512-7QoLPRh/ssEA/HuHBHdVdSgF8xNLz/Bc5m9fZkArJE5bb6NmVkDm3anKxXPmN1zh6b5WKZPRr3697xKT/yM3qQ==", + "license": "MIT", + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/oauth-methods": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/@octokit/oauth-methods/-/oauth-methods-6.0.2.tgz", + "integrity": "sha512-HiNOO3MqLxlt5Da5bZbLV8Zarnphi4y9XehrbaFMkcoJ+FL7sMxH/UlUsCVxpddVu4qvNDrBdaTVE2o4ITK8ng==", + "license": "MIT", + "dependencies": { + "@octokit/oauth-authorization-url": "^8.0.0", + "@octokit/request": "^10.0.6", + "@octokit/request-error": "^7.0.2", + "@octokit/types": "^16.0.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/oauth-methods/node_modules/@octokit/openapi-types": { + "version": "27.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", + "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", + "license": "MIT" + }, + "node_modules/@octokit/oauth-methods/node_modules/@octokit/types": { + "version": "16.0.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", + "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^27.0.0" + } + }, "node_modules/@octokit/openapi-types": { "version": "26.0.0", "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-26.0.0.tgz", "integrity": "sha512-7AtcfKtpo77j7Ts73b4OWhOZHTKo/gGY8bB3bNBQz4H+GRSWqx2yvj8TXRsbdTE0eRmYmXOEY66jM7mJ7LzfsA==", "license": "MIT" }, + "node_modules/@octokit/openapi-webhooks-types": { + "version": "12.0.3", + "resolved": "https://registry.npmjs.org/@octokit/openapi-webhooks-types/-/openapi-webhooks-types-12.0.3.tgz", + "integrity": "sha512-90MF5LVHjBedwoHyJsgmaFhEN1uzXyBDRLEBe7jlTYx/fEhPAk3P3DAJsfZwC54m8hAIryosJOL+UuZHB3K3yA==", + "license": "MIT" + }, + "node_modules/@octokit/plugin-paginate-graphql": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-graphql/-/plugin-paginate-graphql-6.0.0.tgz", + "integrity": "sha512-crfpnIoFiBtRkvPqOyLOsw12XsveYuY2ieP6uYDosoUegBJpSVxGwut9sxUgFFcll3VTOTqpUf8yGd8x1OmAkQ==", + "license": "MIT", + "engines": { + "node": ">= 20" + }, + "peerDependencies": { + "@octokit/core": ">=6" + } + }, "node_modules/@octokit/plugin-paginate-rest": { "version": "13.2.1", "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-13.2.1.tgz", @@ -1351,6 +1911,41 @@ "@octokit/openapi-types": "^26.0.0" } }, + "node_modules/@octokit/webhooks": { + "version": "14.1.3", + "resolved": "https://registry.npmjs.org/@octokit/webhooks/-/webhooks-14.1.3.tgz", + "integrity": "sha512-gcK4FNaROM9NjA0mvyfXl0KPusk7a1BeA8ITlYEZVQCXF5gcETTd4yhAU0Kjzd8mXwYHppzJBWgdBVpIR9wUcQ==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-webhooks-types": "12.0.3", + "@octokit/request-error": "^7.0.0", + "@octokit/webhooks-methods": "^6.0.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/webhooks-methods": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@octokit/webhooks-methods/-/webhooks-methods-6.0.0.tgz", + "integrity": "sha512-MFlzzoDJVw/GcbfzVC1RLR36QqkTLUf79vLVO3D+xn7r0QgxnFoLZgtrzxiQErAjFUOdH6fas2KeQJ1yr/qaXQ==", + "license": "MIT", + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/webhooks-types": { + "version": "7.6.1", + "resolved": "https://registry.npmjs.org/@octokit/webhooks-types/-/webhooks-types-7.6.1.tgz", + "integrity": "sha512-S8u2cJzklBC0FgTwWVLaM8tMrDuDMVE4xiTK4EYXM9GntyvrdbSoxqDQa+Fh57CCNApyIpyeqPhhFEmHPfrXgw==", + "license": "MIT" + }, + "node_modules/@types/aws-lambda": { + "version": "8.10.159", + "resolved": "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.159.tgz", + "integrity": "sha512-SAP22WSGNN12OQ8PlCzGzRCZ7QDCwI85dQZbmpz7+mAk+L7j+wI7qnvmdKh+o7A5LaOp6QnOZ2NJphAZQTTHQg==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -2207,6 +2802,15 @@ "dev": true, "license": "MIT" }, + "node_modules/cross-fetch": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.1.0.tgz", + "integrity": "sha512-uKm5PU+MHTootlWEY+mZ4vvXoCn4fLQxT9dSc1sXVMSFkINTJVN8cAQROpwcKm8bJ/c7rgZVIBWzH5T78sNZZw==", + "license": "MIT", + "dependencies": { + "node-fetch": "^2.7.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -2247,6 +2851,12 @@ "dev": true, "license": "MIT" }, + "node_modules/deprecation": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", + "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==", + "license": "ISC" + }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -3075,6 +3685,102 @@ "dev": true, "license": "MIT" }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/octokit": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/octokit/-/octokit-5.0.5.tgz", + "integrity": "sha512-4+/OFSqOjoyULo7eN7EA97DE0Xydj/PW5aIckxqQIoFjFwqXKuFCvXUJObyJfBF9Khu4RL/jlDRI9FPaMGfPnw==", + "license": "MIT", + "dependencies": { + "@octokit/app": "^16.1.2", + "@octokit/core": "^7.0.6", + "@octokit/oauth-app": "^8.0.3", + "@octokit/plugin-paginate-graphql": "^6.0.0", + "@octokit/plugin-paginate-rest": "^14.0.0", + "@octokit/plugin-rest-endpoint-methods": "^17.0.0", + "@octokit/plugin-retry": "^8.0.3", + "@octokit/plugin-throttling": "^11.0.3", + "@octokit/request-error": "^7.0.2", + "@octokit/types": "^16.0.0", + "@octokit/webhooks": "^14.0.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/octokit/node_modules/@octokit/openapi-types": { + "version": "27.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", + "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", + "license": "MIT" + }, + "node_modules/octokit/node_modules/@octokit/plugin-paginate-rest": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-14.0.0.tgz", + "integrity": "sha512-fNVRE7ufJiAA3XUrha2omTA39M6IXIc6GIZLvlbsm8QOQCYvpq/LkMNGyFlB1d8hTDzsAXa3OKtybdMAYsV/fw==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^16.0.0" + }, + "engines": { + "node": ">= 20" + }, + "peerDependencies": { + "@octokit/core": ">=6" + } + }, + "node_modules/octokit/node_modules/@octokit/plugin-rest-endpoint-methods": { + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-17.0.0.tgz", + "integrity": "sha512-B5yCyIlOJFPqUUeiD0cnBJwWJO8lkJs5d8+ze9QDP6SvfiXSz1BF+91+0MeI1d2yxgOhU/O+CvtiZ9jSkHhFAw==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^16.0.0" + }, + "engines": { + "node": ">= 20" + }, + "peerDependencies": { + "@octokit/core": ">=6" + } + }, + "node_modules/octokit/node_modules/@octokit/types": { + "version": "16.0.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", + "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^27.0.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -3455,6 +4161,21 @@ "node": ">=8.0" } }, + "node_modules/toad-cache": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/toad-cache/-/toad-cache-3.7.0.tgz", + "integrity": "sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/ts-api-utils": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz", @@ -3494,6 +4215,15 @@ "fsevents": "~2.3.3" } }, + "node_modules/tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", + "license": "MIT", + "engines": { + "node": ">=0.6.11 <=0.7.0 || >=0.7.3" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -3522,6 +4252,18 @@ "node": ">=14.17" } }, + "node_modules/undici": { + "version": "5.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.29.0.tgz", + "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==", + "license": "MIT", + "dependencies": { + "@fastify/busboy": "^2.0.0" + }, + "engines": { + "node": ">=14.0" + } + }, "node_modules/undici-types": { "version": "7.16.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", @@ -3529,6 +4271,12 @@ "devOptional": true, "license": "MIT" }, + "node_modules/universal-github-app-jwt": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/universal-github-app-jwt/-/universal-github-app-jwt-2.2.2.tgz", + "integrity": "sha512-dcmbeSrOdTnsjGjUfAlqNDJrhxXizjAz94ija9Qw8YkZ1uu0d+GoZzyH+Jb9tIIqvGsadUfwg+22k5aDqqwzbw==", + "license": "MIT" + }, "node_modules/universal-user-agent": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-7.0.3.tgz", @@ -3545,6 +4293,22 @@ "punycode": "^2.1.0" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -3585,6 +4349,12 @@ "node": ">=8" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index 218991b..61f0574 100644 --- a/package.json +++ b/package.json @@ -14,12 +14,12 @@ "lint": "eslint . --ext .ts --max-warnings=0", "test": "node dist/test-fixture-match.js", "test:branch-search": "node dist/test-branch-search.js" - }, "engines": { "node": ">=18.0.0" }, "dependencies": { + "@github/dependency-submission-toolkit": "^2.0.5", "@octokit/core": "^7.0.6", "@octokit/graphql": "^9.0.1", "@octokit/plugin-paginate-rest": "^13.2.1", @@ -27,7 +27,9 @@ "@octokit/plugin-retry": "^8.0.3", "@octokit/plugin-throttling": "^11.0.3", "chalk": "^5.6.2", + "cross-fetch": "^4.1.0", "inquirer": "^12.11.0", + "octokit": "^5.0.5", "p-limit": "^7.2.0", "packageurl-js": "^2.0.1", "semver": "^7.7.3", @@ -45,4 +47,4 @@ "tsx": "^4.20.6", "typescript": "^5.9.3" } -} \ No newline at end of file +} diff --git a/src/cli.ts b/src/cli.ts index 5b96ee7..2bd5050 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -13,6 +13,7 @@ async function main() { .option("token", { type: "string", describe: "GitHub token with repo + security_events scope" }) .option("enterprise", { type: "string", describe: "Enterprise slug (mutually exclusive with --org)" }) .option("org", { type: "string", describe: "Single organization login" }) + .option("repo", { type: "string", describe: "Single repository name" }) .option("base-url", { type: "string", describe: "GitHub Enterprise Server base URL, e.g. https://github.mycompany.com/api/v3" }) .option("concurrency", { type: "number", default: 5 }) .option("sbom-delay", { type: "number", default: 3000, describe: "Delay (ms) between SBOM fetch requests" }) @@ -48,8 +49,9 @@ async function main() { .check(args => { const syncing = !!args.syncSboms; if (syncing) { - if (!args.enterprise && !args.org) throw new Error("Provide --enterprise or --org with --sync-sboms"); + if (!args.enterprise && !args.org && !args.repo) throw new Error("Provide --enterprise, --org or --repo with --sync-sboms"); if (args.enterprise && args.org) throw new Error("Specify only one of --enterprise or --org"); + if (args.repo && (args.enterprise || args.org)) throw new Error("Specify only one of --enterprise, --org, or --repo"); } else { if (!args.sbomCache) throw new Error("Offline mode requires --sbom-cache (omit --sync-sboms)"); } @@ -98,6 +100,7 @@ async function main() { token: token, enterprise: argv.enterprise as string | undefined, org: argv.org as string | undefined, + repo: argv.repo as string | undefined, baseUrl: argv["base-url"] as string | undefined, concurrency: argv.concurrency as number, delayMsBetweenRepos: argv["sbom-delay"] as number, diff --git a/src/componentDetection.ts b/src/componentDetection.ts new file mode 100644 index 0000000..f57e9f1 --- /dev/null +++ b/src/componentDetection.ts @@ -0,0 +1,340 @@ +import { Octokit } from "octokit" +import { + PackageCache, + Package, + Manifest, +} from '@github/dependency-submission-toolkit' +import fetch from 'cross-fetch' +import fs from 'fs' +import { spawn } from 'child_process'; +//import dotenv from 'dotenv' +import path from 'path'; +//dotenv.config(); + +export default class ComponentDetection { + public static componentDetectionPath = process.platform === "win32" ? './component-detection.exe' : './component-detection'; + public static outputPath = './output.json'; + + // This is the default entry point for this class. + static async scanAndGetManifests(path: string): Promise { + await this.downloadLatestRelease(); + await this.runComponentDetection(path); + return await this.getManifestsFromResults(); + } + // Get the latest release from the component-detection repo, download the tarball, and extract it + public static async downloadLatestRelease() { + const statResult = fs.statSync(this.componentDetectionPath); + if (statResult && statResult.isFile()) { + console.debug(`Component-detection binary already exists at ${this.componentDetectionPath}, skipping download.`); + return; + } + + try { + console.debug(`Downloading latest release for ${process.platform}`); + const downloadURL = await this.getLatestReleaseURL(); + const blob = await (await fetch(new URL(downloadURL))).blob(); + const arrayBuffer = await blob.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + + // Write the blob to a file + console.debug(`Writing binary to file ${this.componentDetectionPath}`); + await fs.writeFileSync(this.componentDetectionPath, buffer, { mode: 0o777, flag: 'w' }); + } catch (error: any) { + console.error(error); + } + } + + // Run the component-detection CLI on the path specified + public static async runComponentDetection(path: string) { + console.info("Running component-detection"); + + try { + await spawn(`${this.componentDetectionPath}`, ['scan', '--SourceDirectory', path, '--ManifestFile', this.outputPath, ...this.getComponentDetectionParameters()]); + } catch (error: any) { + console.error(error); + } + } + + private static getComponentDetectionParameters(): Array { + var parameters: Array = []; + // parameters.push((console.getInput('directoryExclusionList')) ? ` --DirectoryExclusionList ${console.getInput('directoryExclusionList')}` : ""); + // parameters.push((console.getInput('detectorArgs')) ? ` --DetectorArgs ${console.getInput('detectorArgs')}` : ""); + // parameters.push((console.getInput('detectorsFilter')) ? ` --DetectorsFilter ${console.getInput('detectorsFilter')}` : ""); + // parameters.push((console.getInput('detectorsCategories')) ? ` --DetectorCategories ${console.getInput('detectorsCategories')}` : ""); + // parameters.push((console.getInput('dockerImagesToScan')) ? ` --DockerImagesToScan ${console.getInput('dockerImagesToScan')}` : ""); + return parameters; + } + + public static async getManifestsFromResults(): Promise { + console.info("Getting manifests from results"); + const results = await fs.readFileSync(this.outputPath, 'utf8'); + var json: any = JSON.parse(results); + let dependencyGraphs: DependencyGraphs = this.normalizeDependencyGraphPaths(json.dependencyGraphs, '.'); + return this.processComponentsToManifests(json.componentsFound, dependencyGraphs); + } + + public static processComponentsToManifests(componentsFound: any[], dependencyGraphs: DependencyGraphs): Manifest[] { + // Parse the result file and add the packages to the package cache + const packageCache = new PackageCache(); + const packages: Array = []; + + componentsFound.forEach(async (component: any) => { + // Skip components without packageUrl + if (!component.component.packageUrl) { + console.debug(`Skipping component detected without packageUrl: ${JSON.stringify({ + id: component.component.id, + name: component.component.name || 'unnamed', + type: component.component.type || 'unknown' + }, null, 2)}`); + return; + } + + const packageUrl = ComponentDetection.makePackageUrl(component.component.packageUrl); + + // Skip if the packageUrl is empty (indicates an invalid or missing packageUrl) + if (!packageUrl) { + console.debug(`Skipping component with invalid packageUrl: ${component.component.id}`); + return; + } + + if (!packageCache.hasPackage(packageUrl)) { + const pkg = new ComponentDetectionPackage(packageUrl, component.component.id, + component.isDevelopmentDependency, component.topLevelReferrers, component.locationsFoundAt, component.containerDetailIds, component.containerLayerIds); + packageCache.addPackage(pkg); + packages.push(pkg); + } + }); + + // Set the transitive dependencies + console.debug("Sorting out transitive dependencies"); + packages.forEach(async (pkg: ComponentDetectionPackage) => { + pkg.topLevelReferrers.forEach(async (referrer: any) => { + // Skip if referrer doesn't have a valid packageUrl + if (!referrer.packageUrl) { + console.debug(`Skipping referrer without packageUrl for component: ${pkg.id}`); + return; + } + + const referrerUrl = ComponentDetection.makePackageUrl(referrer.packageUrl); + referrer.packageUrlString = referrerUrl + + // Skip if the generated packageUrl is empty + if (!referrerUrl) { + console.debug(`Skipping referrer with invalid packageUrl for component: ${pkg.id}`); + return; + } + + try { + const referrerPackage = packageCache.lookupPackage(referrerUrl); + if (referrerPackage === pkg) { + console.debug(`Skipping self-reference for package: ${pkg.id}`); + return; // Skip self-references + } + if (referrerPackage) { + referrerPackage.dependsOn(pkg); + } + } catch (error) { + console.debug(`Error looking up referrer package: ${error}`); + } + }); + }); + + // Create manifests + const manifests: Array = []; + + // Check the locationsFoundAt for every package and add each as a manifest + this.addPackagesToManifests(packages, manifests, dependencyGraphs); + + return manifests; + } + + private static addPackagesToManifests(packages: Array, manifests: Array, dependencyGraphs: DependencyGraphs): void { + packages.forEach((pkg: ComponentDetectionPackage) => { + pkg.locationsFoundAt.forEach((location: any) => { + // Use the normalized path (remove leading slash if present) + let normalizedLocation = location.startsWith('/') ? location.substring(1) : location; + // Unescape the path, as upstream ComponentDetection emits locationsFoundAt in URL-encoded form + normalizedLocation = decodeURIComponent(normalizedLocation); + + if (!manifests.find((manifest: Manifest) => manifest.name == normalizedLocation)) { + const manifest = new Manifest(normalizedLocation, normalizedLocation); + manifests.push(manifest); + } + + const depGraphEntry = dependencyGraphs[normalizedLocation]; + if (!depGraphEntry) { + console.warn(`No dependency graph entry found for manifest location: ${normalizedLocation}`); + return; // Skip this location if not found in dependencyGraphs + } + + const directDependencies = depGraphEntry.explicitlyReferencedComponentIds; + if (directDependencies.includes(pkg.id)) { + manifests + .find((manifest: Manifest) => manifest.name == normalizedLocation) + ?.addDirectDependency( + pkg, + ComponentDetection.getDependencyScope(pkg) + ); + } else { + manifests + .find((manifest: Manifest) => manifest.name == normalizedLocation) + ?.addIndirectDependency( + pkg, + ComponentDetection.getDependencyScope(pkg) + ); + } + }); + }); + } + + private static getDependencyScope(pkg: ComponentDetectionPackage) { + return pkg.isDevelopmentDependency ? 'development' : 'runtime' + } + + public static makePackageUrl(packageUrlJson: any): string { + // Handle case when packageUrlJson is null or undefined + if ( + !packageUrlJson || + typeof packageUrlJson.Scheme !== 'string' || + typeof packageUrlJson.Type !== 'string' || + !packageUrlJson.Scheme || + !packageUrlJson.Type + ) { + console.debug(`Warning: Received null or undefined packageUrlJson. Unable to create package URL.`); + return ""; // Return a blank string for unknown packages + } + + try { + var packageUrl = `${packageUrlJson.Scheme}:${packageUrlJson.Type}/`; + if (packageUrlJson.Namespace) { + packageUrl += `${packageUrlJson.Namespace.replaceAll("@", "%40")}/`; + } + packageUrl += `${packageUrlJson.Name.replaceAll("@", "%40")}`; + if (packageUrlJson.Version) { + packageUrl += `@${packageUrlJson.Version}`; + } + if (typeof packageUrlJson.Qualifiers === "object" + && packageUrlJson.Qualifiers !== null + && Object.keys(packageUrlJson.Qualifiers).length > 0) { + const qualifierString = Object.entries(packageUrlJson.Qualifiers) + .map(([key, value]) => `${key}=${value}`) + .join("&"); + packageUrl += `?${qualifierString}`; + } + return packageUrl; + } catch (error) { + console.debug(`Error creating package URL from packageUrlJson: ${JSON.stringify(packageUrlJson, null, 2)}`); + console.debug(`Error details: ${error}`); + return ""; // Return a blank string for error cases + } + } + + private static async getLatestReleaseURL(): Promise { + let githubToken = process.env.GITHUB_TOKEN || ""; + + const githubAPIURL = process.env.GITHUB_API_URL || 'https://api.github.com'; + + let ghesMode = process.env.GITHUB_API_URL != githubAPIURL; + // If the we're running in GHES, then use an empty string as the token + if (ghesMode) { + githubToken = ""; + } + const octokit = new Octokit({ auth: githubToken, baseUrl: githubAPIURL, request: { fetch: fetch}, log: { + debug: console.debug, + info: console.info, + warn: console.warn, + error: console.error + }, }); + + const owner = "microsoft"; + const repo = "component-detection"; + console.debug("Attempting to download latest release from " + githubAPIURL); + + try { + const latestRelease = await octokit.request("GET /repos/{owner}/{repo}/releases/latest", {owner, repo}); + + var downloadURL: string = ""; + const assetName = process.platform === "win32" ? "component-detection-win-x64.exe" : "component-detection-linux-x64"; + latestRelease.data.assets.forEach((asset: any) => { + if (asset.name === assetName) { + downloadURL = asset.browser_download_url; + } + }); + + return downloadURL; + } catch (error: any) { + console.error(error); + console.debug(error.message); + console.debug(error.stack); + throw new Error("Failed to download latest release"); + } + } + + /** + * Normalizes the keys of a DependencyGraphs object to be relative paths from the resolved filePath input. + * @param dependencyGraphs The DependencyGraphs object to normalize. + * @param filePathInput The filePath input (relative or absolute) from the action configuration. + * @returns A new DependencyGraphs object with relative path keys. + */ + public static normalizeDependencyGraphPaths( + dependencyGraphs: DependencyGraphs, + filePathInput: string + ): DependencyGraphs { + // Resolve the base directory from filePathInput (relative to cwd if not absolute) + const baseDir = path.resolve(process.cwd(), filePathInput); + const normalized: DependencyGraphs = {}; + for (const absPath in dependencyGraphs) { + // Make the path relative to the baseDir + let relPath = path.relative(baseDir, absPath).replace(/\\/g, '/'); + normalized[relPath] = dependencyGraphs[absPath]; + } + return normalized; + } +} + +class ComponentDetectionPackage extends Package { + public packageUrlString: string; + + constructor(packageUrl: string, public id: string, public isDevelopmentDependency: boolean, public topLevelReferrers: [], + public locationsFoundAt: [], public containerDetailIds: [], public containerLayerIds: []) { + super(packageUrl); + this.packageUrlString = packageUrl; + } +} + +/** + * Types for the dependencyGraphs section of output.json + */ +export type DependencyGraph = { + /** + * The dependency graph: keys are component IDs, values are either null (no dependencies) or an array of component IDs (dependencies) + */ + graph: Record; + /** + * Explicitly referenced component IDs + */ + explicitlyReferencedComponentIds: string[]; + /** + * Development dependencies + */ + developmentDependencies: string[]; + /** + * Regular dependencies + */ + dependencies: string[]; +}; + +/** + * The top-level dependencyGraphs object: keys are manifest file paths, values are DependencyGraph objects + */ +export type DependencyGraphs = Record; + + + + + + + + + + diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index 48b6551..ba6aa47 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -1,16 +1,24 @@ import chalk from 'chalk'; -import { spawn } from 'child_process'; +import { spawn, ChildProcess } from 'child_process'; import path from 'path'; +import fs from 'fs'; +import type { Context } from '@actions/github/lib/context.js' + +import ComponentDetection from './componentDetection'; +import { + Snapshot, + submitSnapshot +} from '@github/dependency-submission-toolkit'; export interface SubmitOpts { - owner: string; - repo: string; - branch: string; - token?: string; - baseUrl?: string; - caBundlePath?: string; - quiet?: boolean; - languages?: string[]; + owner: string; + repo: string; + branch: string; + token?: string; + baseUrl?: string; + caBundlePath?: string; + quiet?: boolean; + languages?: string[]; } // This helper attempts to run the Component Detection + Dependency Submission action @@ -18,117 +26,140 @@ export interface SubmitOpts { // `component-detection-dependency-submission-action`. // It falls back to returning false if not available. export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise { - const root = process.cwd(); - const actionDir = path.join(root, 'component-detection-dependency-submission-action'); - const entry = path.join(actionDir, 'dist/index.js'); - const fs = await import('fs'); - if (!fs.existsSync(entry)) { - if (!opts.quiet) console.error(chalk.yellow('Component Detection action not found; ensure submodule initialized and built.')); - return false; - } - const token = opts.token || process.env.GITHUB_TOKEN || ''; - if (!token) { - if (!opts.quiet) console.error(chalk.red('GITHUB_TOKEN required to submit dependency snapshot')); - return false; - } - - // If languages filter provided, inspect repo languages and perform sparse checkout of relevant manifests - let cwd = root; - if (opts.languages && opts.languages.length) { - const { createOctokit } = await import('./octokit.js'); - const o = createOctokit({ token, baseUrl: opts.baseUrl }); - try { - const langResp = await o.request('GET /repos/{owner}/{repo}/languages', { owner: opts.owner, repo: opts.repo }); - const repoLangs = Object.keys(langResp.data || {}); - const wanted = opts.languages; - const intersect = repoLangs.filter(l => wanted.some(w => w.toLowerCase() === l.toLowerCase())); - if (!intersect.length) { - if (!opts.quiet) console.error(chalk.yellow(`Skipping submission: none of selected languages present in repo (${repoLangs.join(', ')})`)); + const token = opts.token || process.env.GITHUB_TOKEN || ''; + if (!token) { + if (!opts.quiet) console.error(chalk.red('GITHUB_TOKEN required to submit dependency snapshot')); return false; - } - // Create temp dir and sparse checkout only manifest files according to selected languages - const os = await import('os'); - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'cd-submission-')); - cwd = tmp; - const repoUrl = (opts.baseUrl && opts.baseUrl.includes('api/v3')) - ? opts.baseUrl.replace(/\/api\/v3$/, '') + `/${opts.owner}/${opts.repo}.git` - : `https://github.com/${opts.owner}/${opts.repo}.git`; - const patterns = buildSparsePatterns(intersect); - // init repo - await execGit(['init'], { cwd }); - await execGit(['remote', 'add', 'origin', repoUrl], { cwd }); - await execGit(['config', 'core.sparseCheckout', 'true'], { cwd }); - fs.mkdirSync(path.join(cwd, '.git', 'info'), { recursive: true }); - fs.writeFileSync(path.join(cwd, '.git', 'info', 'sparse-checkout'), patterns.join('\n') + '\n', 'utf8'); - await execGit(['fetch', '--depth=1', 'origin', opts.branch], { cwd }); - await execGit(['checkout', 'FETCH_HEAD'], { cwd }); - } catch (e) { - if (!opts.quiet) console.error(chalk.red(`Sparse checkout failed: ${(e as Error).message}`)); - return false; } - } - - // Run the action entrypoint pointing at the sparse checkout dir (or root if none) - await new Promise((resolve, reject) => { - const env = { - ...process.env, - GITHUB_TOKEN: token, - GITHUB_BASE_URL: opts.baseUrl || process.env.GITHUB_BASE_URL || '', - }; - const child = spawn('node', [entry], { env, stdio: opts.quiet ? 'ignore' : 'inherit', cwd }); - child.on('error', reject); - child.on('exit', code => code === 0 ? resolve() : reject(new Error(`entrypoint exit ${code}`))); - }); - return true; + + // If languages filter provided, inspect repo languages and perform sparse checkout of relevant manifests + if (opts.languages && opts.languages.length) { + const { createOctokit } = await import('./octokit.js'); + const o = createOctokit({ token, baseUrl: opts.baseUrl }); + try { + const langResp = await o.request('GET /repos/{owner}/{repo}/languages', { owner: opts.owner, repo: opts.repo }); + const repoLangs = Object.keys(langResp.data || {}); + const wanted = opts.languages; + const intersect = repoLangs.filter(l => wanted.some(w => w.toLowerCase() === l.toLowerCase())); + if (!intersect.length) { + if (!opts.quiet) console.error(chalk.yellow(`Skipping submission: none of selected languages present in repo (${repoLangs.join(', ')})`)); + return false; + } + // Create temp dir and sparse checkout only manifest files according to selected languages + const os = await import('os'); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'cd-submission-')); + const cwd = tmp; + const repoUrl = (opts.baseUrl && opts.baseUrl.includes('api/v3')) + ? opts.baseUrl.replace(/\/api\/v3$/, '') + `/${opts.owner}/${opts.repo}.git` + : `https://github.com/${opts.owner}/${opts.repo}.git`; + const patterns = buildSparsePatterns(intersect); + // init repo + await execGit(['init'], { cwd }); + await execGit(['remote', 'add', 'origin', repoUrl], { cwd }); + await execGit(['config', 'core.sparseCheckout', 'true'], { cwd }); + fs.mkdirSync(path.join(cwd, '.git', 'info'), { recursive: true }); + fs.writeFileSync(path.join(cwd, '.git', 'info', 'sparse-checkout'), patterns.join('\n') + '\n', 'utf8'); + await execGit(['fetch', '--depth=1', 'origin', opts.branch], { cwd }); + await execGit(['checkout', 'FETCH_HEAD'], { cwd }); + + // Run the ComponentDetection module to detect components and submit snapshot + const ref = (await execGit(['rev-parse', 'HEAD'], { cwd }))?.stdout?.toString().trim(); + const sha = (await execGit(['rev-parse', 'HEAD'], { cwd }))?.stdout?.toString().trim(); + if (!sha || !ref) { + if (!opts.quiet) console.error(chalk.red(`Failed to determine SHA or ref for ${opts.owner}/${opts.repo} on branch ${opts.branch}`)); + return false; + } + await run(path.resolve(opts.owner, opts.repo), sha, ref); + + } catch (e) { + if (!opts.quiet) console.error(chalk.red(`Sparse checkout failed: ${(e as Error).message}`)); + return false; + } + } + + return false; } function buildSparsePatterns(langs: string[]): string[] { - const set = new Set(); - const add = (p: string) => set.add(p); - for (const l of langs) { - const ll = l.toLowerCase(); - if (ll === 'javascript' || ll === 'typescript') { - add('**/package.json'); - add('**/package-lock.json'); - add('**/yarn.lock'); - add('**/pnpm-lock.yaml'); - } else if (ll === 'python') { - add('**/requirements.txt'); - add('**/Pipfile.lock'); - add('**/poetry.lock'); - add('**/pyproject.toml'); - } else if (ll === 'go') { - add('**/go.mod'); - add('**/go.sum'); - } else if (ll === 'ruby') { - add('**/Gemfile.lock'); - add('**/gems.locked'); - } else if (ll === 'rust') { - add('**/Cargo.toml'); - add('**/Cargo.lock'); - } else if (ll === 'java') { - // Maven & Gradle - add('**/pom.xml'); - add('**/build.gradle'); - add('**/build.gradle.kts'); - add('**/settings.gradle'); - add('**/settings.gradle.kts'); - add('**/gradle.lockfile'); - } else if (ll === 'c#' || ll === 'csharp') { - add('**/packages.lock.json'); - add('**/*.csproj'); - add('**/*.sln'); + const set = new Set(); + const add = (p: string) => set.add(p); + for (const l of langs) { + const ll = l.toLowerCase(); + if (ll === 'javascript' || ll === 'typescript') { + add('**/package.json'); + add('**/package-lock.json'); + add('**/yarn.lock'); + add('**/pnpm-lock.yaml'); + } else if (ll === 'python') { + add('**/requirements.txt'); + add('**/Pipfile.lock'); + add('**/poetry.lock'); + add('**/pyproject.toml'); + } else if (ll === 'go') { + add('**/go.mod'); + add('**/go.sum'); + } else if (ll === 'ruby') { + add('**/Gemfile.lock'); + add('**/gems.locked'); + } else if (ll === 'rust') { + add('**/Cargo.toml'); + add('**/Cargo.lock'); + } else if (ll === 'java') { + // Maven & Gradle + add('**/pom.xml'); + add('**/build.gradle'); + add('**/build.gradle.kts'); + add('**/settings.gradle'); + add('**/settings.gradle.kts'); + add('**/gradle.lockfile'); + } else if (ll === 'c#' || ll === 'csharp') { + add('**/packages.lock.json'); + add('**/*.csproj'); + add('**/*.sln'); + } } - } - // Always include root lockfiles just in case - add('package.json'); add('package-lock.json'); add('yarn.lock'); add('pnpm-lock.yaml'); - return Array.from(set); + // Always include root lockfiles just in case + add('package.json'); add('package-lock.json'); add('yarn.lock'); add('pnpm-lock.yaml'); + return Array.from(set); } -async function execGit(args: string[], opts: { cwd: string }): Promise { - await new Promise((resolve, reject) => { - const child = spawn('git', args, { cwd: opts.cwd, stdio: 'inherit' }); - child.on('error', reject); - child.on('exit', code => code === 0 ? resolve() : reject(new Error(`git ${args.join(' ')} exit ${code}`))); - }); +async function execGit(args: string[], opts: { cwd: string, quiet?: boolean }): Promise { + return await new Promise((resolve, reject) => { + const child = spawn('git', args, { cwd: opts.cwd, stdio: opts.quiet ? 'ignore' : 'inherit' }); + child.on('error', reject); + child.on('exit', code => code === 0 ? resolve(child) : reject(new Error(`git ${args.join(' ')} exit ${code}`))); + }); +} + +export async function run(filePath: string, sha: string, ref: string) { + let manifests = await ComponentDetection.scanAndGetManifests( + filePath + ); + + // Get detector configuration inputs + const detectorName = "Component Detection in GitHub SBOM Toolkit: advanced-security/github-sbom-toolkit"; + const detectorVersion = "0.0.1"; + const detectorUrl = "https://github.com/advanced-security/github-sbom-toolkit"; + + // Use provided detector config or defaults + const detector = { + name: detectorName, + version: detectorVersion, + url: detectorUrl, + }; + + const context: Context = { payload: {}, eventName: '', sha: '', ref: '', workflow: '', action: '', actor: '', job: '', runNumber: 0, runId: 0, runAttempt: 0, apiUrl: '', serverUrl: '', graphqlUrl: '', issue: { owner: '', repo: '', number: 0 }, repo: { owner: '', repo: '' } }; + + context.sha = sha; + context.ref = ref; + context.job = 'github-sbom-toolkit'; + context.runId = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); + + let snapshot = new Snapshot(detector, context); + + manifests?.forEach((manifest) => { + snapshot.addManifest(manifest); + }); + + submitSnapshot(snapshot); } diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index d68faf2..2abf658 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -1,6 +1,6 @@ import { createOctokit } from "./octokit.js"; import type { RepositorySbom, CollectionSummary, SbomPackage, Sbom, BranchSbom, BranchDependencyDiff, DependencyReviewPackageChange } from "./types.js"; -import { submitSnapshotIfPossible } from "./componentSubmission.js"; +import { run } from "./componentSubmission.js"; import * as semver from "semver"; import { readAll, writeOne } from "./serialization.js"; // p-limit lacks bundled types in some versions; declare minimal shape @@ -13,6 +13,7 @@ export interface CollectorOptions { token: string | undefined; // GitHub token with repo + security_events scope enterprise?: string; // Enterprise slug to enumerate orgs org?: string; // Single org alternative + repo?: string; // Single repo alternative baseUrl?: string; // For GHES concurrency?: number; // parallel repo SBOM fetches includePrivate?: boolean; @@ -42,8 +43,8 @@ export class SbomCollector { private decisions: Record = {}; // repo -> reason constructor(options: CollectorOptions) { - if (!options.loadFromDir && !options.enterprise && !options.org) { - throw new Error("Either enterprise/org or loadFromDir must be specified"); + if (!options.loadFromDir && !options.enterprise && !options.org && !options.repo) { + throw new Error("One of enterprise/org/repo or loadFromDir must be specified"); } // Spread user options first then apply defaults via nullish coalescing so that // passing undefined does not erase defaults @@ -52,6 +53,7 @@ export class SbomCollector { token: o.token, enterprise: o.enterprise, org: o.org, + repo: o.repo, baseUrl: o.baseUrl, concurrency: o.concurrency ?? 5, includePrivate: o.includePrivate ?? true, @@ -108,8 +110,8 @@ export class SbomCollector { async collect(): Promise { // Offline mode: load from directory if provided if (this.opts.loadFromDir) { - // find just the path for a single org, if given - const loadPath = this.opts.org ? `${this.opts.loadFromDir}/${this.opts.org}` : this.opts.loadFromDir; + // find just the path for a single org or repo, if given + const loadPath = this.opts.org ? `${this.opts.loadFromDir}/${this.opts.org}` : this.opts.repo ? `${this.opts.loadFromDir}/${this.opts.repo}` : this.opts.loadFromDir; if (!this.opts.quiet) process.stderr.write(chalk.blue(`Loading SBOMs from cache at ${loadPath}`) + "\n"); @@ -145,19 +147,28 @@ export class SbomCollector { process.stderr.write(chalk.blue(`Getting list of organizations for enterprise ${this.opts.enterprise}`) + "\n"); } - const orgs = this.opts.org ? [this.opts.org] : await this.listEnterpriseOrgs(this.opts.enterprise!); + const orgs = this.opts.org ? [this.opts.org] : this.opts.enterprise ? await this.listEnterpriseOrgs(this.opts.enterprise!) : [this.opts.repo.split("/")[0]]; this.summary.orgs = orgs; // Pre-list all repos if showing progress bar so we know the total upfront const orgRepoMap: Record = {}; let totalRepos = 0; - for (const org of orgs) { - if (!this.opts.quiet) process.stderr.write(chalk.blue(`Listing repositories for org ${org}`) + "\n"); - if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); - const repos = await this.listOrgRepos(org); - orgRepoMap[org] = repos; - totalRepos += repos.length; + + if (!this.opts.repo) { + for (const org of orgs) { + if (!this.opts.quiet) process.stderr.write(chalk.blue(`Listing repositories for org ${org}`) + "\n"); + if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); + const repos = await this.listOrgRepos(org); + orgRepoMap[org] = repos; + totalRepos += repos.length; + } + } else { + totalRepos = 1; + const [org, repoName] = this.opts.repo.split("/"); + orgRepoMap[org] = [await this.getRepo(org, repoName)]; + this.summary.orgs = orgs; } + this.summary.repositoryCount = totalRepos; let processed = 0; @@ -248,27 +259,26 @@ export class SbomCollector { } // Branch scanning (optional) - if (this.opts.includeBranches && res.defaultBranch) { + // TODO: do this even if we have the main SBOM, since we may not have branch diffs + // implement some check to see if the diff info we have is already fresher than the branch info + if (this.opts.includeBranches && res.sbom) { console.log(chalk.blue(`Scanning branches for ${fullName}...`)); try { const branches = await this.listBranches(org, repo.name); const nonDefault = branches.filter(b => b.name !== res.defaultBranch); const limited = this.opts.branchLimit && this.opts.branchLimit > 0 ? nonDefault.slice(0, this.opts.branchLimit) : nonDefault; - const branchSboms: BranchSbom[] = []; const branchDiffs: BranchDependencyDiff[] = []; for (const b of limited) { if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); - const bSbom = await this.fetchBranchSbom(org, repo.name, b.name, b.commit?.sha); - branchSboms.push(bSbom); const base = this.opts.branchDiffBase || res.defaultBranch; - if (this.opts.includeDependencyReviewDiffs) { + if (!base) { console.error(chalk.red(`Cannot compute branch diff for ${fullName} branch ${b.name} because base branch is undefined.`)); continue; } + if (base && this.opts.includeDependencyReviewDiffs) { if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); const diff = await this.fetchDependencyReviewDiff(org, repo.name, base, b.name); branchDiffs.push(diff); } } - if (branchSboms.length) res.branchSboms = branchSboms; if (branchDiffs.length) res.branchDiffs = branchDiffs; } catch (e) { // Non-fatal; annotate decision @@ -345,6 +355,19 @@ export class SbomCollector { return repos; } + private async getRepo(org: string, repo: string): Promise<{ name: string; pushed_at?: string; updated_at?: string; default_branch?: string }> { + if (!this.octokit) throw new Error("No Octokit instance"); + + try { + const resp = await this.octokit.request("GET /repos/{owner}/{repo}", { owner: org, repo }); + const data = resp.data as { name: string; pushed_at?: string; updated_at?: string; default_branch?: string }; + return data; + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + throw new Error(`Failed to get repo metadata for ${org}/${repo}: ${msg}`); + } + } + private async fetchSbom(org: string, repo: string, repoMeta?: { pushed_at?: string; updated_at?: string; default_branch?: string }): Promise { if (!this.octokit) throw new Error("No Octokit instance"); @@ -399,18 +422,6 @@ export class SbomCollector { return branches; } - private async fetchBranchSbom(org: string, repo: string, branch: string, commitSha?: string): Promise { - if (!this.octokit) throw new Error("No Octokit instance"); - try { - const resp = await this.octokit.request("GET /repos/{owner}/{repo}/dependency-graph/sbom", { owner: org, repo, ref: branch, headers: { Accept: "application/vnd.github+json" } }); - const sbomWrapper = resp.data as { sbom?: Sbom }; - const packages: SbomPackage[] = sbomWrapper?.sbom?.packages ?? []; - return { branch, commitSha, retrievedAt: new Date().toISOString(), sbom: sbomWrapper?.sbom, packages }; - } catch (e) { - return { branch, commitSha, retrievedAt: new Date().toISOString(), packages: [], error: e instanceof Error ? e.message : String(e) }; - } - } - private async fetchDependencyReviewDiff(org: string, repo: string, base: string, head: string): Promise { if (!this.octokit) throw new Error("No Octokit instance"); try { @@ -442,13 +453,16 @@ export class SbomCollector { reason = "Dependency review unavailable (missing snapshot or feature disabled)"; // Optional retry path: submit snapshot then retry once if (this.opts.submitOnMissingSnapshot) { + console.log(chalk.blue(`Attempting to submit component snapshot for ${org}/${repo} branch ${head} before retrying dependency review diff...`)); try { const ok = await this.trySubmitSnapshot(org, repo, head); if (ok) { + console.log(chalk.blue(`Snapshot submission attempted; waiting 3 seconds before retrying dependency review diff for ${org}/${repo} ${base}...${head}...`)); await new Promise(r => setTimeout(r, 3000)); return await this.fetchDependencyReviewDiff(org, repo, base, head); } } catch (subErr) { + console.error(chalk.red(`Snapshot submission failed for ${org}/${repo} branch ${head}: ${(subErr as Error).message}`)); reason += ` (submission attempt failed: ${(subErr as Error).message})`; } } @@ -457,17 +471,11 @@ export class SbomCollector { } } + // TODO: attach to 'run' from componentSubmission.ts with appropriate parameters private async trySubmitSnapshot(org: string, repo: string, branch: string): Promise { - // Dynamically import to avoid hard dependency when submodule not present - try { - const mod = await import("./componentSubmission.js"); - if (typeof mod.submitSnapshotIfPossible === "function") { - return await mod.submitSnapshotIfPossible({ owner: org, repo, branch, token: this.opts.token, baseUrl: this.opts.baseUrl, caBundlePath: this.opts.caBundlePath, quiet: this.opts.quiet, languages: this.opts.submitLanguages }); - } - return false; - } catch { - return false; - } + return new Promise(async (resolve, reject) => { + reject(new Error("Not implemented: snapshot submission requires additional context and is not implemented in this example.")); + }); } // New method including the query that produced each match diff --git a/tmp-branch-search-cache/example-org/demo-repo/sbom.json b/tmp-branch-search-cache/example-org/demo-repo/sbom.json new file mode 100644 index 0000000..9dcf7cb --- /dev/null +++ b/tmp-branch-search-cache/example-org/demo-repo/sbom.json @@ -0,0 +1,59 @@ +{ + "repo": "example-org/demo-repo", + "org": "example-org", + "retrievedAt": "2025-11-26T15:02:48.754Z", + "packages": [ + { + "name": "chalk", + "version": "5.6.1", + "purl": "pkg:npm/chalk@5.6.1" + }, + { + "name": "react", + "version": "18.2.0", + "purl": "pkg:npm/react@18.2.0" + } + ], + "branchSboms": [ + { + "branch": "feature-x", + "retrievedAt": "2025-11-26T15:02:48.755Z", + "packages": [ + { + "name": "react", + "version": "18.3.0-beta", + "purl": "pkg:npm/react@18.3.0-beta" + }, + { + "name": "lodash", + "version": "4.17.21", + "purl": "pkg:npm/lodash@4.17.21" + } + ] + } + ], + "branchDiffs": [ + { + "base": "main", + "head": "feature-x", + "retrievedAt": "2025-11-26T15:02:48.755Z", + "changes": [ + { + "changeType": "added", + "name": "lodash", + "ecosystem": "npm", + "purl": "pkg:npm/lodash@4.17.21", + "newVersion": "4.17.21" + }, + { + "changeType": "updated", + "name": "react", + "ecosystem": "npm", + "purl": "pkg:npm/react@18.3.0-beta", + "previousVersion": "18.2.0", + "newVersion": "18.3.0-beta" + } + ] + } + ] +} \ No newline at end of file From 506b9a226db7b3a8d39ba9b0462872153beef12a Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Thu, 27 Nov 2025 18:30:53 +0000 Subject: [PATCH 006/108] Runs component-detection now, but is killed on MacOS --- .gitignore | 3 ++- src/componentDetection.ts | 16 ++++++++---- src/componentSubmission.ts | 53 +++++++++++++++++++------------------- src/sbomCollector.ts | 13 +++------- 4 files changed, 42 insertions(+), 43 deletions(-) diff --git a/.gitignore b/.gitignore index c3c66e4..fc00982 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ dist/ .env data/ .vscode/ -.DS_Store \ No newline at end of file +.DS_Store +component-detection \ No newline at end of file diff --git a/src/componentDetection.ts b/src/componentDetection.ts index f57e9f1..716b487 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -23,10 +23,14 @@ export default class ComponentDetection { } // Get the latest release from the component-detection repo, download the tarball, and extract it public static async downloadLatestRelease() { - const statResult = fs.statSync(this.componentDetectionPath); - if (statResult && statResult.isFile()) { - console.debug(`Component-detection binary already exists at ${this.componentDetectionPath}, skipping download.`); - return; + try { + const statResult = fs.statSync(this.componentDetectionPath); + if (statResult && statResult.isFile()) { + console.debug(`Component-detection binary already exists at ${this.componentDetectionPath}, skipping download.`); + return; + } + } catch (error) { + // File does not exist, proceed to download } try { @@ -254,7 +258,9 @@ export default class ComponentDetection { const latestRelease = await octokit.request("GET /repos/{owner}/{repo}/releases/latest", {owner, repo}); var downloadURL: string = ""; - const assetName = process.platform === "win32" ? "component-detection-win-x64.exe" : "component-detection-linux-x64"; + // TODO: do we need to handle different architectures here? + // can we allow x64 on MacOS? We could allow an input parameter to override? + const assetName = process.platform === "win32" ? "component-detection-win-x64.exe" : process.platform === "linux" ? "component-detection-linux-x64" : "component-detection-osx-arm64"; latestRelease.data.assets.forEach((asset: any) => { if (asset.name === assetName) { downloadURL = asset.browser_download_url; diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index ba6aa47..f973b56 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -2,15 +2,17 @@ import chalk from 'chalk'; import { spawn, ChildProcess } from 'child_process'; import path from 'path'; import fs from 'fs'; +import os from 'os'; import type { Context } from '@actions/github/lib/context.js' -import ComponentDetection from './componentDetection'; +import ComponentDetection from './componentDetection.js'; import { Snapshot, submitSnapshot } from '@github/dependency-submission-toolkit'; export interface SubmitOpts { + octokit?: any; // Octokit instance, optional owner: string; repo: string; branch: string; @@ -21,23 +23,15 @@ export interface SubmitOpts { languages?: string[]; } -// This helper attempts to run the Component Detection + Dependency Submission action -// as a local script, assuming the repository has the submodule checked out at -// `component-detection-dependency-submission-action`. -// It falls back to returning false if not available. export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise { - const token = opts.token || process.env.GITHUB_TOKEN || ''; - if (!token) { - if (!opts.quiet) console.error(chalk.red('GITHUB_TOKEN required to submit dependency snapshot')); - return false; + if (!opts.octokit) { + throw new Error('Octokit instance is required in opts.octokit'); } // If languages filter provided, inspect repo languages and perform sparse checkout of relevant manifests if (opts.languages && opts.languages.length) { - const { createOctokit } = await import('./octokit.js'); - const o = createOctokit({ token, baseUrl: opts.baseUrl }); try { - const langResp = await o.request('GET /repos/{owner}/{repo}/languages', { owner: opts.owner, repo: opts.repo }); + const langResp = await opts.octokit.request('GET /repos/{owner}/{repo}/languages', { owner: opts.owner, repo: opts.repo }); const repoLangs = Object.keys(langResp.data || {}); const wanted = opts.languages; const intersect = repoLangs.filter(l => wanted.some(w => w.toLowerCase() === l.toLowerCase())); @@ -46,7 +40,6 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise { return await new Promise((resolve, reject) => { - const child = spawn('git', args, { cwd: opts.cwd, stdio: opts.quiet ? 'ignore' : 'inherit' }); + const child = spawn('git', args, { cwd: opts.cwd, stdio: 'pipe' }); child.on('error', reject); child.on('exit', code => code === 0 ? resolve(child) : reject(new Error(`git ${args.join(' ')} exit ${code}`))); }); } -export async function run(filePath: string, sha: string, ref: string) { +export async function run(owner: string, repo: string, sha: string, ref: string) { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sbom-')); + let manifests = await ComponentDetection.scanAndGetManifests( - filePath + tmpDir ); // Get detector configuration inputs @@ -148,12 +144,15 @@ export async function run(filePath: string, sha: string, ref: string) { url: detectorUrl, }; - const context: Context = { payload: {}, eventName: '', sha: '', ref: '', workflow: '', action: '', actor: '', job: '', runNumber: 0, runId: 0, runAttempt: 0, apiUrl: '', serverUrl: '', graphqlUrl: '', issue: { owner: '', repo: '', number: 0 }, repo: { owner: '', repo: '' } }; - - context.sha = sha; - context.ref = ref; - context.job = 'github-sbom-toolkit'; - context.runId = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); + const context: Context = { + repo: { owner: owner, repo: repo }, + job: 'github-sbom-toolkit', + runId: Math.floor(Math.random() * Number.MAX_SAFE_INTEGER), + ref: ref, + sha: sha, + // required for Context type but not used in snapshot submission + payload: {}, eventName: '', workflow: '', action: '', actor: '', runNumber: 0, runAttempt: 0, apiUrl: '', serverUrl: '', graphqlUrl: '', issue: { owner: '', repo: '', number: 0 } + }; let snapshot = new Snapshot(detector, context); diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 2abf658..5d26d7c 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -1,8 +1,8 @@ import { createOctokit } from "./octokit.js"; -import type { RepositorySbom, CollectionSummary, SbomPackage, Sbom, BranchSbom, BranchDependencyDiff, DependencyReviewPackageChange } from "./types.js"; -import { run } from "./componentSubmission.js"; +import type { RepositorySbom, CollectionSummary, SbomPackage, Sbom, BranchDependencyDiff, DependencyReviewPackageChange } from "./types.js"; import * as semver from "semver"; import { readAll, writeOne } from "./serialization.js"; +import { submitSnapshotIfPossible } from "./componentSubmission.js"; // p-limit lacks bundled types in some versions; declare minimal shape // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore @@ -455,7 +455,7 @@ export class SbomCollector { if (this.opts.submitOnMissingSnapshot) { console.log(chalk.blue(`Attempting to submit component snapshot for ${org}/${repo} branch ${head} before retrying dependency review diff...`)); try { - const ok = await this.trySubmitSnapshot(org, repo, head); + const ok = await submitSnapshotIfPossible({ octokit: this.octokit, owner: org, repo: repo, branch: head, languages: this.opts.submitLanguages, quiet: this.opts.quiet }); if (ok) { console.log(chalk.blue(`Snapshot submission attempted; waiting 3 seconds before retrying dependency review diff for ${org}/${repo} ${base}...${head}...`)); await new Promise(r => setTimeout(r, 3000)); @@ -471,13 +471,6 @@ export class SbomCollector { } } - // TODO: attach to 'run' from componentSubmission.ts with appropriate parameters - private async trySubmitSnapshot(org: string, repo: string, branch: string): Promise { - return new Promise(async (resolve, reject) => { - reject(new Error("Not implemented: snapshot submission requires additional context and is not implemented in this example.")); - }); - } - // New method including the query that produced each match searchByPurlsWithReasons(purls: string[]): Map { purls = purls.map(q => q.startsWith("pkg:") ? q : `pkg:${q}`); From 68f5d22b24c53312c0c67699948686f29ff97529 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Fri, 28 Nov 2025 11:44:16 +0000 Subject: [PATCH 007/108] Allow using component-detection at a provided path --- README.md | 35 +++++++++++++++++++++++++++++++++-- src/cli.ts | 5 +++++ src/componentDetection.ts | 9 +++++++-- src/componentSubmission.ts | 8 +++++--- src/sbomCollector.ts | 4 +++- 5 files changed, 53 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 819db7e..9303afd 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,37 @@ Build the action (if not already built) so its `dist/entrypoint.js` exists. The If submission fails, the original 404 reason is retained and collection proceeds. +##### Using a Local Component Detection Binary + +Instead of downloading the latest release automatically, you can point the toolkit at a local `component-detection` executable. This is useful if you already manage the binary or need a custom build. + +Pass the path via `--component-detection-bin` and optionally limit languages to reduce sparse checkout size: + +```bash +npm run start -- \ + --sync-sboms --org my-org --sbom-cache sboms \ + --branch-scan --submit-on-missing-snapshot \ + --submit-languages JavaScript,TypeScript \ + --component-detection-bin /usr/local/bin/component-detection +``` + +GitHub Enterprise Server example: + +```bash +npm run start -- \ + --sync-sboms --org my-org --sbom-cache sboms \ + --base-url https://ghe.example.com/api/v3 \ + --branch-scan --submit-on-missing-snapshot \ + --submit-languages Python \ + --component-detection-bin /opt/tools/component-detection +``` + +Notes: + +- Providing `--component-detection-bin` skips any download logic and uses your binary directly. +- Snapshot submission performs a language-aware sparse checkout of common manifest/lock files (e.g., `package.json`, `requirements.txt`, `pom.xml`). +- After submission, the toolkit waits briefly and retries the dependency review diff once. + ### 🔑 Authentication A GitHub token with appropriate scope is required when performing network operations such as `--sync-sboms`, `--sync-malware` and `--upload-sarif`. @@ -153,7 +184,7 @@ npm run start -- --sbom-cache sboms --purl-file queries.txt npm run start -- --sync-sboms --org my-org --sbom-cache sboms ``` -2. Later offline search (no API calls; uses previously written per‑repo JSON): +1. Later offline search (no API calls; uses previously written per‑repo JSON): ```bash npm run start -- --sbom-cache sboms --purl pkg:npm/react@18.2.0 @@ -397,7 +428,7 @@ npm install npm run build ``` -2. Run the test harness script: +1. Run the test harness script: ```bash node dist/test-fixture-match.js diff --git a/src/cli.ts b/src/cli.ts index 2bd5050..2717cbc 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -46,6 +46,7 @@ async function main() { .option("diff-base", { type: "string", describe: "Override base branch for dependency review diffs (defaults to default branch)" }) .option("submit-on-missing-snapshot", { type: "boolean", default: false, describe: "When dependency review diff returns 404 (missing snapshot), run Component Detection to submit a snapshot, then retry." }) .option("submit-languages", { type: "array", describe: "Limit snapshot submission to these languages (e.g., JavaScript,TypeScript,Python,Maven)." }) + .option("component-detection-bin", { type: "string", describe: "Path to a local component-detection executable to use for snapshot submission (skips download)." }) .check(args => { const syncing = !!args.syncSboms; if (syncing) { @@ -117,6 +118,10 @@ async function main() { branchDiffBase: argv["diff-base"] as string | undefined, submitOnMissingSnapshot: argv["submit-on-missing-snapshot"] as boolean, submitLanguages: (argv["submit-languages"] as string[] | undefined) || undefined, + // Pass through as part of options bag used by submission helper via collector + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + componentDetectionBinPath: argv["component-detection-bin"] as string | undefined, }); if (!quiet) process.stderr.write(chalk.cyan(offline ? "Loading SBOMs from cache..." : "Collecting SBOMs from cache & GitHub...") + "\n"); diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 716b487..6ae8ee0 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -16,8 +16,13 @@ export default class ComponentDetection { public static outputPath = './output.json'; // This is the default entry point for this class. - static async scanAndGetManifests(path: string): Promise { - await this.downloadLatestRelease(); + // If executablePath is provided, use it directly and skip download. + static async scanAndGetManifests(path: string, executablePath?: string): Promise { + if (executablePath) { + this.componentDetectionPath = executablePath; + } else { + await this.downloadLatestRelease(); + } await this.runComponentDetection(path); return await this.getManifestsFromResults(); } diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index f973b56..3ee8d23 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -21,6 +21,7 @@ export interface SubmitOpts { caBundlePath?: string; quiet?: boolean; languages?: string[]; + componentDetectionBinPath?: string; // optional path to component-detection executable } export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise { @@ -63,7 +64,7 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise; if (this.opts.token) { @@ -455,7 +457,7 @@ export class SbomCollector { if (this.opts.submitOnMissingSnapshot) { console.log(chalk.blue(`Attempting to submit component snapshot for ${org}/${repo} branch ${head} before retrying dependency review diff...`)); try { - const ok = await submitSnapshotIfPossible({ octokit: this.octokit, owner: org, repo: repo, branch: head, languages: this.opts.submitLanguages, quiet: this.opts.quiet }); + const ok = await submitSnapshotIfPossible({ octokit: this.octokit, owner: org, repo: repo, branch: head, languages: this.opts.submitLanguages, quiet: this.opts.quiet, componentDetectionBinPath: this.opts.componentDetectionBinPath }); if (ok) { console.log(chalk.blue(`Snapshot submission attempted; waiting 3 seconds before retrying dependency review diff for ${org}/${repo} ${base}...${head}...`)); await new Promise(r => setTimeout(r, 3000)); From 51a111ec5617a572b43cb02e2d56f1c1b91388ce Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Fri, 28 Nov 2025 16:17:04 +0000 Subject: [PATCH 008/108] Refactor, make branch submission independent of freshly fetching SBOM --- src/componentDetection.ts | 2 +- src/componentSubmission.ts | 62 +++++++++++++++++++++++--------------- src/sbomCollector.ts | 59 ++++++++++++++++++++---------------- 3 files changed, 71 insertions(+), 52 deletions(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 6ae8ee0..39e0b51 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -58,7 +58,7 @@ export default class ComponentDetection { console.info("Running component-detection"); try { - await spawn(`${this.componentDetectionPath}`, ['scan', '--SourceDirectory', path, '--ManifestFile', this.outputPath, ...this.getComponentDetectionParameters()]); + await spawn(`${this.componentDetectionPath}`, ['scan', '--SourceDirectory', path, '--ManifestFile', this.outputPath, ...this.getComponentDetectionParameters()], { stdio: 'pipe' }); } catch (error: any) { console.error(error); } diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index 3ee8d23..db264ee 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -24,6 +24,38 @@ export interface SubmitOpts { componentDetectionBinPath?: string; // optional path to component-detection executable } +export async function getLanguageIntersection(octokit: any, owner: string, repo: string, languages: string[], quiet: boolean = false): Promise { + const langResp = await octokit.request('GET /repos/{owner}/{repo}/languages', { owner, repo }); + const repoLangs = Object.keys(langResp.data || {}); + const wanted = languages; + const intersect = repoLangs.filter(l => wanted.some(w => w.toLowerCase() === l.toLowerCase())); + if (!intersect.length) { + if (!quiet) console.error(chalk.yellow(`Skipping submission: none of selected languages present in repo (${repoLangs.join(', ')})`)); + return []; + } + return intersect; +} + +export async function sparseCheckout(owner: string, repo: string, branch: string, destDir: string, intersect: string[], baseUrl?: string) { + const cwd = destDir; + const repoUrl = (baseUrl && baseUrl.includes('api/v3')) + ? baseUrl.replace(/\/api\/v3$/, '') + `/${owner}/${repo}.git` + : `https://github.com/${owner}/${repo}.git`; + const patterns = buildSparsePatterns(intersect); + // init repo + await execGit(['init'], { cwd }); + await execGit(['remote', 'add', 'origin', repoUrl], { cwd }); + await execGit(['config', 'core.sparseCheckout', 'true'], { cwd }); + fs.mkdirSync(path.join(cwd, '.git', 'info'), { recursive: true }); + fs.writeFileSync(path.join(cwd, '.git', 'info', 'sparse-checkout'), patterns.join('\n') + '\n', 'utf8'); + await execGit(['fetch', '--depth=1', 'origin', branch], { cwd }); + await execGit(['checkout', 'FETCH_HEAD'], { cwd }); + + const process = await execGit(['rev-parse', 'HEAD'], { cwd: destDir }); + const sha = process?.stdout?.toString().trim(); + return sha; +} + export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise { if (!opts.octokit) { throw new Error('Octokit instance is required in opts.octokit'); @@ -32,34 +64,14 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise wanted.some(w => w.toLowerCase() === l.toLowerCase())); - if (!intersect.length) { - if (!opts.quiet) console.error(chalk.yellow(`Skipping submission: none of selected languages present in repo (${repoLangs.join(', ')})`)); - return false; - } + const intersect = await getLanguageIntersection(opts.octokit, opts.owner, opts.repo, opts.languages); // Create temp dir and sparse checkout only manifest files according to selected languages const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'cd-submission-')); - const cwd = tmp; - const repoUrl = (opts.baseUrl && opts.baseUrl.includes('api/v3')) - ? opts.baseUrl.replace(/\/api\/v3$/, '') + `/${opts.owner}/${opts.repo}.git` - : `https://github.com/${opts.owner}/${opts.repo}.git`; - const patterns = buildSparsePatterns(intersect); - // init repo - await execGit(['init'], { cwd }); - await execGit(['remote', 'add', 'origin', repoUrl], { cwd }); - await execGit(['config', 'core.sparseCheckout', 'true'], { cwd }); - fs.mkdirSync(path.join(cwd, '.git', 'info'), { recursive: true }); - fs.writeFileSync(path.join(cwd, '.git', 'info', 'sparse-checkout'), patterns.join('\n') + '\n', 'utf8'); - await execGit(['fetch', '--depth=1', 'origin', opts.branch], { cwd }); - await execGit(['checkout', 'FETCH_HEAD'], { cwd }); + console.log(chalk.green(`Sparse checkout into ${tmp} for languages: ${intersect.join(', ')}`)); - // Run the ComponentDetection module to detect components and submit snapshot - const process = await execGit(['rev-parse', 'HEAD'], { cwd }); - const sha = process?.stdout?.toString().trim(); + const sha = await sparseCheckout(opts.owner, opts.repo, opts.branch, tmp, intersect, opts.baseUrl); + // Run the ComponentDetection module to detect components and submit snapshot if (!sha) { if (!opts.quiet) console.error(chalk.red(`Failed to determine SHA for ${opts.owner}/${opts.repo} on branch ${opts.branch}`)); return false; @@ -67,7 +79,7 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise b.name !== res.defaultBranch); - const limited = this.opts.branchLimit && this.opts.branchLimit > 0 ? nonDefault.slice(0, this.opts.branchLimit) : nonDefault; - const branchDiffs: BranchDependencyDiff[] = []; - for (const b of limited) { + sbom = res; + } else { + sbom = baseline; + } + + // Branch scanning (optional) + // TODO: implement some check to see if the diff info we have is already fresher than the branch info + if (this.opts.includeBranches && sbom?.sbom) { + + console.log(chalk.blue(`Scanning branches for ${fullName}...`)); + + try { + const branches = await this.listBranches(org, repo.name); + const nonDefault = branches.filter(b => b.name !== sbom.defaultBranch); + const limited = this.opts.branchLimit && this.opts.branchLimit > 0 ? nonDefault.slice(0, this.opts.branchLimit) : nonDefault; + const branchDiffs: BranchDependencyDiff[] = []; + for (const b of limited) { + if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); + const base = this.opts.branchDiffBase || sbom?.defaultBranch; + if (!base) { console.error(chalk.red(`Cannot compute branch diff for ${fullName} branch ${b.name} because base branch is undefined.`)); continue; } + if (base && this.opts.includeDependencyReviewDiffs) { if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); - const base = this.opts.branchDiffBase || res.defaultBranch; - if (!base) { console.error(chalk.red(`Cannot compute branch diff for ${fullName} branch ${b.name} because base branch is undefined.`)); continue; } - if (base && this.opts.includeDependencyReviewDiffs) { - if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); - const diff = await this.fetchDependencyReviewDiff(org, repo.name, base, b.name); - branchDiffs.push(diff); - } + const diff = await this.fetchDependencyReviewDiff(org, repo.name, base, b.name); + branchDiffs.push(diff); } - if (branchDiffs.length) res.branchDiffs = branchDiffs; - } catch (e) { - // Non-fatal; annotate decision - this.decisions[fullName] = (this.decisions[fullName] || "") + ` (branch scan error: ${(e as Error).message})`; } + if (branchDiffs.length) sbom.branchDiffs = branchDiffs; + } catch (e) { + // Non-fatal; annotate decision + this.decisions[fullName] = (this.decisions[fullName] || "") + ` (branch scan error: ${(e as Error).message})`; } - newSboms.push(res); - if (res.error) this.summary.failedCount++; else this.summary.successCount++; + newSboms.push(sbom); + if (sbom.error) this.summary.failedCount++; else this.summary.successCount++; // Write freshly fetched SBOM immediately if a cache directory is configured if (this.opts.loadFromDir && this.opts.syncSboms && this.opts.loadFromDir.length) { - try { writeOne(res, { outDir: this.opts.loadFromDir }); } catch { /* ignore write errors */ } + try { writeOne(sbom, { outDir: this.opts.loadFromDir }); } catch { /* ignore write errors */ } } } processed++; From 6f43ed073969b6869fe4ed9395098f2163352082 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Fri, 28 Nov 2025 16:32:19 +0000 Subject: [PATCH 009/108] Removed unused function --- src/componentDetection.ts | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 39e0b51..c640102 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -58,22 +58,12 @@ export default class ComponentDetection { console.info("Running component-detection"); try { - await spawn(`${this.componentDetectionPath}`, ['scan', '--SourceDirectory', path, '--ManifestFile', this.outputPath, ...this.getComponentDetectionParameters()], { stdio: 'pipe' }); + await spawn(`${this.componentDetectionPath}`, ['scan', '--SourceDirectory', path, '--ManifestFile', this.outputPath], { stdio: 'pipe' }); } catch (error: any) { console.error(error); } } - private static getComponentDetectionParameters(): Array { - var parameters: Array = []; - // parameters.push((console.getInput('directoryExclusionList')) ? ` --DirectoryExclusionList ${console.getInput('directoryExclusionList')}` : ""); - // parameters.push((console.getInput('detectorArgs')) ? ` --DetectorArgs ${console.getInput('detectorArgs')}` : ""); - // parameters.push((console.getInput('detectorsFilter')) ? ` --DetectorsFilter ${console.getInput('detectorsFilter')}` : ""); - // parameters.push((console.getInput('detectorsCategories')) ? ` --DetectorCategories ${console.getInput('detectorsCategories')}` : ""); - // parameters.push((console.getInput('dockerImagesToScan')) ? ` --DockerImagesToScan ${console.getInput('dockerImagesToScan')}` : ""); - return parameters; - } - public static async getManifestsFromResults(): Promise { console.info("Getting manifests from results"); const results = await fs.readFileSync(this.outputPath, 'utf8'); From c002406f587ce27f866a124c4902ae1821bcdafd Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Fri, 28 Nov 2025 18:05:04 +0000 Subject: [PATCH 010/108] Works for branch changes --- src/cli.ts | 1 - src/componentDetection.ts | 5 ++++- src/sbomCollector.ts | 30 ++++++++++++++---------------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 2717cbc..9f8c466 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -114,7 +114,6 @@ async function main() { caBundlePath: argv["ca-bundle"] as string | undefined, includeBranches: argv["branch-scan"] as boolean, branchLimit: argv["branch-limit"] as number, - includeDependencyReviewDiffs: argv["dependency-review"] as boolean, branchDiffBase: argv["diff-base"] as string | undefined, submitOnMissingSnapshot: argv["submit-on-missing-snapshot"] as boolean, submitLanguages: (argv["submit-languages"] as string[] | undefined) || undefined, diff --git a/src/componentDetection.ts b/src/componentDetection.ts index c640102..5d8541f 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -9,11 +9,12 @@ import fs from 'fs' import { spawn } from 'child_process'; //import dotenv from 'dotenv' import path from 'path'; +import { tmpdir } from 'os'; //dotenv.config(); export default class ComponentDetection { public static componentDetectionPath = process.platform === "win32" ? './component-detection.exe' : './component-detection'; - public static outputPath = './output.json'; + public static outputPath = path.join(tmpdir(), `component-detection-output-${Date.now()}.json`); // This is the default entry point for this class. // If executablePath is provided, use it directly and skip download. @@ -66,6 +67,8 @@ export default class ComponentDetection { public static async getManifestsFromResults(): Promise { console.info("Getting manifests from results"); + console.info(`Reading results from ${this.outputPath}`); + console.info(`Stat: ${fs.statSync(this.outputPath)}`); const results = await fs.readFileSync(this.outputPath, 'utf8'); var json: any = JSON.parse(results); let dependencyGraphs: DependencyGraphs = this.normalizeDependencyGraphPaths(json.dependencyGraphs, '.'); diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 3d41819..537c316 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -28,7 +28,6 @@ export interface CollectorOptions { caBundlePath?: string; // path to PEM CA bundle for self-signed/internal certs includeBranches?: boolean; // when true, fetch SBOM for non-default branches branchLimit?: number; // limit number of branches per repo (excluding default) - includeDependencyReviewDiffs?: boolean; // fetch dependency review diff base->branch branchDiffBase?: string; // override base branch for diffs (defaults to default branch) submitOnMissingSnapshot?: boolean; // run component detection submission when diff 404 submitLanguages?: string[]; // limit submission to these languages @@ -69,7 +68,6 @@ export class SbomCollector { caBundlePath: o.caBundlePath ,includeBranches: o.includeBranches ?? false ,branchLimit: o.branchLimit ?? 20 - ,includeDependencyReviewDiffs: o.includeDependencyReviewDiffs ?? true ,branchDiffBase: o.branchDiffBase ,submitOnMissingSnapshot: o.submitOnMissingSnapshot ?? false ,submitLanguages: o.submitLanguages ?? undefined @@ -283,11 +281,10 @@ export class SbomCollector { if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); const base = this.opts.branchDiffBase || sbom?.defaultBranch; if (!base) { console.error(chalk.red(`Cannot compute branch diff for ${fullName} branch ${b.name} because base branch is undefined.`)); continue; } - if (base && this.opts.includeDependencyReviewDiffs) { - if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); - const diff = await this.fetchDependencyReviewDiff(org, repo.name, base, b.name); - branchDiffs.push(diff); - } + if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); + const diff = await this.fetchDependencyReviewDiff(org, repo.name, base, b.name); + console.log(diff); + branchDiffs.push(diff); } if (branchDiffs.length) sbom.branchDiffs = branchDiffs; } catch (e) { @@ -434,26 +431,27 @@ export class SbomCollector { private async fetchDependencyReviewDiff(org: string, repo: string, base: string, head: string): Promise { if (!this.octokit) throw new Error("No Octokit instance"); try { - const resp = await this.octokit.request("GET /repos/{owner}/{repo}/dependency-graph/dependency-review", { owner: org, repo, base, head, headers: { Accept: "application/vnd.github+json" } }); + const basehead = `${base}...${head}`; + const resp = await this.octokit.request("GET /repos/{owner}/{repo}/dependency-graph/compare/{basehead}", { owner: org, repo, basehead, headers: { Accept: "application/vnd.github+json" } }); // Response shape includes change_set array (per docs). We normalize to DependencyReviewPackageChange[] - const raw = resp.data as { change_set?: unknown[] }; + const raw = resp.data; + const changes: DependencyReviewPackageChange[] = []; - for (const c of raw.change_set ?? []) { + for (const c of raw) { const obj = c as Record; const change: DependencyReviewPackageChange = { - changeType: String(obj.change_type || obj.changeType || "unknown"), + changeType: String(obj.change_type || "unknown"), name: obj.name as string | undefined, ecosystem: obj.ecosystem as string | undefined, packageURL: obj.package_url as string | undefined, - purl: obj.purl as string | undefined, license: obj.license as string | undefined, manifest: obj.manifest as string | undefined, scope: obj.scope as string | undefined, - previousVersion: obj.previous_version as string | undefined, - newVersion: obj.version as string | undefined + version: obj.version as string | undefined }; changes.push(change); } + console.log(`Parsed dependency review diff for ${org}/${repo} ${base}...${head}: ${JSON.stringify(changes)}`); return { base, head, retrievedAt: new Date().toISOString(), changes }; } catch (e) { const status = (e as { status?: number })?.status; @@ -609,8 +607,8 @@ export class SbomCollector { if (repoSbom.branchDiffs) { for (const diff of repoSbom.branchDiffs) { for (const change of diff.changes) { - if (change.changeType !== "added" && change.changeType !== "updated") continue; - const p = (change.purl || change.packageURL || (change.ecosystem ? `pkg:${change.ecosystem}/${change.name || ""}${change.newVersion ? "@" + change.newVersion : ""}` : undefined)); + if (change.changeType !== "added") continue; + const p = change.packageURL; if (!p) continue; const pLower = p.toLowerCase(); for (const q of queries) { From 4787c2e52c490f1ff9b5ae49977441fdcf8ff059 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 1 Dec 2025 10:12:11 +0000 Subject: [PATCH 011/108] Formatting --- src/cli.ts | 12 ++++++------ src/componentDetection.ts | 38 ++++++++++++++++++++------------------ src/ignore.ts | 2 +- src/sbomCollector.ts | 16 ++++++++-------- 4 files changed, 35 insertions(+), 33 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 9f8c466..2ad7bbd 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -117,10 +117,10 @@ async function main() { branchDiffBase: argv["diff-base"] as string | undefined, submitOnMissingSnapshot: argv["submit-on-missing-snapshot"] as boolean, submitLanguages: (argv["submit-languages"] as string[] | undefined) || undefined, - // Pass through as part of options bag used by submission helper via collector - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - componentDetectionBinPath: argv["component-detection-bin"] as string | undefined, + // Pass through as part of options bag used by submission helper via collector + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + componentDetectionBinPath: argv["component-detection-bin"] as string | undefined, }); if (!quiet) process.stderr.write(chalk.cyan(offline ? "Loading SBOMs from cache..." : "Collecting SBOMs from cache & GitHub...") + "\n"); @@ -407,8 +407,8 @@ async function main() { { name: "purl", message: "Enter a PURL (blank to exit)", type: "input" } ]); if (!ans.purl) break; - const map = collector.searchByPurlsWithReasons([ans.purl.startsWith("pkg:") ? ans.purl : `pkg:${ans.purl}`]); - runSearchCli([ans.purl], map); + const map = collector.searchByPurlsWithReasons([ans.purl.startsWith("pkg:") ? ans.purl : `pkg:${ans.purl}`]); + runSearchCli([ans.purl], map); } } } diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 5d8541f..934abc6 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -241,31 +241,33 @@ export default class ComponentDetection { if (ghesMode) { githubToken = ""; } - const octokit = new Octokit({ auth: githubToken, baseUrl: githubAPIURL, request: { fetch: fetch}, log: { - debug: console.debug, - info: console.info, - warn: console.warn, - error: console.error - }, }); + const octokit = new Octokit({ + auth: githubToken, baseUrl: githubAPIURL, request: { fetch: fetch }, log: { + debug: console.debug, + info: console.info, + warn: console.warn, + error: console.error + }, + }); const owner = "microsoft"; const repo = "component-detection"; console.debug("Attempting to download latest release from " + githubAPIURL); try { - const latestRelease = await octokit.request("GET /repos/{owner}/{repo}/releases/latest", {owner, repo}); - - var downloadURL: string = ""; - // TODO: do we need to handle different architectures here? - // can we allow x64 on MacOS? We could allow an input parameter to override? - const assetName = process.platform === "win32" ? "component-detection-win-x64.exe" : process.platform === "linux" ? "component-detection-linux-x64" : "component-detection-osx-arm64"; - latestRelease.data.assets.forEach((asset: any) => { - if (asset.name === assetName) { - downloadURL = asset.browser_download_url; - } - }); + const latestRelease = await octokit.request("GET /repos/{owner}/{repo}/releases/latest", { owner, repo }); + + var downloadURL: string = ""; + // TODO: do we need to handle different architectures here? + // can we allow x64 on MacOS? We could allow an input parameter to override? + const assetName = process.platform === "win32" ? "component-detection-win-x64.exe" : process.platform === "linux" ? "component-detection-linux-x64" : "component-detection-osx-arm64"; + latestRelease.data.assets.forEach((asset: any) => { + if (asset.name === assetName) { + downloadURL = asset.browser_download_url; + } + }); - return downloadURL; + return downloadURL; } catch (error: any) { console.error(error); console.debug(error.message); diff --git a/src/ignore.ts b/src/ignore.ts index e64e330..d56ffba 100644 --- a/src/ignore.ts +++ b/src/ignore.ts @@ -67,7 +67,7 @@ function parsePurlIgnore(raw: string): ParsedPurlIgnore | null { export class IgnoreMatcher { private globalAdvisories: Set = new Set(); private globalPurls: ParsedPurlIgnore[] = []; - private scoped: Array<{ scope: string; isRepo: boolean; advisories: Set; purls: ParsedPurlIgnore[] } > = []; + private scoped: Array<{ scope: string; isRepo: boolean; advisories: Set; purls: ParsedPurlIgnore[] }> = []; static load(filePath: string, opts?: IgnoreMatcherOptions): IgnoreMatcher | undefined { const abs = path.isAbsolute(filePath) ? filePath : path.join(opts?.cwd || process.cwd(), filePath); diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 537c316..3ecdd9e 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -65,13 +65,13 @@ export class SbomCollector { showProgressBar: o.showProgressBar ?? false, suppressSecondaryRateLimitLogs: o.suppressSecondaryRateLimitLogs ?? false, quiet: o.quiet ?? false, - caBundlePath: o.caBundlePath - ,includeBranches: o.includeBranches ?? false - ,branchLimit: o.branchLimit ?? 20 - ,branchDiffBase: o.branchDiffBase - ,submitOnMissingSnapshot: o.submitOnMissingSnapshot ?? false - ,submitLanguages: o.submitLanguages ?? undefined - ,componentDetectionBinPath: o.componentDetectionBinPath + caBundlePath: o.caBundlePath, + includeBranches: o.includeBranches ?? false, + branchLimit: o.branchLimit ?? 20, + branchDiffBase: o.branchDiffBase, + submitOnMissingSnapshot: o.submitOnMissingSnapshot ?? false, + submitLanguages: o.submitLanguages ?? undefined, + componentDetectionBinPath: o.componentDetectionBinPath } as Required; if (this.opts.token) { @@ -249,7 +249,7 @@ export class SbomCollector { this.decisions[fullName] = baseline ? `Fetching because missing pushed_at (${baseline.repoPushedAt} / ${repo.pushed_at})` : "Fetching because no baseline"; } - let sbom : RepositorySbom | undefined = undefined; + let sbom: RepositorySbom | undefined = undefined; if (!skipped) { const res = await this.fetchSbom(org, repo.name, repo); From 290c0701432c3b38b0ac3619d9e02b07200bf91a Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 1 Dec 2025 15:29:46 +0000 Subject: [PATCH 012/108] Updated malware matching to do diffs --- .../test-sbom-repo/sbom.json | 20 ++ src/cli.ts | 21 +- src/malwareMatcher.ts | 52 ++++- src/sbomCollector.ts | 212 ++++++++---------- src/serialization.ts | 30 ++- src/test-branch-search.ts | 30 ++- src/test-fixture-match.ts | 10 +- src/types.ts | 6 +- .../example-org/demo-repo/sbom.json | 18 -- 9 files changed, 219 insertions(+), 180 deletions(-) diff --git a/fixtures/sboms/advanced-security/test-sbom-repo/sbom.json b/fixtures/sboms/advanced-security/test-sbom-repo/sbom.json index 9b4ef08..b621365 100644 --- a/fixtures/sboms/advanced-security/test-sbom-repo/sbom.json +++ b/fixtures/sboms/advanced-security/test-sbom-repo/sbom.json @@ -69,5 +69,25 @@ } ] } + ], + "branchDiffs": [ + { + "latestCommitDate": "2025-12-01T12:39:01.734Z", + "base": "main", + "head": "test", + "retrievedAt": "2025-12-01T12:39:01.734Z", + "changes": [ + { + "changeType": "added", + "name": "chalk", + "ecosystem": "npm", + "packageURL": "pkg:npm/chalk@5.6.1", + "license": "MIT", + "manifest": "package-lock.json", + "scope": "runtime", + "version": "5.6.1" + } + ] + } ] } diff --git a/src/cli.ts b/src/cli.ts index 2ad7bbd..6b8aa10 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -47,6 +47,7 @@ async function main() { .option("submit-on-missing-snapshot", { type: "boolean", default: false, describe: "When dependency review diff returns 404 (missing snapshot), run Component Detection to submit a snapshot, then retry." }) .option("submit-languages", { type: "array", describe: "Limit snapshot submission to these languages (e.g., JavaScript,TypeScript,Python,Maven)." }) .option("component-detection-bin", { type: "string", describe: "Path to a local component-detection executable to use for snapshot submission (skips download)." }) + .option("debug", { type: "boolean", default: false, describe: "Enable debug logging" }) .check(args => { const syncing = !!args.syncSboms; if (syncing) { @@ -81,6 +82,14 @@ async function main() { .help() .parseAsync(); + const debug = argv.debug as boolean; + + if (debug) { + console.debug(chalk.blue("Debug logging enabled")); + } else { + console.debug = () => {}; + } + const token = argv.token as string | undefined || process.env.GITHUB_TOKEN; // Require a token for any network operation (syncing SBOMs, malware advisories, or SARIF upload) @@ -185,7 +194,8 @@ async function main() { const showMalwareCli = (!wantJson && !wantCsv) || wantCli; // show only in pure CLI or combined mode if (showMalwareCli && !quiet) { for (const m of malwareMatches) { - process.stdout.write(`${m.repo} :: ${m.purl} => ${m.advisoryGhsaId} (${m.vulnerableVersionRange ?? "(no range)"}) {advisory: ${m.reason}} ${m.advisoryPermalink}\n`); + const branchInfo = m.branch ? ` [branch: ${m.branch}]` : ""; + process.stdout.write(`${m.repo} :: ${m.purl} => ${m.advisoryGhsaId} (${m.vulnerableVersionRange ?? "(no range)"}){advisory: ${m.reason}}${branchInfo} ${m.advisoryPermalink}\n`); } } if (argv.sarifDir) { @@ -275,14 +285,14 @@ async function main() { for (const { purl, reason } of entries) searchRows.push({ repo, purl, reason }); } } - const malwareRows: Array<{ repo: string; purl: string; advisory: string; range: string | null; updatedAt: string }> = []; + const malwareRows: Array<{ repo: string; purl: string; advisory: string; range: string | null; updatedAt: string; branch: string | undefined }> = []; if (malwareMatches) { for (const m of malwareMatches) { - malwareRows.push({ repo: m.repo, purl: m.purl, advisory: m.advisoryGhsaId, range: m.vulnerableVersionRange, updatedAt: m.advisoryUpdatedAt }); + malwareRows.push({ repo: m.repo, purl: m.purl, advisory: m.advisoryGhsaId, range: m.vulnerableVersionRange, updatedAt: m.advisoryUpdatedAt, branch: m.branch }); } } // CSV columns: type,repo,purl,reason_or_advisory,range,updatedAt - const header = ["type", "repo", "purl", "reason_or_advisory", "range", "updatedAt"]; + const header = ["type", "repo", "purl", "reason_or_advisory", "range", "updatedAt", "branch"]; const sanitize = (val: unknown): string => { if (val === null || val === undefined) return ""; let s = String(val); @@ -310,7 +320,8 @@ async function main() { sanitize(r.purl), sanitize(r.advisory), sanitize(r.range ?? ""), - sanitize(r.updatedAt) + sanitize(r.updatedAt), + sanitize(r.branch ?? "") ].join(",")); } const csvPayload = lines.join("\n") + "\n"; diff --git a/src/malwareMatcher.ts b/src/malwareMatcher.ts index 15f7abd..95cf3c6 100644 --- a/src/malwareMatcher.ts +++ b/src/malwareMatcher.ts @@ -15,6 +15,7 @@ export interface MalwareMatch { packageName: string; ecosystem: string; version: string | null; + branch?: string; // default branch or named head branch from diff advisoryGhsaId: string; advisoryPermalink: string; vulnerableVersionRange: string | null; @@ -130,22 +131,50 @@ export function matchMalware(advisories: MalwareAdvisoryNode[], sboms: Repositor } } - // Helper to enumerate packages with fallback to raw SPDX packages inside repoSbom.sbom if flattened list empty - const enumeratePackages = (repo: RepositorySbom): Array => { + // Enumerate base packages, falling back to raw SPDX if flattened list empty + const enumerateBasePackages = (repo: RepositorySbom): Array => { const explicit: SbomPackage[] = Array.isArray(repo.packages) ? repo.packages : []; - if (explicit.length > 0) return explicit as Array; - const rawMaybe: unknown = repo.sbom?.packages; - if (Array.isArray(rawMaybe)) { - return rawMaybe as Array; + let list: Array = []; + if (explicit.length > 0) { + list = explicit as Array; + } else { + const rawMaybe: unknown = repo.sbom?.packages; + if (Array.isArray(rawMaybe)) { + list = rawMaybe as Array; + } + } + // Annotate with default branch for reporting (if known) + return list.map(p => ({ ...p, __branch: repo.defaultBranch || undefined })); + }; + + // Enumerate packages implied by branch diffs (added/updated head-side versions) + const enumerateDiffPackages = (repo: RepositorySbom): Array<{ purl: string; name?: string; ecosystem?: string; version?: string; __branch: string }> => { + const out: Array<{ purl: string; name?: string; ecosystem?: string; version?: string; __branch: string }> = []; + if (!(repo.branchDiffs instanceof Map)) return out; + for (const diff of repo.branchDiffs.values()) { + const branchName = diff.head; + for (const change of diff.changes) { + if (change.changeType !== 'added' && change.changeType !== 'updated') continue; + let p: string | undefined = (change as { purl?: string }).purl; + if (!p && change.packageURL && change.packageURL.startsWith('pkg:')) p = change.packageURL; + if (!p && change.ecosystem && change.name && change.newVersion) { + // Dependency review ecosystems are lower-case purl types already (e.g. npm, maven, pip, gem) + p = `pkg:${change.ecosystem}/${change.name}${change.newVersion ? '@' + change.newVersion : ''}`; + } + if (!p) continue; + out.push({ purl: p, name: change.name, ecosystem: change.ecosystem, version: change.newVersion, __branch: branchName }); + } } - return []; + return out; }; for (const repoSbom of sboms) { - const pkgs = enumeratePackages(repoSbom); - if (!pkgs.length) continue; + const basePkgs = enumerateBasePackages(repoSbom); + const diffPkgs = enumerateDiffPackages(repoSbom); + const combined = [...basePkgs, ...diffPkgs]; + if (!combined.length) continue; - for (const pkg of pkgs) { + for (const pkg of combined) { const pkgAny = pkg as unknown as { purl?: string; externalRefs?: Array<{ referenceType?: string; referenceLocator?: string }>; name?: string; version?: string; versionInfo?: string }; const candidatePurls = new Set(); if (pkgAny.purl) candidatePurls.add(pkgAny.purl); @@ -154,6 +183,8 @@ export function matchMalware(advisories: MalwareAdvisoryNode[], sboms: Repositor if (ref?.referenceType === "purl" && ref.referenceLocator) candidatePurls.add(ref.referenceLocator); } } + // If this is a diff-derived synthetic package, candidate set may be empty except constructed purl + if (candidatePurls.size === 0 && (pkg as { purl?: string }).purl) candidatePurls.add((pkg as { purl?: string }).purl as string); // If no purls found, skip (can't map ecosystem reliably) if (candidatePurls.size === 0) continue; @@ -185,6 +216,7 @@ export function matchMalware(advisories: MalwareAdvisoryNode[], sboms: Repositor packageName: parsed.name, ecosystem, version, + branch: (pkg as any).__branch, advisoryGhsaId: adv.ghsaId, advisoryPermalink: adv.permalink, vulnerableVersionRange: vuln.vulnerableVersionRange, diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 3ecdd9e..d841621 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -267,26 +267,39 @@ export class SbomCollector { } // Branch scanning (optional) - // TODO: implement some check to see if the diff info we have is already fresher than the branch info if (this.opts.includeBranches && sbom?.sbom) { - console.log(chalk.blue(`Scanning branches for ${fullName}...`)); + console.debug(chalk.blue(`Scanning branches for ${fullName}...`)); try { const branches = await this.listBranches(org, repo.name); const nonDefault = branches.filter(b => b.name !== sbom.defaultBranch); const limited = this.opts.branchLimit && this.opts.branchLimit > 0 ? nonDefault.slice(0, this.opts.branchLimit) : nonDefault; - const branchDiffs: BranchDependencyDiff[] = []; + const branchDiffs: Map = new Map(); for (const b of limited) { + + // get the commits, compare to the stored diff info. If the latest commit is newer, then fetch diff, otherwise skip + const latestCommit = await this.getLatestCommit(org, repo.name, b.name); + if (!latestCommit) { + console.error(chalk.red(`Failed to get latest commit for ${fullName} branch ${b.name}.`)); + continue; + } + const existing = sbom?.branchDiffs instanceof Map ? sbom.branchDiffs.get(b.name) : undefined; + if (await this.isCommitNewer(latestCommit, existing)) { + console.debug(chalk.green(`Fetching branch diff for ${fullName} branch ${b.name}...`)); + } else { + console.debug(chalk.yellow(`Skipping branch diff for ${fullName} branch ${b.name} (no new commits).`)); + continue; + } + if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); const base = this.opts.branchDiffBase || sbom?.defaultBranch; if (!base) { console.error(chalk.red(`Cannot compute branch diff for ${fullName} branch ${b.name} because base branch is undefined.`)); continue; } if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); const diff = await this.fetchDependencyReviewDiff(org, repo.name, base, b.name); - console.log(diff); - branchDiffs.push(diff); + branchDiffs.set(b.name, diff); } - if (branchDiffs.length) sbom.branchDiffs = branchDiffs; + if (branchDiffs.size) sbom.branchDiffs = branchDiffs; } catch (e) { // Non-fatal; annotate decision this.decisions[fullName] = (this.decisions[fullName] || "") + ` (branch scan error: ${(e as Error).message})`; @@ -311,11 +324,32 @@ export class SbomCollector { return this.sboms; } - private async listEnterpriseOrgs(enterprise: string): Promise { - // GitHub API: GET /enterprises/{enterprise}/orgs (preview might require accept header) - + private async getLatestCommit(org: string, repo: string, branch: string): Promise<{ sha?: string; commitDate?: string } | null> { if (!this.octokit) throw new Error("No Octokit instance"); + try { + const resp = await this.octokit.request("GET /repos/{owner}/{repo}/commits", { owner: org, repo, sha: branch }); + const commitSha = resp.data?.[0]?.sha; + const commitDate = resp.data?.[0]?.commit?.author?.date; + return { sha: commitSha, commitDate }; + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + console.error(`Failed to get latest commit for ${org}/${repo} branch ${branch}: ${msg}`); + return null; + } + } + private async isCommitNewer(latestCommit: { sha?: string; commitDate?: string }, existingDiff?: BranchDependencyDiff): Promise { + if (!existingDiff || !existingDiff.latestCommitDate) { + return true; + } + if (latestCommit.commitDate && existingDiff.latestCommitDate) { + return new Date(latestCommit.commitDate) > new Date(existingDiff.latestCommitDate); + } + return false; + } + + private async listEnterpriseOrgs(enterprise: string): Promise { + if (!this.octokit) throw new Error("No Octokit instance"); interface Org { login: string } try { const orgs: string[] = []; @@ -338,7 +372,6 @@ export class SbomCollector { private async listOrgRepos(org: string): Promise<{ name: string; pushed_at?: string; updated_at?: string; default_branch?: string }[]> { if (!this.octokit) throw new Error("No Octokit instance"); - // GET /orgs/{org}/repos interface RepoMeta { name: string; pushed_at?: string; updated_at?: string; default_branch?: string } const repos: RepoMeta[] = []; const per_page = 100; @@ -451,8 +484,8 @@ export class SbomCollector { }; changes.push(change); } - console.log(`Parsed dependency review diff for ${org}/${repo} ${base}...${head}: ${JSON.stringify(changes)}`); - return { base, head, retrievedAt: new Date().toISOString(), changes }; + console.debug(`Parsed dependency review diff for ${org}/${repo} ${base}...${head}: ${JSON.stringify(changes)}`); + return { latestCommitDate: new Date().toISOString(), base, head, retrievedAt: new Date().toISOString(), changes }; } catch (e) { const status = (e as { status?: number })?.status; let reason = e instanceof Error ? e.message : String(e); @@ -474,7 +507,7 @@ export class SbomCollector { } } } - return { base, head, retrievedAt: new Date().toISOString(), changes: [], error: reason }; + return { latestCommitDate: new Date().toISOString(), base, head, retrievedAt: new Date().toISOString(), changes: [], error: reason }; } } @@ -517,6 +550,42 @@ export class SbomCollector { const queries: ParsedQuery[] = purls.map(parseQuery).filter((q): q is ParsedQuery => !!q); const results = new Map(); if (!queries.length) return results; + const applyQueries = (candidatePurls: string[], queries: ParsedQuery[], found: Map, branchTag?: string, fallbackVersion?: string) => { + const unique = Array.from(new Set(candidatePurls)); + for (const p of unique) { + const pLower = p.toLowerCase(); + const outKey = branchTag ? `${p}@${branchTag}` : p; + for (const q of queries) { + if (q.isPrefixWildcard) { + const prefix = q.lower.slice(0, -1); + if (pLower.startsWith(prefix)) { if (!found.has(outKey)) found.set(outKey, q.raw); } + continue; + } + if (q.versionConstraint && q.type && q.name) { + if (!pLower.startsWith("pkg:")) continue; + const body = p.slice(4); + const atIdx = body.indexOf("@"); + const main = atIdx >= 0 ? body.slice(0, atIdx) : body; + const ver = atIdx >= 0 ? body.slice(atIdx + 1) : fallbackVersion; + const slashIdx = main.indexOf("/"); + if (slashIdx < 0) continue; + const pType = main.slice(0, slashIdx).toLowerCase(); + const pName = main.slice(slashIdx + 1); + if (pType === q.type && pName.toLowerCase() === q.name.toLowerCase() && ver) { + try { + const coerced = semver.coerce(ver)?.version || ver; + if (semver.valid(coerced) && semver.satisfies(coerced, q.versionConstraint, { includePrerelease: true })) { + if (!found.has(outKey)) found.set(outKey, q.raw); + } + } catch { /* ignore */ } + } + } else if (q.exact) { + if (pLower === q.exact) { if (!found.has(outKey)) found.set(outKey, q.raw); } + } + } + } + }; + for (const repoSbom of this.sboms) { if (repoSbom.error) continue; interface ExtRef { referenceType: string; referenceLocator: string } @@ -526,119 +595,18 @@ export class SbomCollector { const candidatePurls: string[] = []; if (refs) for (const r of refs) if (r.referenceType === "purl" && r.referenceLocator) candidatePurls.push(r.referenceLocator); if ((pkg as { purl?: string }).purl) candidatePurls.push((pkg as { purl?: string }).purl as string); - const unique = Array.from(new Set(candidatePurls)); - for (const p of unique) { - const pLower = p.toLowerCase(); - for (const q of queries) { - if (q.isPrefixWildcard) { - const prefix = q.lower.slice(0, -1); - if (pLower.startsWith(prefix)) { if (!found.has(p)) found.set(p, q.raw); } - continue; - } - if (q.versionConstraint && q.type && q.name) { - if (!pLower.startsWith("pkg:")) continue; - const body = p.slice(4); - const atIdx = body.indexOf("@"); - const main = atIdx >= 0 ? body.slice(0, atIdx) : body; - const ver = atIdx >= 0 ? body.slice(atIdx + 1) : (pkg.version as string | undefined) || undefined; - const slashIdx = main.indexOf("/"); - if (slashIdx < 0) continue; - const pType = main.slice(0, slashIdx).toLowerCase(); - const pName = main.slice(slashIdx + 1); - if (pType === q.type && pName.toLowerCase() === q.name.toLowerCase() && ver) { - try { - const coerced = semver.coerce(ver)?.version || ver; - if (semver.valid(coerced) && semver.satisfies(coerced, q.versionConstraint, { includePrerelease: true })) { - if (!found.has(p)) found.set(p, q.raw); - } - } catch { /* ignore */ } - } - } else if (q.exact) { - if (pLower === q.exact) { if (!found.has(p)) found.set(p, q.raw); } - } - } - } - } - // Include branch SBOM packages - if (repoSbom.branchSboms) { - for (const b of repoSbom.branchSboms) { - if (b.error) continue; - for (const pkg of b.packages as Array) { - const refs = (pkg as { externalRefs?: ExtRef[] }).externalRefs; - const candidatePurls: string[] = []; - if (refs) for (const r of refs) if (r.referenceType === "purl" && r.referenceLocator) candidatePurls.push(r.referenceLocator); - if ((pkg as { purl?: string }).purl) candidatePurls.push((pkg as { purl?: string }).purl as string); - const unique = Array.from(new Set(candidatePurls)); - for (const p of unique) { - const pLower = p.toLowerCase(); - for (const q of queries) { - if (q.isPrefixWildcard) { - const prefix = q.lower.slice(0, -1); - if (pLower.startsWith(prefix)) { if (!found.has(`${p}@${b.branch}`)) found.set(`${p}@${b.branch}`, q.raw); } - continue; - } - if (q.versionConstraint && q.type && q.name) { - if (!pLower.startsWith("pkg:")) continue; - const body = p.slice(4); - const atIdx = body.indexOf("@"); - const main = atIdx >= 0 ? body.slice(0, atIdx) : body; - const ver = atIdx >= 0 ? body.slice(atIdx + 1) : (pkg.version as string | undefined) || undefined; - const slashIdx = main.indexOf("/"); - if (slashIdx < 0) continue; - const pType = main.slice(0, slashIdx).toLowerCase(); - const pName = main.slice(slashIdx + 1); - if (pType === q.type && pName.toLowerCase() === q.name.toLowerCase() && ver) { - try { - const coerced = semver.coerce(ver)?.version || ver; - if (semver.valid(coerced) && semver.satisfies(coerced, q.versionConstraint, { includePrerelease: true })) { - if (!found.has(`${p}@${b.branch}`)) found.set(`${p}@${b.branch}`, q.raw); - } - } catch { /* ignore */ } - } - } else if (q.exact) { - if (pLower === q.exact) { if (!found.has(`${p}@${b.branch}`)) found.set(`${p}@${b.branch}`, q.raw); } - } - } - } - } - } + applyQueries(candidatePurls, queries, found, undefined, (pkg.version as string | undefined) || undefined); } // Include dependency review diff additions/updates (head packages only) if (repoSbom.branchDiffs) { - for (const diff of repoSbom.branchDiffs) { + const diffs = repoSbom.branchDiffs.values(); + for (const diff of diffs) { for (const change of diff.changes) { - if (change.changeType !== "added") continue; - const p = change.packageURL; - if (!p) continue; - const pLower = p.toLowerCase(); - for (const q of queries) { - if (q.isPrefixWildcard) { - const prefix = q.lower.slice(0, -1); - if (pLower.startsWith(prefix)) { if (!found.has(`${p}@${diff.head}`)) found.set(`${p}@${diff.head}`, q.raw); } - continue; - } - if (q.versionConstraint && q.type && q.name) { - if (!pLower.startsWith("pkg:")) continue; - const body = p.slice(4); - const atIdx = body.indexOf("@"); - const main = atIdx >= 0 ? body.slice(0, atIdx) : body; - const ver = atIdx >= 0 ? body.slice(atIdx + 1) : change.newVersion; - const slashIdx = main.indexOf("/"); - if (slashIdx < 0) continue; - const pType = main.slice(0, slashIdx).toLowerCase(); - const pName = main.slice(slashIdx + 1); - if (pType === q.type && pName.toLowerCase() === q.name.toLowerCase() && ver) { - try { - const coerced = semver.coerce(ver)?.version || ver; - if (semver.valid(coerced) && semver.satisfies(coerced, q.versionConstraint, { includePrerelease: true })) { - if (!found.has(`${p}@${diff.head}`)) found.set(`${p}@${diff.head}`, q.raw); - } - } catch { /* ignore */ } - } - } else if (q.exact) { - if (pLower === q.exact) { if (!found.has(`${p}@${diff.head}`)) found.set(`${p}@${diff.head}`, q.raw); } - } - } + if (change.changeType !== "added" && change.changeType !== "updated") continue; + const candidatePurls: string[] = []; + if ((change as { purl?: string }).purl) candidatePurls.push((change as { purl?: string }).purl as string); + if (change.packageURL) candidatePurls.push(change.packageURL); + applyQueries(candidatePurls, queries, found, diff.head, (change as any).newVersion); } } } diff --git a/src/serialization.ts b/src/serialization.ts index 385f076..c68b4cf 100644 --- a/src/serialization.ts +++ b/src/serialization.ts @@ -13,7 +13,7 @@ export function writeAll(sboms: RepositorySbom[], { outDir, flatten = false }: S const fileDir = path.join(outDir, repoPath); const filePath = flatten ? path.join(outDir, `${repoPath}.json`) : path.join(fileDir, "sbom.json"); fs.mkdirSync(flatten ? path.dirname(filePath) : fileDir, { recursive: true }); - fs.writeFileSync(filePath, JSON.stringify(s, null, 2), "utf8"); + fs.writeFileSync(filePath, JSON.stringify(prepareForWrite(s), null, 2), "utf8"); } } @@ -22,7 +22,7 @@ export function writeOne(sbom: RepositorySbom, { outDir, flatten = false }: Seri const fileDir = path.join(outDir, repoPath); const filePath = flatten ? path.join(outDir, `${repoPath}.json`) : path.join(fileDir, "sbom.json"); fs.mkdirSync(flatten ? path.dirname(filePath) : fileDir, { recursive: true }); - fs.writeFileSync(filePath, JSON.stringify(sbom, null, 2), "utf8"); + fs.writeFileSync(filePath, JSON.stringify(prepareForWrite(sbom), null, 2), "utf8"); } export interface ReadOptions { @@ -48,7 +48,7 @@ export function readAll(dir: string, opts: ReadOptions = {}): RepositorySbom[] { const pushIfValid = (filePath: string) => { try { const raw = fs.readFileSync(filePath, "utf8"); - const obj = JSON.parse(raw); + const obj = reviveAfterRead(JSON.parse(raw)); if (obj && obj.repo && Array.isArray(obj.packages)) { results.push(obj as RepositorySbom); } @@ -91,3 +91,27 @@ export function readAll(dir: string, opts: ReadOptions = {}): RepositorySbom[] { } return results; } + +// Convert Maps to plain serializable structures before JSON.stringify +function prepareForWrite(sbom: RepositorySbom): unknown { + const clone: any = { ...sbom }; + if (clone.branchDiffs instanceof Map) { + // store as array of diff objects for backward compatibility + clone.branchDiffs = Array.from(clone.branchDiffs.values()); + } + return clone; +} + +// Convert array representations back into Maps after JSON.parse +function reviveAfterRead(obj: any): any { + if (obj && obj.branchDiffs && Array.isArray(obj.branchDiffs)) { + const map = new Map(); + for (const diff of obj.branchDiffs) { + if (diff && typeof diff.head === 'string') { + map.set(diff.head, diff); + } + } + obj.branchDiffs = map; + } + return obj; +} diff --git a/src/test-branch-search.ts b/src/test-branch-search.ts index 446a738..19ffd06 100644 --- a/src/test-branch-search.ts +++ b/src/test-branch-search.ts @@ -33,22 +33,20 @@ async function main() { org, retrievedAt: new Date().toISOString(), packages: basePackages, - branchSboms: [ - { - branch: 'feature-x', - retrievedAt: new Date().toISOString(), - packages: featurePackages - } - ], - branchDiffs: [ - { - base: 'main', - head: 'feature-x', - retrievedAt: new Date().toISOString(), - changes: diffChanges - } - ] - }; + // Use Map keyed by branch name per updated type + branchDiffs: new Map([ + [ + 'feature-x', + { + latestCommitDate: new Date().toISOString(), + base: 'main', + head: 'feature-x', + retrievedAt: new Date().toISOString(), + changes: diffChanges + } + ] + ]) + } as RepositorySbom; fs.writeFileSync(path.join(repoDir, 'sbom.json'), JSON.stringify(synthetic, null, 2), 'utf8'); diff --git a/src/test-fixture-match.ts b/src/test-fixture-match.ts index d949e9b..f43b58d 100644 --- a/src/test-fixture-match.ts +++ b/src/test-fixture-match.ts @@ -6,17 +6,21 @@ import path from "path"; // Load SBOM fixture const sboms = readAll(path.join(process.cwd(), "fixtures/sboms")); + // Load malware advisory fixture const cachePath = path.join(process.cwd(), "fixtures/malware-cache/malware-advisories.json"); const cache = JSON.parse(fs.readFileSync(cachePath, "utf8")); const advisories: MalwareAdvisoryNode[] = cache.advisories; const matches = matchMalware(advisories, sboms); + process.stdout.write("Matches:\n"); + for (const m of matches) { - process.stdout.write(`${m.repo} => ${m.purl} matched advisory ${m.advisoryGhsaId} range ${m.vulnerableVersionRange}\n`); + process.stdout.write(`${m.repo} => ${m.purl} matched advisory ${m.advisoryGhsaId} range ${m.vulnerableVersionRange}${m.branch ? ` on branch ${m.branch}` : ""}\n`); } -if (!matches.length) { - console.error("No matches found - expected chalk 5.6.1"); + +if (matches.length !== 2) { + console.error("Did not find 2 matches - expected chalk 5.6.1 on default and test branches"); process.exit(1); } diff --git a/src/types.ts b/src/types.ts index 16ffcdf..cdb5a3a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -77,9 +77,8 @@ export interface RepositorySbom { etag?: string; // ETag from SBOM response (future: conditional requests) defaultBranchCommitSha?: string; // commit SHA of default branch at time of retrieval defaultBranchCommitDate?: string; // ISO date of that commit - // Branch-level SBOMs & diffs (optional when branch scanning enabled) - branchSboms?: BranchSbom[]; - branchDiffs?: BranchDependencyDiff[]; + // Branch-level diffs (optional when branch scanning enabled) + branchDiffs?: Map; } export interface CollectionSummary { @@ -124,6 +123,7 @@ export interface DependencyReviewPackageChange { } export interface BranchDependencyDiff { + latestCommitDate: any; base: string; // base branch head: string; // head branch retrievedAt: string; diff --git a/tmp-branch-search-cache/example-org/demo-repo/sbom.json b/tmp-branch-search-cache/example-org/demo-repo/sbom.json index 9dcf7cb..98fedf1 100644 --- a/tmp-branch-search-cache/example-org/demo-repo/sbom.json +++ b/tmp-branch-search-cache/example-org/demo-repo/sbom.json @@ -14,24 +14,6 @@ "purl": "pkg:npm/react@18.2.0" } ], - "branchSboms": [ - { - "branch": "feature-x", - "retrievedAt": "2025-11-26T15:02:48.755Z", - "packages": [ - { - "name": "react", - "version": "18.3.0-beta", - "purl": "pkg:npm/react@18.3.0-beta" - }, - { - "name": "lodash", - "version": "4.17.21", - "purl": "pkg:npm/lodash@4.17.21" - } - ] - } - ], "branchDiffs": [ { "base": "main", From e55933e577cc82b191fd75f24c9b3e954b174b15 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 1 Dec 2025 18:28:36 +0000 Subject: [PATCH 013/108] Updated README --- README.md | 61 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 9303afd..a0db616 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ Enable automatic submission + retry with: This requires the action repository to be present as a git submodule (or copied) at the path: -``` +```bash component-detection-dependency-submission-action/ ``` @@ -382,31 +382,42 @@ Then type one PURL query per line. Entering a blank line or using Ctrl+C on a bl | Arg | Purpose | |------|---------| -| `--sbom-cache ` | Directory holding per-repo SBOM JSON files (required for offline mode; used as write target when syncing) | -| `--sync-sboms` | Perform API calls to (re)collect SBOMs; without it the CLI runs offline loading cached SBOMs. Requires a GitHub token | -| `--enterprise ` / `--org ` | Scope selection (mutually exclusive when syncing) | -| `--purl ` | Add a PURL/range/wildcard query (repeatable) | -| `--purl-file ` | File with one query per line | -| `--json` | Emit search JSON to stdout (unless overridden by `--output-file`) | -| `--cli` | Also emit human-readable output when producing JSON (requires `--output-file`) | -| `--output-file ` | Write search JSON payload to file; required when using both `--json` and `--cli` | -| `--interactive` | Enter interactive search prompt after initial processing | -| `--sync-malware` | Fetch & cache malware advisories (MALWARE classification). Requires a GitHub token | -| `--match-malware` | Match current SBOM set against cached advisories | -| `--malware-cache ` | Advisory cache directory (required with malware operations) | -| `--malware-cutoff ` | Ignore advisories whose publishedAt AND updatedAt are both before this date/time (e.g. `2025-09-29` or full timestamp) | -| `--ignore-file ` | YAML ignore file (advisories / purls / scoped blocks) to filter malware matches before output | -| `--ignore-unbounded-malware` | Ignore matches whose advisory vulnerable version range covers all versions (e.g. `*`, `>=0`, `0.0.0`) | -| `--sarif-dir ` | Write SARIF 2.1.0 files per repository (with malware matches) | -| `--upload-sarif` | Upload generated SARIF to Code Scanning (requires --match-malware & --sarif-dir and a GitHub token) | +| `--token ` | GitHub token; required for `--sync-sboms`, `--sync-malware`, and `--upload-sarif` (or use `GITHUB_TOKEN`) | +| `--enterprise ` | Collect across all orgs in an Enterprise (mutually exclusive with `--org`/`--repo` when syncing) | +| `--org ` | Single organization scope (mutually exclusive with `--enterprise`/`--repo` when syncing) | +| `--repo ` | Single repository scope (mutually exclusive with `--enterprise`/`--org` when syncing) | +| `--base-url ` | GitHub Enterprise Server REST base URL (e.g. `https://ghe.example.com/api/v3`) | | `--concurrency ` | Parallel SBOM fetches (default 5) | -| `--sbom-delay ` | Delay between SBOM fetch (dependency-graph/sbom) requests (default 5000) | -| `--light-delay ` | Delay between lightweight metadata calls (listing repos, commit head checks) (default 500) | -| `--base-url ` | GitHub Enterprise Server REST base URL (ends with /api/v3) | -| `--progress` | Show a dynamic progress bar during SBOM collection | -| `--suppress-secondary-rate-limit-logs` | Hide secondary rate limit warning lines (automatically applied with `--progress`) | -| `--quiet` | Suppress all non-error and non-result output (progress bar, JSON and human readable output still show) | -| `--ca-bundle ` | Path to a PEM file containing one or more additional CA certificates (self‑signed / internal PKI) | +| `--sbom-delay ` | Delay between SBOM fetch requests (default 3000) | +| `--light-delay ` | Delay between lightweight metadata requests (default 100) | +| `--sbom-cache ` | Directory to read/write per‑repo SBOM JSON; required for offline mode | +| `--sync-sboms` | Perform API calls to collect SBOMs; without it the CLI runs offline using `--sbom-cache` | +| `--progress` | Show a progress bar during SBOM collection | +| `--suppress-secondary-rate-limit-logs` | Suppress secondary rate limit warning logs (useful with `--progress`) | +| `--quiet` | Suppress non‑error output (progress bar and machine output still emitted) | +| `--ca-bundle ` | PEM bundle with additional CA certs for REST/GraphQL/SARIF upload | +| `--purl ` | Add a PURL / semver range / wildcard query (repeatable) | +| `--purl-file ` | File with one query per line (supports comments) | +| `--json` | Emit search results as JSON (to stdout unless `--output-file` specified) | +| `--cli` | Also emit human‑readable output when producing JSON/CSV; requires `--output-file` to avoid mixed stdout | +| `--csv` | Emit results (search + malware matches) as CSV (to stdout or `--output-file`) | +| `--output-file ` | Write JSON/CSV output to file; required when using `--cli` with `--json` or `--csv` | +| `--interactive` | Enter interactive PURL search prompt after initial processing | +| `--sync-malware` | Fetch & cache malware advisories (MALWARE); requires a token | +| `--match-malware` | Match SBOM packages against cached malware advisories | +| `--malware-cache ` | Directory to store malware advisory cache (required with malware operations) | +| `--malware-cutoff ` | Exclude advisories whose `publishedAt` and `updatedAt` are both before cutoff | +| `--ignore-file ` | YAML ignore file (advisories / purls / scoped blocks) to filter matches before output | +| `--ignore-unbounded-malware` | Suppress advisories with effectively unbounded vulnerable ranges (e.g. `*`, `>=0`) | +| `--sarif-dir ` | Write SARIF 2.1.0 files per repository (for malware matches) | +| `--upload-sarif` | Upload generated SARIF to Code Scanning (requires `--match-malware` and `--sarif-dir`) | +| `--branch-scan` | Fetch SBOM diffs for non‑default branches (limited by `--branch-limit`) | +| `--branch-limit ` | Limit number of non‑default branches scanned per repository (default 10) | +| `--diff-base ` | Override base branch for dependency review diffs (defaults to repository default branch) | +| `--submit-on-missing-snapshot` | On diff 404, run Component Detection to submit a snapshot, then retry | +| `--submit-languages ` | Limit snapshot submission to specific languages (comma‑separated) | +| `--component-detection-bin ` | Path to local `component-detection` executable (skip download) | +| `--debug` | Enable debug logging | ## Build & test From 3e948953500b05694f48285c49e6d2a0b01d9c2e Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 1 Dec 2025 18:29:06 +0000 Subject: [PATCH 014/108] Removed default branch limit --- src/cli.ts | 7 +++---- src/sbomCollector.ts | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 6b8aa10..0f082dd 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -40,9 +40,8 @@ async function main() { .option("csv", { type: "boolean", describe: "Emit results (search + malware matches) as CSV" }) .option("ignore-file", { type: "string", describe: "Path to YAML ignore file (advisories, purls, scoped ignores)" }) .option("ignore-unbounded-malware", { type: "boolean", default: false, describe: "Ignore malware advisories whose vulnerable range covers all versions (e.g. '*', '>=0')" }) - .option("branch-scan", { type: "boolean", default: false, describe: "Fetch SBOMs for non-default branches (limited by --branch-limit)" }) - .option("branch-limit", { type: "number", default: 10, describe: "Limit number of non-default branches to scan per repository" }) - .option("dependency-review", { type: "boolean", default: true, describe: "Fetch dependency review diffs for scanned branches" }) + .option("branch-scan", { type: "boolean", default: false, describe: "Fetch SBOM diffs for non-default branches (limited by --branch-limit)" }) + .option("branch-limit", { type: "number", default: undefined, describe: "Limit number of non-default branches to scan per repository" }) .option("diff-base", { type: "string", describe: "Override base branch for dependency review diffs (defaults to default branch)" }) .option("submit-on-missing-snapshot", { type: "boolean", default: false, describe: "When dependency review diff returns 404 (missing snapshot), run Component Detection to submit a snapshot, then retry." }) .option("submit-languages", { type: "array", describe: "Limit snapshot submission to these languages (e.g., JavaScript,TypeScript,Python,Maven)." }) @@ -122,7 +121,7 @@ async function main() { quiet, caBundlePath: argv["ca-bundle"] as string | undefined, includeBranches: argv["branch-scan"] as boolean, - branchLimit: argv["branch-limit"] as number, + branchLimit: argv["branch-limit"] as number | undefined, branchDiffBase: argv["diff-base"] as string | undefined, submitOnMissingSnapshot: argv["submit-on-missing-snapshot"] as boolean, submitLanguages: (argv["submit-languages"] as string[] | undefined) || undefined, diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index d841621..4ecd61c 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -67,7 +67,7 @@ export class SbomCollector { quiet: o.quiet ?? false, caBundlePath: o.caBundlePath, includeBranches: o.includeBranches ?? false, - branchLimit: o.branchLimit ?? 20, + branchLimit: o.branchLimit, branchDiffBase: o.branchDiffBase, submitOnMissingSnapshot: o.submitOnMissingSnapshot ?? false, submitLanguages: o.submitLanguages ?? undefined, From 5364da0cd04cb6ae9cde425ddabd5c4bb784813f Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 1 Dec 2025 21:45:00 +0000 Subject: [PATCH 015/108] Update to better handling rate limiting when querying malware database --- src/cli.ts | 26 +++--- src/malwareAdvisories.ts | 165 ++++++++++++++++++++++++--------------- 2 files changed, 118 insertions(+), 73 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 0f082dd..7cbcc85 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -5,6 +5,7 @@ import chalk from "chalk"; import { SbomCollector } from "./sbomCollector.js"; import inquirer from "inquirer"; // still used elsewhere if needed import readline from "readline"; +import { CollectionSummary, RepositorySbom } from "./types.js"; const { MalwareAdvisorySync } = await import("./malwareAdvisories.js"); async function main() { @@ -54,7 +55,7 @@ async function main() { if (args.enterprise && args.org) throw new Error("Specify only one of --enterprise or --org"); if (args.repo && (args.enterprise || args.org)) throw new Error("Specify only one of --enterprise, --org, or --repo"); } else { - if (!args.sbomCache) throw new Error("Offline mode requires --sbom-cache (omit --sync-sboms)"); + if (!args.sbomCache && !args.malwareCache) throw new Error("Offline mode requires --sbom-cache (omit --sync-sboms)"); } // If --cli is specified in combination with JSON or CSV, require an output file to avoid mixed stdout streams. if (args.cli && !args.outputFile && (args.json || args.csv)) { @@ -105,6 +106,10 @@ async function main() { const wantCsv = !!argv.csv; const hasOutputFile = !!argv.outputFile; const wantCli = !!argv.cli && hasOutputFile; // only allow CLI alongside machine output when writing file + + let sboms: RepositorySbom[] = []; + let summary: CollectionSummary; + const collector = new SbomCollector({ token: token, enterprise: argv.enterprise as string | undefined, @@ -131,20 +136,23 @@ async function main() { componentDetectionBinPath: argv["component-detection-bin"] as string | undefined, }); - if (!quiet) process.stderr.write(chalk.cyan(offline ? "Loading SBOMs from cache..." : "Collecting SBOMs from cache & GitHub...") + "\n"); - const sboms = await collector.collect(); - const summary = collector.getSummary(); - if (!quiet) process.stderr.write(chalk.green(`Done. Success: ${summary.successCount} / ${summary.repositoryCount}. Failed: ${summary.failedCount}. Cached: ${summary.skippedCount}`) + "\n"); + if (argv.sbomCache || argv.syncSboms) { + if (!quiet) process.stderr.write(chalk.cyan(offline ? "Loading SBOMs from cache..." : "Collecting SBOMs from cache & GitHub...") + "\n"); + sboms = await collector.collect(); + summary = collector.getSummary(); + if (!quiet) process.stderr.write(chalk.green(`Done. Success: ${summary.successCount} / ${summary.repositoryCount}. Failed: ${summary.failedCount}. Cached: ${summary.skippedCount}`) + "\n"); + } const mas = new MalwareAdvisorySync({ token: token!, baseUrl: argv["base-url"] ? (argv["base-url"] as string).replace(/\/v3$/, "/graphql") : undefined, cacheDir: argv["malware-cache"] as string | undefined, since: argv["malware-since"] as string | undefined, - caBundlePath: argv["ca-bundle"] as string | undefined + caBundlePath: argv["ca-bundle"] as string | undefined, + quiet }); - if (argv["sync-malware"]) { + if (argv.syncMalware) { if (!quiet) process.stderr.write(chalk.cyan("Syncing malware advisories from GitHub Advisory Database...") + "\n"); @@ -212,10 +220,6 @@ async function main() { } } } - // Incremental write now handled inside collector; retain legacy behavior only if user wants to force a re-write - if (!quiet && argv.syncSboms && argv["sbom-cache"] && summary.repositoryCount === summary.skippedCount) { - process.stderr.write(chalk.blue("All repositories reused from cache (no new SBOM writes).") + "\n"); - } const runSearchCli = (purls: string[], results: Map) => { if (!results.size) { diff --git a/src/malwareAdvisories.ts b/src/malwareAdvisories.ts index 7e449e7..544b01b 100644 --- a/src/malwareAdvisories.ts +++ b/src/malwareAdvisories.ts @@ -32,6 +32,7 @@ export interface MalwareSyncOptions { since?: string; // ISO timestamp to fetch updates since (overrides cache timestamp) pageSize?: number; // default 100 (max allowed depends on API) caBundlePath?: string; // path to PEM bundle for self-signed/internal certs + quiet?: boolean; // suppress informational retry / rate limit logs } const CACHE_FILENAME = "malware-advisories.json"; @@ -215,6 +216,44 @@ export class MalwareAdvisorySync { getAdvisories(): MalwareAdvisoryNode[] { return this.cache.advisories; } getLastSync(): string { return this.cache.lastSync; } + private async runQueryWithRetries(query: string, variables: Record, context: string, maxAttempts = 8): Promise { + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return await this.gql(query as any, variables); + } catch (e) { + // Attempt to classify error + const err = e as any; + const status: number | undefined = err.status || err.response?.status; + const message: string = (err.message || "") + " " + JSON.stringify(err.response?.data || {}); + const retryAfterHeader = err.response?.headers?.['retry-after']; + const isSecondary = status === 403 && /secondary rate limit/i.test(message); + const isRetryableStatus = isSecondary || status === 429 || (status && status >= 500 && status < 600); + const shouldRetry = isRetryableStatus && attempt < maxAttempts; + if (!shouldRetry) { + if (!this.opts.quiet) console.error(`GraphQL ${context} failed (status=${status ?? 'n/a'}): ${message.trim()}`); + throw e; + } + let waitMs: number; + if (retryAfterHeader && /^\d+$/.test(retryAfterHeader)) { + waitMs = parseInt(retryAfterHeader, 10) * 1000; + } else { + // Exponential backoff starting ~2s, capped at 30s + waitMs = Math.min(30000, Math.round(2000 * Math.pow(1.6, attempt - 1))); + } + // Add small jitter (0-500ms) + waitMs += Math.floor(Math.random() * 500); + if (!this.opts.quiet) { + const kind = isSecondary ? "secondary-rate-limit" : `status-${status}`; + console.warn(`Retrying ${context} after ${kind} (${attempt}/${maxAttempts}) in ${waitMs}ms...`); + } + await new Promise(r => setTimeout(r, waitMs)); + } + } + // Should never reach here due to throw in catch when attempts exhausted + throw new Error(`Exhausted retries for ${context}`); + } + async sync(): Promise<{ added: number; updated: number; total: number }> { const since = this.opts.since || this.cache.lastSync; let after: string | null = null; @@ -229,82 +268,84 @@ export class MalwareAdvisorySync { const pendingVulnPages: Array<{ ghsaId: string; cursor: string | null; acc: { ecosystem: string | null; name: string | null; updatedAt: string | null; vulnerableVersionRange: string | null }[] }> = []; for (; ;) { + let result: { securityAdvisories: SecurityAdvisoryConnection } | undefined; try { - const result = await this.gql<{ securityAdvisories: SecurityAdvisoryConnection }>(MALWARE_ADVISORIES_QUERY, { - first: pageSize, - after, - updatedSince: since - }); - - const conn: SecurityAdvisoryConnection = result.securityAdvisories; - for (const edge of conn.edges) { - const node = edge.node; - const initialVulns = (node.vulnerabilities?.nodes || []).map(v => ({ - ecosystem: v.package ? v.package.ecosystem : null, - name: v.package ? v.package.name : null, - updatedAt: v.updatedAt || null, - vulnerableVersionRange: v.vulnerableVersionRange || null - })); - const advisory: MalwareAdvisoryNode = { - ghsaId: node.ghsaId, - permalink: node.permalink, - summary: node.summary, - description: node.description, - updatedAt: node.updatedAt, - publishedAt: node.publishedAt, - withdrawnAt: node.withdrawnAt, - references: node.references || [], - identifiers: node.identifiers || [], - severity: node.severity, - cvss: node.cvss, - vulnerabilities: initialVulns - }; - const existing = existingByGhsa.get(advisory.ghsaId); - if (!existing) { - existingByGhsa.set(advisory.ghsaId, advisory); - added++; - } else if (existing.updatedAt !== advisory.updatedAt || existing.description !== advisory.description) { - existingByGhsa.set(advisory.ghsaId, advisory); - updated++; - } - const vulnPageInfo = node.vulnerabilities?.pageInfo; - if (vulnPageInfo?.hasNextPage) { - pendingVulnPages.push({ ghsaId: advisory.ghsaId, cursor: vulnPageInfo.endCursor, acc: advisory.vulnerabilities }); - } - } - if (!conn.pageInfo.hasNextPage || !conn.pageInfo.endCursor) break; - after = conn.pageInfo.endCursor; + result = await this.runQueryWithRetries<{ securityAdvisories: SecurityAdvisoryConnection }>( + MALWARE_ADVISORIES_QUERY, + { first: pageSize, after, updatedSince: since }, + "malware-advisories-page" + ); } catch (e) { - console.error("Error fetching malware advisories:", e); + // Hard failure (non-retryable or exhausted retries) ends loop + if (!this.opts.quiet) console.error("Aborting advisory sync due to error."); break; } + const conn: SecurityAdvisoryConnection = result.securityAdvisories; + for (const edge of conn.edges) { + const node = edge.node; + const initialVulns = (node.vulnerabilities?.nodes || []).map(v => ({ + ecosystem: v.package ? v.package.ecosystem : null, + name: v.package ? v.package.name : null, + updatedAt: v.updatedAt || null, + vulnerableVersionRange: v.vulnerableVersionRange || null + })); + const advisory: MalwareAdvisoryNode = { + ghsaId: node.ghsaId, + permalink: node.permalink, + summary: node.summary, + description: node.description, + updatedAt: node.updatedAt, + publishedAt: node.publishedAt, + withdrawnAt: node.withdrawnAt, + references: node.references || [], + identifiers: node.identifiers || [], + severity: node.severity, + cvss: node.cvss, + vulnerabilities: initialVulns + }; + const existing = existingByGhsa.get(advisory.ghsaId); + if (!existing) { + existingByGhsa.set(advisory.ghsaId, advisory); + added++; + } else if (existing.updatedAt !== advisory.updatedAt || existing.description !== advisory.description) { + existingByGhsa.set(advisory.ghsaId, advisory); + updated++; + } + const vulnPageInfo = node.vulnerabilities?.pageInfo; + if (vulnPageInfo?.hasNextPage) { + pendingVulnPages.push({ ghsaId: advisory.ghsaId, cursor: vulnPageInfo.endCursor, acc: advisory.vulnerabilities }); + } + } + if (!conn.pageInfo.hasNextPage || !conn.pageInfo.endCursor) break; + after = conn.pageInfo.endCursor; } // Fetch remaining vulnerability pages per advisory sequentially for (const pending of pendingVulnPages) { let vAfter = pending.cursor; for (; ;) { + let res: { securityAdvisory: { vulnerabilities: { pageInfo: { hasNextPage: boolean; endCursor: string | null }; nodes: { package?: { ecosystem: string; name: string }; updatedAt?: string; vulnerableVersionRange?: string }[] } } } | undefined; try { - const res = await this.gql<{ securityAdvisory: { vulnerabilities: { pageInfo: { hasNextPage: boolean; endCursor: string | null }; nodes: { package?: { ecosystem: string; name: string }; updatedAt?: string; vulnerableVersionRange?: string }[] } } }>(SINGLE_ADVISORY_VULNS_QUERY, { - ghsaId: pending.ghsaId, - first: 100, - after: vAfter - }); - const vulnConn = res.securityAdvisory.vulnerabilities; - for (const n of vulnConn.nodes) { - pending.acc.push({ - ecosystem: n.package ? n.package.ecosystem : null, - name: n.package ? n.package.name : null, - updatedAt: n.updatedAt || null, - vulnerableVersionRange: n.vulnerableVersionRange || null - }); - } - if (!vulnConn.pageInfo.hasNextPage || !vulnConn.pageInfo.endCursor) break; - vAfter = vulnConn.pageInfo.endCursor; + res = await this.runQueryWithRetries<{ securityAdvisory: { vulnerabilities: { pageInfo: { hasNextPage: boolean; endCursor: string | null }; nodes: { package?: { ecosystem: string; name: string }; updatedAt?: string; vulnerableVersionRange?: string }[] } } }>( + SINGLE_ADVISORY_VULNS_QUERY, + { ghsaId: pending.ghsaId, first: 100, after: vAfter }, + `advisory-vulns:${pending.ghsaId}` + ); } catch (e) { - console.error(`Error paginating vulnerabilities for ${pending.ghsaId}:`, e); + if (!this.opts.quiet) console.error(`Aborting vuln pagination for ${pending.ghsaId}`); break; } + const vulnConn = res.securityAdvisory.vulnerabilities; + for (const n of vulnConn.nodes) { + pending.acc.push({ + ecosystem: n.package ? n.package.ecosystem : null, + name: n.package ? n.package.name : null, + updatedAt: n.updatedAt || null, + vulnerableVersionRange: n.vulnerableVersionRange || null + }); + } + if (!vulnConn.pageInfo.hasNextPage || !vulnConn.pageInfo.endCursor) break; + vAfter = vulnConn.pageInfo.endCursor; } } From 00116c5a430d405231e0a67e07da24ff6c2d9163 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 2 Dec 2025 13:46:50 +0000 Subject: [PATCH 016/108] Allows just malware sync/caching --- README.md | 16 ++++++++++++++++ src/cli.ts | 24 ++++++++++++++++++------ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a0db616..767e848 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,12 @@ Offline match with already-cached malware advisories (no network calls): ```bash npm run start -- --sbom-cache sboms --malware-cache malware-cache --match-malware + +Malware-only advisory sync (no SBOM cache required): + +```bash +npm run start -- --sync-malware --malware-cache malware-cache --token $GITHUB_TOKEN +``` ``` Write malware matches (and optionally search results later) to a JSON file using `--output-file`: @@ -218,6 +224,16 @@ npm run start -- --sbom-cache sboms --malware-cache malware-cache --match-malwar If you also perform a search in the same invocation (add `--purl` or `--purl-file`), the JSON file will contain both `malwareMatches` and `search` top-level keys. +#### Advisory Rate Limit Handling + +Advisory sync uses GitHub GraphQL with adaptive retry/backoff to handle secondary rate limits and transient errors: + +- Retries on `403` secondary rate limit, `429`, and `5xx` responses. +- Honors `Retry-After` when provided; otherwise uses exponential backoff with jitter. +- Respects `--quiet` to suppress retry log messages. + +If retries are exhausted, the sync aborts gracefully and leaves previously cached advisories intact. + #### Ignoring Matches Provide a YAML ignore file via `--ignore-file` to suppress specific matches (before SARIF generation / JSON output). Structure: diff --git a/src/cli.ts b/src/cli.ts index 7cbcc85..178ffeb 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -54,8 +54,10 @@ async function main() { if (!args.enterprise && !args.org && !args.repo) throw new Error("Provide --enterprise, --org or --repo with --sync-sboms"); if (args.enterprise && args.org) throw new Error("Specify only one of --enterprise or --org"); if (args.repo && (args.enterprise || args.org)) throw new Error("Specify only one of --enterprise, --org, or --repo"); + if (!args.sbomCache) throw new Error("--sync-sboms requires --sbom-cache to write updated SBOMs to disk"); } else { - if (!args.sbomCache && !args.malwareCache) throw new Error("Offline mode requires --sbom-cache (omit --sync-sboms)"); + const malwareOnly = !!args["sync-malware"] && !args.sbomCache && !args.purl && !args["purl-file"] && !args["match-malware"] && !args.uploadSarif && !args.interactive; + if (!malwareOnly && !args.sbomCache) throw new Error("Offline mode requires --sbom-cache unless running --sync-malware by itself"); } // If --cli is specified in combination with JSON or CSV, require an output file to avoid mixed stdout streams. if (args.cli && !args.outputFile && (args.json || args.csv)) { @@ -108,9 +110,10 @@ async function main() { const wantCli = !!argv.cli && hasOutputFile; // only allow CLI alongside machine output when writing file let sboms: RepositorySbom[] = []; - let summary: CollectionSummary; + let summary: CollectionSummary | undefined; - const collector = new SbomCollector({ + const needCollector = !!argv.syncSboms || !!argv.sbomCache || !!argv.purl || !!argv["purl-file"] || !!argv["match-malware"] || !!argv.uploadSarif || !!argv.interactive; + const collector = needCollector ? new SbomCollector({ token: token, enterprise: argv.enterprise as string | undefined, org: argv.org as string | undefined, @@ -134,9 +137,9 @@ async function main() { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore componentDetectionBinPath: argv["component-detection-bin"] as string | undefined, - }); + }) : undefined; - if (argv.sbomCache || argv.syncSboms) { + if (collector && (argv.sbomCache || argv.syncSboms)) { if (!quiet) process.stderr.write(chalk.cyan(offline ? "Loading SBOMs from cache..." : "Collecting SBOMs from cache & GitHub...") + "\n"); sboms = await collector.collect(); summary = collector.getSummary(); @@ -252,7 +255,7 @@ async function main() { const combinedPurlsRaw = [...(argv.purl as string[] ?? []), ...filePurls]; const combinedPurls = combinedPurlsRaw.map(p => p.startsWith("pkg:") ? p : `pkg:${p}`); let searchMap: Map | undefined; - if (combinedPurls.length) { + if (combinedPurls.length && collector) { searchMap = collector.searchByPurlsWithReasons(combinedPurls); } if (wantJson) { @@ -402,6 +405,11 @@ async function main() { } const list = trimmed.split(/[\s,]+/).filter(Boolean); try { + if (!collector) { + console.error(chalk.red("Interactive search requires SBOMs; provide --sbom-cache or run with --sync-sboms.")); + rl.prompt(); + return; + } const map = collector.searchByPurlsWithReasons(list.map(p => p.startsWith("pkg:") ? p : `pkg:${p}`)); runSearchCli(list, map); } catch (e) { @@ -421,6 +429,10 @@ async function main() { { name: "purl", message: "Enter a PURL (blank to exit)", type: "input" } ]); if (!ans.purl) break; + if (!collector) { + console.error(chalk.red("Interactive search requires SBOMs; provide --sbom-cache or run with --sync-sboms.")); + continue; + } const map = collector.searchByPurlsWithReasons([ans.purl.startsWith("pkg:") ? ans.purl : `pkg:${ans.purl}`]); runSearchCli([ans.purl], map); } From 5c9262622bac8128a5deb79c30a00f351f2aea05 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:03:04 +0000 Subject: [PATCH 017/108] Fix single repo behaviour on first pass --- src/cli.ts | 24 +++++++++++++++++++----- src/sbomCollector.ts | 17 +++++++++++++++-- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 178ffeb..918518d 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -6,7 +6,9 @@ import { SbomCollector } from "./sbomCollector.js"; import inquirer from "inquirer"; // still used elsewhere if needed import readline from "readline"; import { CollectionSummary, RepositorySbom } from "./types.js"; -const { MalwareAdvisorySync } = await import("./malwareAdvisories.js"); +import { MalwareAdvisorySync } from "./malwareAdvisories.js"; +import { MalwareMatch } from "./malwareMatcher.js"; +import fs from "fs"; async function main() { const argv = await yargs(hideBin(process.argv)) @@ -156,14 +158,14 @@ async function main() { }); if (argv.syncMalware) { - if (!quiet) process.stderr.write(chalk.cyan("Syncing malware advisories from GitHub Advisory Database...") + "\n"); const { added, updated, total } = await mas.sync(); if (!quiet) process.stderr.write(chalk.green(`Malware advisories sync complete. Added: ${added}, Updated: ${updated}, Total cached: ${total}`) + "\n"); } - let malwareMatches: import("./malwareMatcher.js").MalwareMatch[] | undefined; + let malwareMatches: MalwareMatch[] = []; + if (argv["match-malware"]) { const { matchMalware, buildSarifPerRepo, writeSarifFiles, uploadSarifPerRepo } = await import("./malwareMatcher.js"); malwareMatches = matchMalware(mas.getAdvisories(), sboms, { advisoryDateCutoff: argv["malware-cutoff"] as string | undefined }); @@ -254,16 +256,28 @@ async function main() { } const combinedPurlsRaw = [...(argv.purl as string[] ?? []), ...filePurls]; const combinedPurls = combinedPurlsRaw.map(p => p.startsWith("pkg:") ? p : `pkg:${p}`); + + console.debug(chalk.gray("Searching for purls:")); + console.debug(chalk.gray(combinedPurls)); + + console.debug(collector?.getAllSboms()) + console.debug(sboms); + let searchMap: Map | undefined; if (combinedPurls.length && collector) { searchMap = collector.searchByPurlsWithReasons(combinedPurls); } + + if (searchMap) { + console.debug(chalk.gray("Found purls:")); + console.debug(chalk.gray(Array.from(searchMap.entries()))); + } + if (wantJson) { const jsonSearch = Array.from((searchMap || new Map()).entries()).map(([repo, entries]) => ({ repo, matches: entries })); if (hasOutputFile) { try { - const fs = await import("fs"); - let existing: { search?: unknown; malwareMatches?: import("./malwareMatcher.js").MalwareMatch[] } = {}; + let existing: { search?: unknown; malwareMatches?: MalwareMatch[] } = {}; if (fs.existsSync(argv.outputFile as string)) { try { existing = JSON.parse(fs.readFileSync(argv.outputFile as string, "utf8")); } catch { existing = {}; } } diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 4ecd61c..cd87770 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -266,6 +266,8 @@ export class SbomCollector { sbom = baseline; } + console.debug(sbom); + // Branch scanning (optional) if (this.opts.includeBranches && sbom?.sbom) { @@ -303,19 +305,25 @@ export class SbomCollector { } catch (e) { // Non-fatal; annotate decision this.decisions[fullName] = (this.decisions[fullName] || "") + ` (branch scan error: ${(e as Error).message})`; + console.debug((e as Error).message); } - newSboms.push(sbom); + console.debug(sbom); if (sbom.error) this.summary.failedCount++; else this.summary.successCount++; // Write freshly fetched SBOM immediately if a cache directory is configured if (this.opts.loadFromDir && this.opts.syncSboms && this.opts.loadFromDir.length) { try { writeOne(sbom, { outDir: this.opts.loadFromDir }); } catch { /* ignore write errors */ } } } + if (sbom) { + newSboms.push(sbom); + } processed++; renderBar(); })); await Promise.all(tasks); - newSboms = newSboms.filter(s => repoNames.has(s.repo)); + console.debug(repoNames); + newSboms = newSboms.filter(s => repoNames.has(s.repo) || repoNames.has(s.repo.split("/")[1])); + console.debug(newSboms); this.sboms.push(...newSboms); } if (this.opts.showProgressBar) process.stdout.write("\n"); @@ -548,6 +556,7 @@ export class SbomCollector { return { raw: trimmed, lower, isPrefixWildcard: false, exact: lower }; }; const queries: ParsedQuery[] = purls.map(parseQuery).filter((q): q is ParsedQuery => !!q); + console.debug(queries); const results = new Map(); if (!queries.length) return results; const applyQueries = (candidatePurls: string[], queries: ParsedQuery[], found: Map, branchTag?: string, fallbackVersion?: string) => { @@ -586,7 +595,10 @@ export class SbomCollector { } }; + console.debug(this.sboms); + for (const repoSbom of this.sboms) { + console.debug(repoSbom); if (repoSbom.error) continue; interface ExtRef { referenceType: string; referenceLocator: string } const found = new Map(); // purl -> query @@ -600,6 +612,7 @@ export class SbomCollector { // Include dependency review diff additions/updates (head packages only) if (repoSbom.branchDiffs) { const diffs = repoSbom.branchDiffs.values(); + console.debug(diffs); for (const diff of diffs) { for (const change of diff.changes) { if (change.changeType !== "added" && change.changeType !== "updated") continue; From 9f284c241270ab2da83062e41c30636250954245 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:44:47 +0000 Subject: [PATCH 018/108] Fixed missing diff from skipped branches --- src/cli.ts | 11 ----------- src/sbomCollector.ts | 18 ++++++------------ 2 files changed, 6 insertions(+), 23 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 918518d..f040fe1 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -257,22 +257,11 @@ async function main() { const combinedPurlsRaw = [...(argv.purl as string[] ?? []), ...filePurls]; const combinedPurls = combinedPurlsRaw.map(p => p.startsWith("pkg:") ? p : `pkg:${p}`); - console.debug(chalk.gray("Searching for purls:")); - console.debug(chalk.gray(combinedPurls)); - - console.debug(collector?.getAllSboms()) - console.debug(sboms); - let searchMap: Map | undefined; if (combinedPurls.length && collector) { searchMap = collector.searchByPurlsWithReasons(combinedPurls); } - if (searchMap) { - console.debug(chalk.gray("Found purls:")); - console.debug(chalk.gray(Array.from(searchMap.entries()))); - } - if (wantJson) { const jsonSearch = Array.from((searchMap || new Map()).entries()).map(([repo, entries]) => ({ repo, matches: entries })); if (hasOutputFile) { diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index cd87770..91ee9c2 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -266,10 +266,8 @@ export class SbomCollector { sbom = baseline; } - console.debug(sbom); - // Branch scanning (optional) - if (this.opts.includeBranches && sbom?.sbom) { + if (this.opts.includeBranches && sbom && sbom.sbom) { console.debug(chalk.blue(`Scanning branches for ${fullName}...`)); @@ -286,11 +284,15 @@ export class SbomCollector { console.error(chalk.red(`Failed to get latest commit for ${fullName} branch ${b.name}.`)); continue; } - const existing = sbom?.branchDiffs instanceof Map ? sbom.branchDiffs.get(b.name) : undefined; + const existing = sbom.branchDiffs instanceof Map ? sbom.branchDiffs.get(b.name) : undefined; if (await this.isCommitNewer(latestCommit, existing)) { console.debug(chalk.green(`Fetching branch diff for ${fullName} branch ${b.name}...`)); } else { console.debug(chalk.yellow(`Skipping branch diff for ${fullName} branch ${b.name} (no new commits).`)); + // keep existing diff + if (existing) { + branchDiffs.set(b.name, existing); + } continue; } @@ -307,7 +309,6 @@ export class SbomCollector { this.decisions[fullName] = (this.decisions[fullName] || "") + ` (branch scan error: ${(e as Error).message})`; console.debug((e as Error).message); } - console.debug(sbom); if (sbom.error) this.summary.failedCount++; else this.summary.successCount++; // Write freshly fetched SBOM immediately if a cache directory is configured if (this.opts.loadFromDir && this.opts.syncSboms && this.opts.loadFromDir.length) { @@ -321,9 +322,7 @@ export class SbomCollector { renderBar(); })); await Promise.all(tasks); - console.debug(repoNames); newSboms = newSboms.filter(s => repoNames.has(s.repo) || repoNames.has(s.repo.split("/")[1])); - console.debug(newSboms); this.sboms.push(...newSboms); } if (this.opts.showProgressBar) process.stdout.write("\n"); @@ -556,7 +555,6 @@ export class SbomCollector { return { raw: trimmed, lower, isPrefixWildcard: false, exact: lower }; }; const queries: ParsedQuery[] = purls.map(parseQuery).filter((q): q is ParsedQuery => !!q); - console.debug(queries); const results = new Map(); if (!queries.length) return results; const applyQueries = (candidatePurls: string[], queries: ParsedQuery[], found: Map, branchTag?: string, fallbackVersion?: string) => { @@ -595,10 +593,7 @@ export class SbomCollector { } }; - console.debug(this.sboms); - for (const repoSbom of this.sboms) { - console.debug(repoSbom); if (repoSbom.error) continue; interface ExtRef { referenceType: string; referenceLocator: string } const found = new Map(); // purl -> query @@ -612,7 +607,6 @@ export class SbomCollector { // Include dependency review diff additions/updates (head packages only) if (repoSbom.branchDiffs) { const diffs = repoSbom.branchDiffs.values(); - console.debug(diffs); for (const diff of diffs) { for (const change of diff.changes) { if (change.changeType !== "added" && change.changeType !== "updated") continue; From f6d37c2b8d1d824a6fa3bb4355df1e2437cce804 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 2 Dec 2025 17:02:02 +0000 Subject: [PATCH 019/108] Allow forcing submission with Component Detection --- src/cli.ts | 2 ++ src/componentSubmission.ts | 39 ++++++++++++++++++-------------------- src/sbomCollector.ts | 15 ++++++++++++++- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index f040fe1..b952387 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -49,6 +49,7 @@ async function main() { .option("submit-on-missing-snapshot", { type: "boolean", default: false, describe: "When dependency review diff returns 404 (missing snapshot), run Component Detection to submit a snapshot, then retry." }) .option("submit-languages", { type: "array", describe: "Limit snapshot submission to these languages (e.g., JavaScript,TypeScript,Python,Maven)." }) .option("component-detection-bin", { type: "string", describe: "Path to a local component-detection executable to use for snapshot submission (skips download)." }) + .option("force-submission", { type: "boolean", default: false, describe: "Always run Dependency Submission for scanned branches before fetching diffs." }) .option("debug", { type: "boolean", default: false, describe: "Enable debug logging" }) .check(args => { const syncing = !!args.syncSboms; @@ -134,6 +135,7 @@ async function main() { branchLimit: argv["branch-limit"] as number | undefined, branchDiffBase: argv["diff-base"] as string | undefined, submitOnMissingSnapshot: argv["submit-on-missing-snapshot"] as boolean, + forceSubmission: argv["force-submission"] as boolean, submitLanguages: (argv["submit-languages"] as string[] | undefined) || undefined, // Pass through as part of options bag used by submission helper via collector // eslint-disable-next-line @typescript-eslint/ban-ts-comment diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index db264ee..8173954 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -24,11 +24,11 @@ export interface SubmitOpts { componentDetectionBinPath?: string; // optional path to component-detection executable } -export async function getLanguageIntersection(octokit: any, owner: string, repo: string, languages: string[], quiet: boolean = false): Promise { +export async function getLanguageIntersection(octokit: any, owner: string, repo: string, languages: string[] | undefined, quiet: boolean = false): Promise { const langResp = await octokit.request('GET /repos/{owner}/{repo}/languages', { owner, repo }); const repoLangs = Object.keys(langResp.data || {}); const wanted = languages; - const intersect = repoLangs.filter(l => wanted.some(w => w.toLowerCase() === l.toLowerCase())); + const intersect = wanted ? repoLangs.filter(l => wanted.some(w => w.toLowerCase() === l.toLowerCase())) : repoLangs; if (!intersect.length) { if (!quiet) console.error(chalk.yellow(`Skipping submission: none of selected languages present in repo (${repoLangs.join(', ')})`)); return []; @@ -61,27 +61,24 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise; @@ -285,7 +287,7 @@ export class SbomCollector { continue; } const existing = sbom.branchDiffs instanceof Map ? sbom.branchDiffs.get(b.name) : undefined; - if (await this.isCommitNewer(latestCommit, existing)) { + if (await this.isCommitNewer(latestCommit, existing) || this.opts.forceSubmission) { console.debug(chalk.green(`Fetching branch diff for ${fullName} branch ${b.name}...`)); } else { console.debug(chalk.yellow(`Skipping branch diff for ${fullName} branch ${b.name} (no new commits).`)); @@ -300,6 +302,17 @@ export class SbomCollector { const base = this.opts.branchDiffBase || sbom?.defaultBranch; if (!base) { console.error(chalk.red(`Cannot compute branch diff for ${fullName} branch ${b.name} because base branch is undefined.`)); continue; } if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); + // Optionally perform dependency submission up front for the branch + if (this.opts.forceSubmission) { + try { + console.log(chalk.blue(`Force-submission enabled: submitting component snapshot for ${fullName} branch ${b.name}...`)); + await submitSnapshotIfPossible({ octokit: this.octokit, owner: org, repo: repo.name, branch: b.name, languages: this.opts.submitLanguages, quiet: this.opts.quiet, componentDetectionBinPath: this.opts.componentDetectionBinPath }); + // brief delay to allow snapshot ingestion + await new Promise(r => setTimeout(r, 1500)); + } catch (subErr) { + console.error(chalk.red(`Force submission failed for ${fullName} branch ${b.name}: ${(subErr as Error).message}`)); + } + } const diff = await this.fetchDependencyReviewDiff(org, repo.name, base, b.name); branchDiffs.set(b.name, diff); } From 33415a9c940581b4108c4716213b0e631b494227 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 2 Dec 2025 17:18:10 +0000 Subject: [PATCH 020/108] Fixed not waiting for process --- src/componentDetection.ts | 40 +++++++++++++++++++++++++++++--------- src/componentSubmission.ts | 2 +- src/sbomCollector.ts | 1 - 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 934abc6..44db415 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -24,8 +24,15 @@ export default class ComponentDetection { } else { await this.downloadLatestRelease(); } - await this.runComponentDetection(path); - return await this.getManifestsFromResults(); + + // make an empty file to write results into + fs.writeFileSync(this.outputPath, '', { flag: 'w' }); + + if (!await this.runComponentDetection(this.outputPath)) { + return; + } + + return await this.getManifestsFromResults(this.outputPath); } // Get the latest release from the component-detection repo, download the tarball, and extract it public static async downloadLatestRelease() { @@ -55,21 +62,36 @@ export default class ComponentDetection { } // Run the component-detection CLI on the path specified - public static async runComponentDetection(path: string) { + public static async runComponentDetection(path: string): Promise { console.info("Running component-detection"); try { - await spawn(`${this.componentDetectionPath}`, ['scan', '--SourceDirectory', path, '--ManifestFile', this.outputPath], { stdio: 'pipe' }); + const process = await spawn(`${this.componentDetectionPath}`, ['scan', '--SourceDirectory', path, '--ManifestFile', this.outputPath], { stdio: 'pipe' }); + + const pid = process.pid; + + process.on('exit', (code) => { + console.info(`Component-detection process ${pid} exited with code ${code}`); + if (code === 0) { + console.info(`Component-detection completed successfully.`); + return true; + } else { + console.error(`Component-detection failed with exit code ${code}.`); + return false; + } + }); + } catch (error: any) { console.error(error); + return false; } + + return false; } - public static async getManifestsFromResults(): Promise { - console.info("Getting manifests from results"); - console.info(`Reading results from ${this.outputPath}`); - console.info(`Stat: ${fs.statSync(this.outputPath)}`); - const results = await fs.readFileSync(this.outputPath, 'utf8'); + public static async getManifestsFromResults(file: string): Promise { + console.info(`Reading results from ${file}`); + const results = await fs.readFileSync(file, 'utf8'); var json: any = JSON.parse(results); let dependencyGraphs: DependencyGraphs = this.normalizeDependencyGraphPaths(json.dependencyGraphs, '.'); return this.processComponentsToManifests(json.componentsFound, dependencyGraphs); diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index 8173954..35f5b55 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -65,7 +65,7 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise Date: Tue, 2 Dec 2025 18:12:10 +0000 Subject: [PATCH 021/108] Now runs Component Detection correctly, on correct path --- src/componentDetection.ts | 65 +++++++++++++++++++++++--------------- src/componentSubmission.ts | 19 +++++------ 2 files changed, 47 insertions(+), 37 deletions(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 44db415..ebdf041 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -7,10 +7,9 @@ import { import fetch from 'cross-fetch' import fs from 'fs' import { spawn } from 'child_process'; -//import dotenv from 'dotenv' import path from 'path'; import { tmpdir } from 'os'; -//dotenv.config(); +import { StringDecoder } from 'node:string_decoder'; export default class ComponentDetection { public static componentDetectionPath = process.platform === "win32" ? './component-detection.exe' : './component-detection'; @@ -28,7 +27,7 @@ export default class ComponentDetection { // make an empty file to write results into fs.writeFileSync(this.outputPath, '', { flag: 'w' }); - if (!await this.runComponentDetection(this.outputPath)) { + if (!await this.runComponentDetection(path)) { return; } @@ -62,31 +61,45 @@ export default class ComponentDetection { } // Run the component-detection CLI on the path specified - public static async runComponentDetection(path: string): Promise { + public static runComponentDetection(path: string): Promise { console.info("Running component-detection"); - try { - const process = await spawn(`${this.componentDetectionPath}`, ['scan', '--SourceDirectory', path, '--ManifestFile', this.outputPath], { stdio: 'pipe' }); - - const pid = process.pid; - - process.on('exit', (code) => { - console.info(`Component-detection process ${pid} exited with code ${code}`); - if (code === 0) { - console.info(`Component-detection completed successfully.`); - return true; - } else { - console.error(`Component-detection failed with exit code ${code}.`); - return false; - } - }); - - } catch (error: any) { - console.error(error); - return false; - } - - return false; + console.debug(`Writing to output file: ${this.outputPath}`); + + return new Promise((resolve, reject) => { + try { + const child = spawn(`${this.componentDetectionPath}`, ['scan', '--SourceDirectory', path, '--ManifestFile', this.outputPath], { stdio: 'pipe' }); + const pid = child.pid; + + child.on('error', (err) => { + console.error(`Component-detection process ${pid} error: ${err instanceof Error ? err.message : String(err)}`); + reject(err); + }); + + child.on('exit', (code) => { + console.info(`Component-detection process ${pid} exited with code ${code}`); + if (code === 0) { + console.info(`Component-detection completed successfully.`); + resolve(true); + } else { + console.error(`Component-detection failed with exit code ${code}.`); + const decoder = new StringDecoder('utf8'); + const stdout = child.stdout.read(); + const stderr = child.stderr.read(); + if (stdout) { + console.error(decoder.write(stdout)); + } + if (stderr) { + console.error(decoder.write(stderr)); + } + resolve(false); + } + }); + } catch (error: any) { + console.error(error); + reject(error); + } + }); } public static async getManifestsFromResults(file: string): Promise { diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index 35f5b55..195fea9 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -3,10 +3,10 @@ import { spawn, ChildProcess } from 'child_process'; import path from 'path'; import fs from 'fs'; import os from 'os'; -import type { Context } from '@actions/github/lib/context.js' import ComponentDetection from './componentDetection.js'; import { + Job, Snapshot, submitSnapshot } from '@github/dependency-submission-toolkit'; @@ -155,21 +155,18 @@ export async function run(owner: string, repo: string, sha: string, ref: string, url: detectorUrl, }; - const context: Context = { - repo: { owner: owner, repo: repo }, - job: 'github-sbom-toolkit', - runId: Math.floor(Math.random() * Number.MAX_SAFE_INTEGER), - ref: ref, - sha: sha, - // required for Context type but not used in snapshot submission - payload: {}, eventName: '', workflow: '', action: '', actor: '', runNumber: 0, runAttempt: 0, apiUrl: '', serverUrl: '', graphqlUrl: '', issue: { owner: '', repo: '', number: 0 } + const job: Job = { + correlator: 'github-sbom-toolkit', + id: Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString() }; - let snapshot = new Snapshot(detector, context); + let snapshot = new Snapshot(detector, undefined, job); manifests?.forEach((manifest) => { snapshot.addManifest(manifest); }); - submitSnapshot(snapshot); + console.debug(snapshot.prettyJSON()) + + //submitSnapshot(snapshot); } From 6314b1443b75981b884994387634939cb6184ac2 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 2 Dec 2025 18:31:20 +0000 Subject: [PATCH 022/108] Fixed mistake reading wrong path in Component Detection --- src/componentDetection.ts | 2 +- src/componentSubmission.ts | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index ebdf041..94cbe4f 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -62,7 +62,7 @@ export default class ComponentDetection { // Run the component-detection CLI on the path specified public static runComponentDetection(path: string): Promise { - console.info("Running component-detection"); + console.info(`Running component-detection on ${path}`); console.debug(`Writing to output file: ${this.outputPath}`); diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index 195fea9..d0286d8 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -74,7 +74,7 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise Date: Thu, 4 Dec 2025 14:59:50 +0000 Subject: [PATCH 023/108] Fixed missing graphs by normalising appropriately --- src/componentDetection.ts | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 94cbe4f..34bfedc 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -31,7 +31,7 @@ export default class ComponentDetection { return; } - return await this.getManifestsFromResults(this.outputPath); + return await this.getManifestsFromResults(this.outputPath, path); } // Get the latest release from the component-detection repo, download the tarball, and extract it public static async downloadLatestRelease() { @@ -77,9 +77,9 @@ export default class ComponentDetection { }); child.on('exit', (code) => { - console.info(`Component-detection process ${pid} exited with code ${code}`); + console.debug(`Component-detection process ${pid} exited with code ${code}`); if (code === 0) { - console.info(`Component-detection completed successfully.`); + console.debug(`Component-detection completed successfully.`); resolve(true); } else { console.error(`Component-detection failed with exit code ${code}.`); @@ -102,11 +102,13 @@ export default class ComponentDetection { }); } - public static async getManifestsFromResults(file: string): Promise { + public static async getManifestsFromResults(file: string, path: string): Promise { console.info(`Reading results from ${file}`); const results = await fs.readFileSync(file, 'utf8'); var json: any = JSON.parse(results); - let dependencyGraphs: DependencyGraphs = this.normalizeDependencyGraphPaths(json.dependencyGraphs, '.'); + + let dependencyGraphs: DependencyGraphs = this.normalizeDependencyGraphPaths(json.dependencyGraphs, path); + return this.processComponentsToManifests(json.componentsFound, dependencyGraphs); } @@ -126,6 +128,9 @@ export default class ComponentDetection { return; } + console.debug(`Processing component: ${component.component.id}`); + console.debug(`Component details: ${JSON.stringify(component.component.packageUrl, null, 2)}`); + const packageUrl = ComponentDetection.makePackageUrl(component.component.packageUrl); // Skip if the packageUrl is empty (indicates an invalid or missing packageUrl) @@ -164,7 +169,7 @@ export default class ComponentDetection { try { const referrerPackage = packageCache.lookupPackage(referrerUrl); if (referrerPackage === pkg) { - console.debug(`Skipping self-reference for package: ${pkg.id}`); + console.debug(`Found self-reference for package: ${pkg.id}`); return; // Skip self-references } if (referrerPackage) { @@ -179,6 +184,9 @@ export default class ComponentDetection { // Create manifests const manifests: Array = []; + console.debug("Dependency Graphs:"); + console.debug(JSON.stringify(dependencyGraphs, null, 2)); + // Check the locationsFoundAt for every package and add each as a manifest this.addPackagesToManifests(packages, manifests, dependencyGraphs); From 3a89bfb50830b40c7acefc0235307001eded8124 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:11:13 +0000 Subject: [PATCH 024/108] CHANGELOG --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..05384e7 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,26 @@ +# Changelog + +## [2025-12-04] – 0.2.0 - Branch scanning and dependency submission + +- Branch scanning: + - Fetch SBOM diffs for non‑default branches via Dependency Review API. + - Added `--branch-scan`, `--branch-limit`, and `--diff-base` CLI flags. +- Dependency Submission integration: + - Automatically submits dependency snapshots for branches being scanned, if not already present, using Component Detection. + - Language-aware sparse checkout. + - Use a pre-downloaded binary (`--component-detection-bin`) or an auto-downloaded release. +- Search and matching: + - Refactored search to de-duplicate logic and include branch diffs (added/updated packages only). + - Malware matching enhanced to enumerate packages from diffs; matches annotated with branch. + - CLI and CSV outputs include branch context; CSV adds a `branch` column. +- CLI and UX improvements: + - Argument validation updated: `--sync-sboms` requires `--sbom-cache`. + - Malware-only mode: allow `--sync-malware` without `--sbom-cache` (requires `--malware-cache`). + - JSON/CLI/CSV interaction clarified and documented. + - Added examples for malware-only sync and branch scanning. +- Advisory sync robustness: + - GraphQL advisory sync now implements adaptive retries with exponential backoff and `Retry-After` support; respects `--quiet`. + +## [2025-10-06] - 0.1.0 - Initial public release + +- Initial release, with: SBOM sync; malware sync; malware matching; CLI, file based and interactive PURL searching. SARIF, CSV and JSON outputs supported. From 1887425c573b44b77f6cafb97e3475647359c24f Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:09:38 +0000 Subject: [PATCH 025/108] Fixed submission by fixing calling of git to capture STDOUT properly --- package-lock.json | 4 +- src/componentDetection.ts | 4 +- src/componentSubmission.ts | 93 ++++++++++++++++++++++++++++++++------ src/sbomCollector.ts | 2 +- 4 files changed, 84 insertions(+), 19 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5b5fb56..f5789b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "github-sbom-toolkit", - "version": "0.1.0", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "github-sbom-toolkit", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { "@github/dependency-submission-toolkit": "^2.0.5", "@octokit/core": "^7.0.6", diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 34bfedc..ad0d746 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -62,7 +62,7 @@ export default class ComponentDetection { // Run the component-detection CLI on the path specified public static runComponentDetection(path: string): Promise { - console.info(`Running component-detection on ${path}`); + console.debug(`Running component-detection on ${path}`); console.debug(`Writing to output file: ${this.outputPath}`); @@ -103,7 +103,7 @@ export default class ComponentDetection { } public static async getManifestsFromResults(file: string, path: string): Promise { - console.info(`Reading results from ${file}`); + console.debug(`Reading results from ${file}`); const results = await fs.readFileSync(file, 'utf8'); var json: any = JSON.parse(results); diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index d0286d8..14046bb 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -1,5 +1,5 @@ import chalk from 'chalk'; -import { spawn, ChildProcess } from 'child_process'; +import { spawn, execFile } from 'child_process'; import path from 'path'; import fs from 'fs'; import os from 'os'; @@ -8,11 +8,12 @@ import ComponentDetection from './componentDetection.js'; import { Job, Snapshot, - submitSnapshot } from '@github/dependency-submission-toolkit'; +import { Octokit } from 'octokit'; +import { RequestError } from '@octokit/request-error' export interface SubmitOpts { - octokit?: any; // Octokit instance, optional + octokit: Octokit; owner: string; repo: string; branch: string; @@ -51,8 +52,9 @@ export async function sparseCheckout(owner: string, repo: string, branch: string await execGit(['fetch', '--depth=1', 'origin', branch], { cwd }); await execGit(['checkout', 'FETCH_HEAD'], { cwd }); - const process = await execGit(['rev-parse', 'HEAD'], { cwd: destDir }); - const sha = process?.stdout?.toString().trim(); + const { stdout: shaOut } = await execGit(['rev-parse', 'HEAD'], { cwd: destDir }); + const sha = shaOut.trim(); + console.debug(`Checked out ${owner}/${repo}@${branch} to ${destDir} at commit ${sha}`); return sha; } @@ -74,7 +76,7 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise { - return await new Promise((resolve, reject) => { - const child = spawn('git', args, { cwd: opts.cwd, stdio: 'pipe' }); - child.on('error', reject); - child.on('exit', code => code === 0 ? resolve(child) : reject(new Error(`git ${args.join(' ')} exit ${code}`))); +async function execGit(args: string[], opts: { cwd: string, quiet?: boolean }): Promise<{ stdout: string; stderr: string }> { + return await new Promise<{ stdout: string; stderr: string }>((resolve, reject) => { + execFile('git', args, { cwd: opts.cwd, encoding: 'utf8', maxBuffer: 10 * 1024 * 1024 }, (error, stdout, stderr) => { + if (error) { + const msg = stderr?.trim() || error.message; + reject(new Error(`git ${args.join(' ')} failed: ${msg}`)); + } else { + resolve({ stdout, stderr: stderr ?? '' }); + } + }); }); } -export async function run(tmpDir: string, owner: string, repo: string, sha: string, ref: string, componentDetectionBinPath?: string) { +export async function run(octokit: Octokit, tmpDir: string, owner: string, repo: string, sha: string, ref: string, componentDetectionBinPath?: string) { let manifests = await ComponentDetection.scanAndGetManifests( tmpDir, @@ -160,12 +167,70 @@ export async function run(tmpDir: string, owner: string, repo: string, sha: stri }; let snapshot = new Snapshot(detector, undefined, job); + snapshot.ref = `refs/heads/${ref}`; + snapshot.sha = sha; + + console.debug(`Submitting snapshot for ${owner}/${repo} at ${snapshot.ref} (${snapshot.sha}) with ${manifests?.length || 0} manifests`); manifests?.forEach((manifest) => { snapshot.addManifest(manifest); }); + submitSnapshot(octokit, snapshot, { owner, repo }); +} + +/** + * submitSnapshot submits a snapshot to the Dependency Submission API - vendored in from @github/dependency-submission-toolkit, to make it work at the CLI, vs in Actions. + * + * @param {Snapshot} snapshot + * @param {Repo} repo + */ +export async function submitSnapshot( + octokit: Octokit, + snapshot: Snapshot, + repo: { owner: string; repo: string } +) { + console.debug('Submitting snapshot...') console.debug(snapshot.prettyJSON()) - //submitSnapshot(snapshot); -} + try { + const response = await octokit.request( + 'POST /repos/{owner}/{repo}/dependency-graph/snapshots', + { + headers: { + accept: 'application/vnd.github.foo-bar-preview+json' + }, + owner: repo.owner, + repo: repo.repo, + ...snapshot + } + ) + const result = response.data.result + if (result === 'SUCCESS' || result === 'ACCEPTED') { + console.debug( + `Snapshot successfully created at ${response.data.created_at.toString()}` + + ` with id ${response.data.id}` + ) + } else { + console.error( + `Snapshot creation failed with result: "${result}: ${response.data.message}"` + ) + } + } catch (error) { + if (error instanceof RequestError) { + console.error( + `HTTP Status ${error.status} for request ${error.request.method} ${error.request.url}` + ) + if (error.response) { + console.error( + `Response body:\n${JSON.stringify(error.response.data, undefined, 2)}` + ) + } + } + if (error instanceof Error) { + console.error(error.message) + if (error.stack) console.error(error.stack) + } + throw new Error(`Failed to submit snapshot: ${error}`) + } +} \ No newline at end of file diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 8e713e5..f02c325 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -305,7 +305,7 @@ export class SbomCollector { // Optionally perform dependency submission up front for the branch if (this.opts.forceSubmission) { try { - console.log(chalk.blue(`Force-submission enabled: submitting component snapshot for ${fullName} branch ${b.name}...`)); + console.debug(chalk.blue(`Force-submission enabled: submitting component snapshot for ${fullName} branch ${b.name}...`)); await submitSnapshotIfPossible({ octokit: this.octokit, owner: org, repo: repo.name, branch: b.name, languages: this.opts.submitLanguages, quiet: this.opts.quiet, componentDetectionBinPath: this.opts.componentDetectionBinPath }); // brief delay to allow snapshot ingestion await new Promise(r => setTimeout(r, 1500)); From 9ea2fa01cfa93e0fca450fb0a5b913be37df66d6 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:22:17 +0000 Subject: [PATCH 026/108] Update docs --- README.md | 43 +++++++++++++++---------------------------- 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 767e848..4f35b73 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Supports human-readable, JSON, CSV and SARIF output. SARIF alerts can be uploade - Optional progress bar while fetching SBOMs - Option to suppress secondary rate limit warnings, and full quiet mode to suppress informative messages - Adaptive backoff: each secondary rate limit hit increases the SBOM fetch delay by 10% to reduce future throttling +- Optional branch scanning†: fetch SBOM diffs with Dependency Review for non-default branches and submit missing dependency snapshots if needed with Component Detection + Dependency Submission - Offline caching of SBOMs and security advisories with incremental updates - Matching: - Version-aware matching of SBOM packages against malware advisories @@ -27,9 +28,11 @@ Supports human-readable, JSON, CSV and SARIF output. SARIF alerts can be uploade - Output: - Human-readable console output - JSON or CSV output (to stdout or file) with both search and malware matches - - Optional SARIF 2.1.0 output per repository for malware matches with optional Code Scanning upload + - Optional SARIF 2.1.0 output per repository for malware matches + - includes Code Scanning upload† - Works with GitHub.com, GitHub Enterprise Server, GitHub Enterprise Managed Users and GitHub Enterprise Cloud with Data Residency (custom base URL) -- Optional branch scanning: fetch SBOMs for non-default branches (limited) and compute Dependency Review diffs vs the default (or chosen base) branch + +† GitHub Advanced Security or GitHub Code Security required for this feature ## Usage @@ -83,7 +86,7 @@ If a branch SBOM or diff retrieval fails, the error is recorded but does not sto #### Handling Missing Dependency Review Snapshots -If the Dependency Review API returns a 404 for a branch diff (commonly due to a missing dependency snapshot on either the base or head commit), the toolkit can optionally attempt to generate and submit a snapshot using the Component Detection + Dependency Submission Action. +If the Dependency Review API returns a 404 for a branch diff (commonly due to a missing dependency snapshot on either the base or head commit), the toolkit can optionally attempt to generate and submit a snapshot using Component Detection and Dependency Submission. This is vendored-in and forked from the public [Component Detection Dependency Submission Action](https://github.com/your-org/component-detection-dependency-submission-action). Enable automatic submission + retry with: @@ -91,23 +94,7 @@ Enable automatic submission + retry with: --submit-on-missing-snapshot ``` -This requires the action repository to be present as a git submodule (or copied) at the path: - -```bash -component-detection-dependency-submission-action/ -``` - -After cloning, initialize submodules: - -```bash -git submodule update --init --recursive -``` - -Build the action (if not already built) so its `dist/entrypoint.js` exists. The toolkit will then: - -1. Detect 404 from diff endpoint. -2. Invoke the action locally to produce a snapshot for the target branch. -3. Wait briefly then retry the dependency review diff once. +The tool will attempt to download the latest Component Detection release from GitHub Releases into the current directory, to run it, unless you provide a local binary with `--component-detection-bin`. If submission fails, the original 404 reason is retained and collection proceeds. @@ -125,15 +112,16 @@ npm run start -- \ --component-detection-bin /usr/local/bin/component-detection ``` -GitHub Enterprise Server example: +On MacOS, you may find that system protection prevents running a downloaded binary. You can [check out the .NET code](https://github.com/microsoft/component-detection/) and run it via a wrapper script such as: ```bash -npm run start -- \ - --sync-sboms --org my-org --sbom-cache sboms \ - --base-url https://ghe.example.com/api/v3 \ - --branch-scan --submit-on-missing-snapshot \ - --submit-languages Python \ - --component-detection-bin /opt/tools/component-detection +#!/bin/bash + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +cd "$SCRIPT_DIR" || exit 1 + +dotnet run --project "./src/Microsoft.ComponentDetection/Microsoft.ComponentDetection.csproj" "$@" ``` Notes: @@ -214,7 +202,6 @@ Malware-only advisory sync (no SBOM cache required): ```bash npm run start -- --sync-malware --malware-cache malware-cache --token $GITHUB_TOKEN ``` -``` Write malware matches (and optionally search results later) to a JSON file using `--output-file`: From 7f13c04349879db3dbba9fbb870bbc7b71cf9cab Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:43:10 +0000 Subject: [PATCH 027/108] Update GHES URL for listing orgs --- src/cli.ts | 2 ++ src/sbomCollector.ts | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index b952387..ce88f19 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -18,6 +18,7 @@ async function main() { .option("org", { type: "string", describe: "Single organization login" }) .option("repo", { type: "string", describe: "Single repository name" }) .option("base-url", { type: "string", describe: "GitHub Enterprise Server base URL, e.g. https://github.mycompany.com/api/v3" }) + .option("ghes", { type: "boolean", default: false, describe: "Indicates that the provided base URL is for GitHub Enterprise Server" }) .option("concurrency", { type: "number", default: 5 }) .option("sbom-delay", { type: "number", default: 3000, describe: "Delay (ms) between SBOM fetch requests" }) .option("light-delay", { type: "number", default: 100, describe: "Delay (ms) between lightweight metadata requests (org/repo listing, commit head checks)" }) @@ -122,6 +123,7 @@ async function main() { org: argv.org as string | undefined, repo: argv.repo as string | undefined, baseUrl: argv["base-url"] as string | undefined, + ghes: argv.ghes as boolean | undefined, concurrency: argv.concurrency as number, delayMsBetweenRepos: argv["sbom-delay"] as number, lightDelayMs: argv["light-delay"] as number, diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index f02c325..449589d 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -14,7 +14,8 @@ export interface CollectorOptions { enterprise?: string; // Enterprise slug to enumerate orgs org?: string; // Single org alternative repo?: string; // Single repo alternative - baseUrl?: string; // For GHES + baseUrl?: string; // For GHES, EMU and Data Residency + ghes?: boolean; // Is this a GHES instance? concurrency?: number; // parallel repo SBOM fetches includePrivate?: boolean; delayMsBetweenRepos?: number; @@ -53,6 +54,7 @@ export class SbomCollector { this.opts = { token: o.token, enterprise: o.enterprise, + ghes: o.ghes ?? false, org: o.org, repo: o.repo, baseUrl: o.baseUrl, @@ -149,7 +151,7 @@ export class SbomCollector { process.stderr.write(chalk.blue(`Getting list of organizations for enterprise ${this.opts.enterprise}`) + "\n"); } - const orgs = this.opts.org ? [this.opts.org] : this.opts.enterprise ? await this.listEnterpriseOrgs(this.opts.enterprise!) : [this.opts.repo.split("/")[0]]; + const orgs = this.opts.org ? [this.opts.org] : this.opts.enterprise ? await this.listEnterpriseOrgs(this.opts.enterprise, this.opts.ghes) : [this.opts.repo.split("/")[0]]; this.summary.orgs = orgs; // Pre-list all repos if showing progress bar so we know the total upfront @@ -368,7 +370,7 @@ export class SbomCollector { return false; } - private async listEnterpriseOrgs(enterprise: string): Promise { + private async listEnterpriseOrgs(enterprise: string, ghes: boolean): Promise { if (!this.octokit) throw new Error("No Octokit instance"); interface Org { login: string } try { @@ -377,7 +379,7 @@ export class SbomCollector { let page = 1; let done = false; while (!done) { - const resp = await this.octokit.request("GET /enterprises/{enterprise}/orgs", { enterprise, per_page, page }); + const resp = await this.octokit.request(ghes ? "GET /orgs" : "GET /enterprises/{enterprise}/orgs", { enterprise, per_page, page }); const items = resp.data as unknown as Org[]; for (const o of items) orgs.push(o.login); if (items.length < per_page) done = true; else page++; From dab9c611600ada16b188e12b41557f84b5d7e953 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:54:24 +0000 Subject: [PATCH 028/108] CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05384e7..af7bc12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## [2025-12-04] – 0.2.0 - Branch scanning and dependency submission +Added: + - Branch scanning: - Fetch SBOM diffs for non‑default branches via Dependency Review API. - Added `--branch-scan`, `--branch-limit`, and `--diff-base` CLI flags. @@ -21,6 +23,10 @@ - Advisory sync robustness: - GraphQL advisory sync now implements adaptive retries with exponential backoff and `Retry-After` support; respects `--quiet`. +Fixed: + +- Added `--ghes` flag to ensure proper API URL construction for GitHub Enterprise Server instances. + ## [2025-10-06] - 0.1.0 - Initial public release - Initial release, with: SBOM sync; malware sync; malware matching; CLI, file based and interactive PURL searching. SARIF, CSV and JSON outputs supported. From 12b53904496d48f8cc96e9b3e1082fff5b0db6dc Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:57:46 +0000 Subject: [PATCH 029/108] Potential fix for pull request finding 'Unused variable, import, function or class' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- src/componentSubmission.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index 14046bb..d4dad94 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -1,5 +1,5 @@ import chalk from 'chalk'; -import { spawn, execFile } from 'child_process'; +import { execFile } from 'child_process'; import path from 'path'; import fs from 'fs'; import os from 'os'; From 3b35d1cd73211dc36c0bdb7d0c7669485f52863c Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:00:24 +0000 Subject: [PATCH 030/108] Potential fix for pull request finding 'Semicolon insertion' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- src/componentDetection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index ad0d746..56cd33e 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -158,7 +158,7 @@ export default class ComponentDetection { } const referrerUrl = ComponentDetection.makePackageUrl(referrer.packageUrl); - referrer.packageUrlString = referrerUrl + referrer.packageUrlString = referrerUrl; // Skip if the generated packageUrl is empty if (!referrerUrl) { From f3135f5b1a574d5ca44fd28b00ac6861a141ef73 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:08:52 +0000 Subject: [PATCH 031/108] Improved property assignment from untrusted value --- src/componentDetection.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 56cd33e..865a049 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -331,11 +331,25 @@ export default class ComponentDetection { ): DependencyGraphs { // Resolve the base directory from filePathInput (relative to cwd if not absolute) const baseDir = path.resolve(process.cwd(), filePathInput); - const normalized: DependencyGraphs = {}; + // Use a null-prototype object to avoid prototype pollution + const normalized: DependencyGraphs = Object.create(null); for (const absPath in dependencyGraphs) { + // Only process own properties + if (!Object.prototype.hasOwnProperty.call(dependencyGraphs, absPath)) continue; // Make the path relative to the baseDir let relPath = path.relative(baseDir, absPath).replace(/\\/g, '/'); - normalized[relPath] = dependencyGraphs[absPath]; + // Guard against special keys that could lead to prototype injection + if (relPath === '__proto__' || relPath === 'constructor' || relPath === 'prototype') { + console.warn(`Skipping unsafe manifest key: ${relPath}`); + continue; + } + // Define property safely + Object.defineProperty(normalized, relPath, { + value: dependencyGraphs[absPath], + enumerable: true, + configurable: false, + writable: false, + }); } return normalized; } From 5cea67b7ab5f9a7aeed4f266494184ead79b2add Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:10:17 +0000 Subject: [PATCH 032/108] Update src/componentDetection.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentDetection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 865a049..5ff4d4c 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -300,7 +300,7 @@ export default class ComponentDetection { try { const latestRelease = await octokit.request("GET /repos/{owner}/{repo}/releases/latest", { owner, repo }); - var downloadURL: string = ""; + let downloadURL: string = ""; // TODO: do we need to handle different architectures here? // can we allow x64 on MacOS? We could allow an input parameter to override? const assetName = process.platform === "win32" ? "component-detection-win-x64.exe" : process.platform === "linux" ? "component-detection-linux-x64" : "component-detection-osx-arm64"; From 9364ebc580ffa9437a4aa8abdcd04a048626892b Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:11:57 +0000 Subject: [PATCH 033/108] Update src/cli.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli.ts b/src/cli.ts index ce88f19..cc6ffef 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -58,7 +58,7 @@ async function main() { if (!args.enterprise && !args.org && !args.repo) throw new Error("Provide --enterprise, --org or --repo with --sync-sboms"); if (args.enterprise && args.org) throw new Error("Specify only one of --enterprise or --org"); if (args.repo && (args.enterprise || args.org)) throw new Error("Specify only one of --enterprise, --org, or --repo"); - if (!args.sbomCache) throw new Error("--sync-sboms requires --sbom-cache to write updated SBOMs to disk"); + if (!args.sbomCache) throw new Error("--sync-sboms requires --sbom-cache to write updated SBOMs to disk"); } else { const malwareOnly = !!args["sync-malware"] && !args.sbomCache && !args.purl && !args["purl-file"] && !args["match-malware"] && !args.uploadSarif && !args.interactive; if (!malwareOnly && !args.sbomCache) throw new Error("Offline mode requires --sbom-cache unless running --sync-malware by itself"); From afca11d3ef6c58a7485df9c784a8d8c4255a7d2c Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:12:13 +0000 Subject: [PATCH 034/108] Update src/test-branch-search.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/test-branch-search.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/test-branch-search.ts b/src/test-branch-search.ts index 19ffd06..726f3ec 100644 --- a/src/test-branch-search.ts +++ b/src/test-branch-search.ts @@ -19,10 +19,6 @@ async function main() { { name: 'chalk', version: '5.6.1', purl: 'pkg:npm/chalk@5.6.1' }, { name: 'react', version: '18.2.0', purl: 'pkg:npm/react@18.2.0' } ]; - const featurePackages = [ - { name: 'react', version: '18.3.0-beta', purl: 'pkg:npm/react@18.3.0-beta' }, - { name: 'lodash', version: '4.17.21', purl: 'pkg:npm/lodash@4.17.21' } - ]; const diffChanges = [ { changeType: 'added', name: 'lodash', ecosystem: 'npm', purl: 'pkg:npm/lodash@4.17.21', newVersion: '4.17.21' }, { changeType: 'updated', name: 'react', ecosystem: 'npm', purl: 'pkg:npm/react@18.3.0-beta', previousVersion: '18.2.0', newVersion: '18.3.0-beta' } From 5d18ee560ab7d116634a7a0c1d23b3b2308c3ade Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:12:49 +0000 Subject: [PATCH 035/108] Update src/componentDetection.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentDetection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 5ff4d4c..973ea13 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -105,7 +105,7 @@ export default class ComponentDetection { public static async getManifestsFromResults(file: string, path: string): Promise { console.debug(`Reading results from ${file}`); const results = await fs.readFileSync(file, 'utf8'); - var json: any = JSON.parse(results); + const json: any = JSON.parse(results); let dependencyGraphs: DependencyGraphs = this.normalizeDependencyGraphPaths(json.dependencyGraphs, path); From b4fa0467fe7fe3a10f95f22e2add6c198a9a5d1a Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:13:37 +0000 Subject: [PATCH 036/108] Update src/types.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.ts b/src/types.ts index cdb5a3a..62f6b19 100644 --- a/src/types.ts +++ b/src/types.ts @@ -123,7 +123,7 @@ export interface DependencyReviewPackageChange { } export interface BranchDependencyDiff { - latestCommitDate: any; + latestCommitDate: string; base: string; // base branch head: string; // head branch retrievedAt: string; From fc20a4c9fe5395edd841229694c2f7a8d9643182 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:18:13 +0000 Subject: [PATCH 037/108] Update src/componentSubmission.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentSubmission.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index d4dad94..a873229 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -83,7 +83,7 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise Date: Thu, 4 Dec 2025 18:21:06 +0000 Subject: [PATCH 038/108] Initial plan From 8554a0a12f5a448c01fb3009671fe3b54f880f18 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:26:30 +0000 Subject: [PATCH 039/108] Fix some issues --- package-lock.json | 150 +----------------- src/malwareMatcher.ts | 6 +- src/sbomCollector.ts | 2 +- src/test-branch-search.ts | 4 +- src/types.ts | 3 +- .../example-org/demo-repo/sbom.json | 11 +- 6 files changed, 15 insertions(+), 161 deletions(-) diff --git a/package-lock.json b/package-lock.json index c3b3ec8..7207bd6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "github-sbom-toolkit", - "version": "0.2.0", + "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "github-sbom-toolkit", - "version": "0.2.0", + "version": "0.1.0", "dependencies": { "@github/dependency-submission-toolkit": "^2.0.5", "@octokit/core": "^7.0.6", @@ -1385,12 +1385,6 @@ "node": ">= 20" } }, - "node_modules/@octokit/app/node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, "node_modules/@octokit/app/node_modules/@octokit/plugin-paginate-rest": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-14.0.0.tgz", @@ -1406,15 +1400,6 @@ "@octokit/core": ">=6" } }, - "node_modules/@octokit/app/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/@octokit/auth-app": { "version": "8.1.2", "resolved": "https://registry.npmjs.org/@octokit/auth-app/-/auth-app-8.1.2.tgz", @@ -1434,21 +1419,6 @@ "node": ">= 20" } }, - "node_modules/@octokit/auth-app/node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, - "node_modules/@octokit/auth-app/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/@octokit/auth-oauth-app": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-app/-/auth-oauth-app-9.0.3.tgz", @@ -1465,21 +1435,6 @@ "node": ">= 20" } }, - "node_modules/@octokit/auth-oauth-app/node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, - "node_modules/@octokit/auth-oauth-app/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/@octokit/auth-oauth-device": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-device/-/auth-oauth-device-8.0.3.tgz", @@ -1495,21 +1450,6 @@ "node": ">= 20" } }, - "node_modules/@octokit/auth-oauth-device/node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, - "node_modules/@octokit/auth-oauth-device/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/@octokit/auth-oauth-user": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-user/-/auth-oauth-user-6.0.2.tgz", @@ -1526,21 +1466,6 @@ "node": ">= 20" } }, - "node_modules/@octokit/auth-oauth-user/node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, - "node_modules/@octokit/auth-oauth-user/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/@octokit/auth-token": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-6.0.0.tgz", @@ -1563,21 +1488,6 @@ "node": ">= 20" } }, - "node_modules/@octokit/auth-unauthenticated/node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, - "node_modules/@octokit/auth-unauthenticated/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/@octokit/core": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/@octokit/core/-/core-7.0.6.tgz", @@ -1624,21 +1534,6 @@ "node": ">= 20" } }, - "node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, - "node_modules/@octokit/graphql/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/@octokit/oauth-app": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/@octokit/oauth-app/-/oauth-app-8.0.3.tgz", @@ -1682,27 +1577,12 @@ "node": ">= 20" } }, - "node_modules/@octokit/oauth-methods/node_modules/@octokit/openapi-types": { + "node_modules/@octokit/openapi-types": { "version": "27.0.0", "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", "license": "MIT" }, - "node_modules/@octokit/oauth-methods/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, - "node_modules/@octokit/openapi-types": { - "version": "26.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-26.0.0.tgz", - "integrity": "sha512-7AtcfKtpo77j7Ts73b4OWhOZHTKo/gGY8bB3bNBQz4H+GRSWqx2yvj8TXRsbdTE0eRmYmXOEY66jM7mJ7LzfsA==", - "license": "MIT" - }, "node_modules/@octokit/openapi-webhooks-types": { "version": "12.0.3", "resolved": "https://registry.npmjs.org/@octokit/openapi-webhooks-types/-/openapi-webhooks-types-12.0.3.tgz", @@ -1821,15 +1701,6 @@ "@octokit/openapi-types": "^27.0.0" } }, - "node_modules/@octokit/types": { - "version": "15.0.2", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-15.0.2.tgz", - "integrity": "sha512-rR+5VRjhYSer7sC51krfCctQhVTmjyUMAaShfPB8mscVa8tSoLyon3coxQmXu0ahJoLVWl8dSGD/3OGZlFV44Q==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^26.0.0" - } - }, "node_modules/@octokit/webhooks": { "version": "14.1.3", "resolved": "https://registry.npmjs.org/@octokit/webhooks/-/webhooks-14.1.3.tgz", @@ -3646,12 +3517,6 @@ "node": ">= 20" } }, - "node_modules/octokit/node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, "node_modules/octokit/node_modules/@octokit/plugin-paginate-rest": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-14.0.0.tgz", @@ -3682,15 +3547,6 @@ "@octokit/core": ">=6" } }, - "node_modules/octokit/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", diff --git a/src/malwareMatcher.ts b/src/malwareMatcher.ts index 95cf3c6..103c81d 100644 --- a/src/malwareMatcher.ts +++ b/src/malwareMatcher.ts @@ -157,12 +157,12 @@ export function matchMalware(advisories: MalwareAdvisoryNode[], sboms: Repositor if (change.changeType !== 'added' && change.changeType !== 'updated') continue; let p: string | undefined = (change as { purl?: string }).purl; if (!p && change.packageURL && change.packageURL.startsWith('pkg:')) p = change.packageURL; - if (!p && change.ecosystem && change.name && change.newVersion) { + if (!p && change.ecosystem && change.name && change.version) { // Dependency review ecosystems are lower-case purl types already (e.g. npm, maven, pip, gem) - p = `pkg:${change.ecosystem}/${change.name}${change.newVersion ? '@' + change.newVersion : ''}`; + p = `pkg:${change.ecosystem}/${change.name}${change.version ? '@' + change.version : ''}`; } if (!p) continue; - out.push({ purl: p, name: change.name, ecosystem: change.ecosystem, version: change.newVersion, __branch: branchName }); + out.push({ purl: p, name: change.name, ecosystem: change.ecosystem, version: change.version, __branch: branchName }); } } return out; diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 449589d..23ceceb 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -627,7 +627,7 @@ export class SbomCollector { const candidatePurls: string[] = []; if ((change as { purl?: string }).purl) candidatePurls.push((change as { purl?: string }).purl as string); if (change.packageURL) candidatePurls.push(change.packageURL); - applyQueries(candidatePurls, queries, found, diff.head, (change as any).newVersion); + applyQueries(candidatePurls, queries, found, diff.head, (change as any).version); } } } diff --git a/src/test-branch-search.ts b/src/test-branch-search.ts index 19ffd06..3d09c1b 100644 --- a/src/test-branch-search.ts +++ b/src/test-branch-search.ts @@ -24,8 +24,8 @@ async function main() { { name: 'lodash', version: '4.17.21', purl: 'pkg:npm/lodash@4.17.21' } ]; const diffChanges = [ - { changeType: 'added', name: 'lodash', ecosystem: 'npm', purl: 'pkg:npm/lodash@4.17.21', newVersion: '4.17.21' }, - { changeType: 'updated', name: 'react', ecosystem: 'npm', purl: 'pkg:npm/react@18.3.0-beta', previousVersion: '18.2.0', newVersion: '18.3.0-beta' } + { changeType: 'added', name: 'lodash', ecosystem: 'npm', purl: 'pkg:npm/lodash@4.17.21', version: '4.17.21' }, + { changeType: 'updated', name: 'react', ecosystem: 'npm', purl: 'pkg:npm/react@18.3.0-beta', version: '18.2.0', newVersion: '18.3.0-beta' } ]; const synthetic: RepositorySbom = { diff --git a/src/types.ts b/src/types.ts index cdb5a3a..ab74791 100644 --- a/src/types.ts +++ b/src/types.ts @@ -117,8 +117,7 @@ export interface DependencyReviewPackageChange { license?: string; manifest?: string; // manifest path scope?: string; // e.g. runtime, development - previousVersion?: string; // for updated/removed - newVersion?: string; // for added/updated + version?: string; // for added/removed [k: string]: unknown; } diff --git a/tmp-branch-search-cache/example-org/demo-repo/sbom.json b/tmp-branch-search-cache/example-org/demo-repo/sbom.json index 98fedf1..b0927f4 100644 --- a/tmp-branch-search-cache/example-org/demo-repo/sbom.json +++ b/tmp-branch-search-cache/example-org/demo-repo/sbom.json @@ -25,15 +25,14 @@ "name": "lodash", "ecosystem": "npm", "purl": "pkg:npm/lodash@4.17.21", - "newVersion": "4.17.21" + "version": "4.17.21" }, { - "changeType": "updated", - "name": "react", + "changeType": "removed", + "name": "lodash", "ecosystem": "npm", - "purl": "pkg:npm/react@18.3.0-beta", - "previousVersion": "18.2.0", - "newVersion": "18.3.0-beta" + "purl": "pkg:npm/lodash@3.0.0", + "version": "3.0.0" } ] } From 83fb83479ff5008fcbec29bd5a326f7732e35a43 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:28:28 +0000 Subject: [PATCH 040/108] Update src/componentDetection.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentDetection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 973ea13..97ae7d4 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -250,7 +250,7 @@ export default class ComponentDetection { } try { - var packageUrl = `${packageUrlJson.Scheme}:${packageUrlJson.Type}/`; + const packageUrl = `${packageUrlJson.Scheme}:${packageUrlJson.Type}/`; if (packageUrlJson.Namespace) { packageUrl += `${packageUrlJson.Namespace.replaceAll("@", "%40")}/`; } From ac8ea00863624ed911abba70f3efbf774eaf7db0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:29:12 +0000 Subject: [PATCH 041/108] Fix submitSnapshot to return boolean indicating success/failure Co-authored-by: aegilops <41705651+aegilops@users.noreply.github.com> --- package-lock.json | 158 +------------------------------------ src/componentSubmission.ts | 15 ++-- 2 files changed, 11 insertions(+), 162 deletions(-) diff --git a/package-lock.json b/package-lock.json index c3b3ec8..5b81466 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "github-sbom-toolkit", - "version": "0.2.0", + "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "github-sbom-toolkit", - "version": "0.2.0", + "version": "0.1.0", "dependencies": { "@github/dependency-submission-toolkit": "^2.0.5", "@octokit/core": "^7.0.6", @@ -91,7 +91,6 @@ "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.2.tgz", "integrity": "sha512-/g2d4sW9nUDJOMz3mabVQvOGhVa4e/BN/Um7yca9Bb2XTzPPnfTWHWQg+IsEYO7M3Vx+EXvaM/I2pJWIMun1bg==", "license": "MIT", - "peer": true, "dependencies": { "@octokit/auth-token": "^4.0.0", "@octokit/graphql": "^7.1.0", @@ -1385,12 +1384,6 @@ "node": ">= 20" } }, - "node_modules/@octokit/app/node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, "node_modules/@octokit/app/node_modules/@octokit/plugin-paginate-rest": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-14.0.0.tgz", @@ -1406,15 +1399,6 @@ "@octokit/core": ">=6" } }, - "node_modules/@octokit/app/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/@octokit/auth-app": { "version": "8.1.2", "resolved": "https://registry.npmjs.org/@octokit/auth-app/-/auth-app-8.1.2.tgz", @@ -1434,21 +1418,6 @@ "node": ">= 20" } }, - "node_modules/@octokit/auth-app/node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, - "node_modules/@octokit/auth-app/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/@octokit/auth-oauth-app": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-app/-/auth-oauth-app-9.0.3.tgz", @@ -1465,21 +1434,6 @@ "node": ">= 20" } }, - "node_modules/@octokit/auth-oauth-app/node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, - "node_modules/@octokit/auth-oauth-app/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/@octokit/auth-oauth-device": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-device/-/auth-oauth-device-8.0.3.tgz", @@ -1495,21 +1449,6 @@ "node": ">= 20" } }, - "node_modules/@octokit/auth-oauth-device/node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, - "node_modules/@octokit/auth-oauth-device/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/@octokit/auth-oauth-user": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@octokit/auth-oauth-user/-/auth-oauth-user-6.0.2.tgz", @@ -1526,21 +1465,6 @@ "node": ">= 20" } }, - "node_modules/@octokit/auth-oauth-user/node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, - "node_modules/@octokit/auth-oauth-user/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/@octokit/auth-token": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-6.0.0.tgz", @@ -1563,27 +1487,11 @@ "node": ">= 20" } }, - "node_modules/@octokit/auth-unauthenticated/node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, - "node_modules/@octokit/auth-unauthenticated/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/@octokit/core": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/@octokit/core/-/core-7.0.6.tgz", "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "license": "MIT", - "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -1624,21 +1532,6 @@ "node": ">= 20" } }, - "node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, - "node_modules/@octokit/graphql/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/@octokit/oauth-app": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/@octokit/oauth-app/-/oauth-app-8.0.3.tgz", @@ -1682,27 +1575,12 @@ "node": ">= 20" } }, - "node_modules/@octokit/oauth-methods/node_modules/@octokit/openapi-types": { + "node_modules/@octokit/openapi-types": { "version": "27.0.0", "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", "license": "MIT" }, - "node_modules/@octokit/oauth-methods/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, - "node_modules/@octokit/openapi-types": { - "version": "26.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-26.0.0.tgz", - "integrity": "sha512-7AtcfKtpo77j7Ts73b4OWhOZHTKo/gGY8bB3bNBQz4H+GRSWqx2yvj8TXRsbdTE0eRmYmXOEY66jM7mJ7LzfsA==", - "license": "MIT" - }, "node_modules/@octokit/openapi-webhooks-types": { "version": "12.0.3", "resolved": "https://registry.npmjs.org/@octokit/openapi-webhooks-types/-/openapi-webhooks-types-12.0.3.tgz", @@ -1821,15 +1699,6 @@ "@octokit/openapi-types": "^27.0.0" } }, - "node_modules/@octokit/types": { - "version": "15.0.2", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-15.0.2.tgz", - "integrity": "sha512-rR+5VRjhYSer7sC51krfCctQhVTmjyUMAaShfPB8mscVa8tSoLyon3coxQmXu0ahJoLVWl8dSGD/3OGZlFV44Q==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^26.0.0" - } - }, "node_modules/@octokit/webhooks": { "version": "14.1.3", "resolved": "https://registry.npmjs.org/@octokit/webhooks/-/webhooks-14.1.3.tgz", @@ -1896,7 +1765,6 @@ "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -1981,7 +1849,6 @@ "integrity": "sha512-jCzKdm/QK0Kg4V4IK/oMlRZlY+QOcdjv89U2NgKHZk1CYTj82/RVSx1mV/0gqCVMJ/DA+Zf/S4NBWNF8GQ+eqQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.48.0", "@typescript-eslint/types": "8.48.0", @@ -2458,7 +2325,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2852,7 +2718,6 @@ "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -3646,12 +3511,6 @@ "node": ">= 20" } }, - "node_modules/octokit/node_modules/@octokit/openapi-types": { - "version": "27.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", - "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", - "license": "MIT" - }, "node_modules/octokit/node_modules/@octokit/plugin-paginate-rest": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-14.0.0.tgz", @@ -3682,15 +3541,6 @@ "@octokit/core": ">=6" } }, - "node_modules/octokit/node_modules/@octokit/types": { - "version": "16.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", - "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", - "license": "MIT", - "dependencies": { - "@octokit/openapi-types": "^27.0.0" - } - }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -3823,7 +3673,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -4162,7 +4011,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index a873229..851a25a 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -76,14 +76,12 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise { let manifests = await ComponentDetection.scanAndGetManifests( tmpDir, @@ -176,7 +174,7 @@ export async function run(octokit: Octokit, tmpDir: string, owner: string, repo: snapshot.addManifest(manifest); }); - submitSnapshot(octokit, snapshot, { owner, repo }); + return await submitSnapshot(octokit, snapshot, { owner, repo }); } /** @@ -184,12 +182,13 @@ export async function run(octokit: Octokit, tmpDir: string, owner: string, repo: * * @param {Snapshot} snapshot * @param {Repo} repo + * @returns {Promise} true if submission was successful, false otherwise */ export async function submitSnapshot( octokit: Octokit, snapshot: Snapshot, repo: { owner: string; repo: string } -) { +): Promise { console.debug('Submitting snapshot...') console.debug(snapshot.prettyJSON()) @@ -211,10 +210,12 @@ export async function submitSnapshot( `Snapshot successfully created at ${response.data.created_at.toString()}` + ` with id ${response.data.id}` ) + return true } else { console.error( `Snapshot creation failed with result: "${result}: ${response.data.message}"` ) + return false } } catch (error) { if (error instanceof RequestError) { @@ -231,6 +232,6 @@ export async function submitSnapshot( console.error(error.message) if (error.stack) console.error(error.stack) } - throw new Error(`Failed to submit snapshot: ${error}`) + return false } } \ No newline at end of file From 15bd0e95797181896ff4fd7447fdfbbb7c109058 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:30:15 +0000 Subject: [PATCH 042/108] Initial plan From 988b8ef407d280b51bf527ac8e3c9f5ae006ef51 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:30:37 +0000 Subject: [PATCH 043/108] Add missing JSDoc parameter documentation for octokit Co-authored-by: aegilops <41705651+aegilops@users.noreply.github.com> --- src/componentSubmission.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index 851a25a..34e0621 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -180,8 +180,9 @@ export async function run(octokit: Octokit, tmpDir: string, owner: string, repo: /** * submitSnapshot submits a snapshot to the Dependency Submission API - vendored in from @github/dependency-submission-toolkit, to make it work at the CLI, vs in Actions. * - * @param {Snapshot} snapshot - * @param {Repo} repo + * @param {Octokit} octokit - The Octokit instance for GitHub API requests + * @param {Snapshot} snapshot - The dependency snapshot to submit + * @param {Repo} repo - The repository owner and name * @returns {Promise} true if submission was successful, false otherwise */ export async function submitSnapshot( From 1d0d38c566479b65f07ab0ced305cb117beab9d5 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:32:58 +0000 Subject: [PATCH 044/108] Fix numbering in MD --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4f35b73..55baf48 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ npm run start -- --sbom-cache sboms --purl-file queries.txt npm run start -- --sync-sboms --org my-org --sbom-cache sboms ``` -1. Later offline search (no API calls; uses previously written per‑repo JSON): +2. Later offline search (no API calls; uses previously written per‑repo JSON): ```bash npm run start -- --sbom-cache sboms --purl pkg:npm/react@18.2.0 @@ -442,7 +442,7 @@ npm install npm run build ``` -1. Run the test harness script: +2. Run the test harness script: ```bash node dist/test-fixture-match.js From e262cf55e854f65b4c4478286c5929bccb9b3f40 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:35:44 +0000 Subject: [PATCH 045/108] Update src/cli.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli.ts b/src/cli.ts index cc6ffef..ebaca1b 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -58,7 +58,7 @@ async function main() { if (!args.enterprise && !args.org && !args.repo) throw new Error("Provide --enterprise, --org or --repo with --sync-sboms"); if (args.enterprise && args.org) throw new Error("Specify only one of --enterprise or --org"); if (args.repo && (args.enterprise || args.org)) throw new Error("Specify only one of --enterprise, --org, or --repo"); - if (!args.sbomCache) throw new Error("--sync-sboms requires --sbom-cache to write updated SBOMs to disk"); + if (syncing && !args.sbomCache) throw new Error("--sync-sboms requires --sbom-cache to write updated SBOMs to disk"); } else { const malwareOnly = !!args["sync-malware"] && !args.sbomCache && !args.purl && !args["purl-file"] && !args["match-malware"] && !args.uploadSarif && !args.interactive; if (!malwareOnly && !args.sbomCache) throw new Error("Offline mode requires --sbom-cache unless running --sync-malware by itself"); From ebf5b451105bc2bbfd80337858b2278c2f107f90 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:37:16 +0000 Subject: [PATCH 046/108] Fixed const/let --- src/componentDetection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 97ae7d4..8fe0afe 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -250,7 +250,7 @@ export default class ComponentDetection { } try { - const packageUrl = `${packageUrlJson.Scheme}:${packageUrlJson.Type}/`; + let packageUrl = `${packageUrlJson.Scheme}:${packageUrlJson.Type}/`; if (packageUrlJson.Namespace) { packageUrl += `${packageUrlJson.Namespace.replaceAll("@", "%40")}/`; } From 410f95e019fe9a7fe12e2baea6f46bcb8d42516b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:37:46 +0000 Subject: [PATCH 047/108] Fix latestCommitDate to use actual commit date from API Co-authored-by: aegilops <41705651+aegilops@users.noreply.github.com> --- package-lock.json | 8 -------- src/sbomCollector.ts | 10 +++++----- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7207bd6..5b81466 100644 --- a/package-lock.json +++ b/package-lock.json @@ -91,7 +91,6 @@ "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.2.tgz", "integrity": "sha512-/g2d4sW9nUDJOMz3mabVQvOGhVa4e/BN/Um7yca9Bb2XTzPPnfTWHWQg+IsEYO7M3Vx+EXvaM/I2pJWIMun1bg==", "license": "MIT", - "peer": true, "dependencies": { "@octokit/auth-token": "^4.0.0", "@octokit/graphql": "^7.1.0", @@ -1493,7 +1492,6 @@ "resolved": "https://registry.npmjs.org/@octokit/core/-/core-7.0.6.tgz", "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "license": "MIT", - "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -1767,7 +1765,6 @@ "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -1852,7 +1849,6 @@ "integrity": "sha512-jCzKdm/QK0Kg4V4IK/oMlRZlY+QOcdjv89U2NgKHZk1CYTj82/RVSx1mV/0gqCVMJ/DA+Zf/S4NBWNF8GQ+eqQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.48.0", "@typescript-eslint/types": "8.48.0", @@ -2329,7 +2325,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2723,7 +2718,6 @@ "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -3679,7 +3673,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -4018,7 +4011,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 23ceceb..700742f 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -315,7 +315,7 @@ export class SbomCollector { console.error(chalk.red(`Force submission failed for ${fullName} branch ${b.name}: ${(subErr as Error).message}`)); } } - const diff = await this.fetchDependencyReviewDiff(org, repo.name, base, b.name); + const diff = await this.fetchDependencyReviewDiff(org, repo.name, base, b.name, latestCommit); branchDiffs.set(b.name, diff); } if (branchDiffs.size) sbom.branchDiffs = branchDiffs; @@ -483,7 +483,7 @@ export class SbomCollector { return branches; } - private async fetchDependencyReviewDiff(org: string, repo: string, base: string, head: string): Promise { + private async fetchDependencyReviewDiff(org: string, repo: string, base: string, head: string, latestCommit?: { sha?: string; commitDate?: string }): Promise { if (!this.octokit) throw new Error("No Octokit instance"); try { const basehead = `${base}...${head}`; @@ -506,7 +506,7 @@ export class SbomCollector { }; changes.push(change); } - return { latestCommitDate: new Date().toISOString(), base, head, retrievedAt: new Date().toISOString(), changes }; + return { latestCommitDate: latestCommit?.commitDate || new Date().toISOString(), base, head, retrievedAt: new Date().toISOString(), changes }; } catch (e) { const status = (e as { status?: number })?.status; let reason = e instanceof Error ? e.message : String(e); @@ -520,7 +520,7 @@ export class SbomCollector { if (ok) { console.log(chalk.blue(`Snapshot submission attempted; waiting 3 seconds before retrying dependency review diff for ${org}/${repo} ${base}...${head}...`)); await new Promise(r => setTimeout(r, 3000)); - return await this.fetchDependencyReviewDiff(org, repo, base, head); + return await this.fetchDependencyReviewDiff(org, repo, base, head, latestCommit); } } catch (subErr) { console.error(chalk.red(`Snapshot submission failed for ${org}/${repo} branch ${head}: ${(subErr as Error).message}`)); @@ -528,7 +528,7 @@ export class SbomCollector { } } } - return { latestCommitDate: new Date().toISOString(), base, head, retrievedAt: new Date().toISOString(), changes: [], error: reason }; + return { latestCommitDate: latestCommit?.commitDate || new Date().toISOString(), base, head, retrievedAt: new Date().toISOString(), changes: [], error: reason }; } } From a7a725bcbdb0410fc64e499c1239342bfbf4d743 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:39:54 +0000 Subject: [PATCH 048/108] Removed newVersion --- src/test-branch-search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test-branch-search.ts b/src/test-branch-search.ts index aa1f078..f004bc1 100644 --- a/src/test-branch-search.ts +++ b/src/test-branch-search.ts @@ -21,7 +21,7 @@ async function main() { ]; const diffChanges = [ { changeType: 'added', name: 'lodash', ecosystem: 'npm', purl: 'pkg:npm/lodash@4.17.21', version: '4.17.21' }, - { changeType: 'updated', name: 'react', ecosystem: 'npm', purl: 'pkg:npm/react@18.3.0-beta', version: '18.2.0', newVersion: '18.3.0-beta' } + { changeType: 'updated', name: 'react', ecosystem: 'npm', purl: 'pkg:npm/react@18.3.0-beta', version: '18.2.0' } ]; const synthetic: RepositorySbom = { From 5544df590cfa60699d034a4a9cb246dce46add59 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Thu, 4 Dec 2025 18:41:40 +0000 Subject: [PATCH 049/108] Update src/sbomCollector.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/sbomCollector.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 23ceceb..9a539a1 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -528,7 +528,7 @@ export class SbomCollector { } } } - return { latestCommitDate: new Date().toISOString(), base, head, retrievedAt: new Date().toISOString(), changes: [], error: reason }; + return { latestCommitDate: undefined, base, head, retrievedAt: new Date().toISOString(), changes: [], error: reason }; } } From 6c6c8b95138e75ef74ff4f73f69218188c4df250 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 09:21:06 +0000 Subject: [PATCH 050/108] Update src/componentDetection.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentDetection.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 8fe0afe..19eb2e1 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -13,7 +13,10 @@ import { StringDecoder } from 'node:string_decoder'; export default class ComponentDetection { public static componentDetectionPath = process.platform === "win32" ? './component-detection.exe' : './component-detection'; - public static outputPath = path.join(tmpdir(), `component-detection-output-${Date.now()}.json`); + public static outputPath = (() => { + const tmpDir = fs.mkdtempSync(path.join(tmpdir(), 'component-detection-')); + return path.join(tmpDir, 'output.json'); + })(); // This is the default entry point for this class. // If executablePath is provided, use it directly and skip download. From a001a19d55ad781b7cf0b0bc2902d6098962e549 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 09:25:14 +0000 Subject: [PATCH 051/108] Update src/componentSubmission.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentSubmission.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index a873229..b445deb 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -77,13 +77,11 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise Date: Fri, 5 Dec 2025 09:25:52 +0000 Subject: [PATCH 052/108] Update package.json Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5b244da..6f44367 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "@octokit/plugin-throttling": "^11.0.3", "chalk": "^5.6.2", "cross-fetch": "^4.1.0", - "inquirer": "^12.11.0", + "inquirer": "^12.11.1", "octokit": "^5.0.5", "p-limit": "^7.2.0", "packageurl-js": "^2.0.1", From 3c40bc23d2b937fb6b0e73d17ef484f305a4da7a Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 09:44:49 +0000 Subject: [PATCH 053/108] Allow undefined commit date --- src/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.ts b/src/types.ts index 0bed464..cb26664 100644 --- a/src/types.ts +++ b/src/types.ts @@ -122,7 +122,7 @@ export interface DependencyReviewPackageChange { } export interface BranchDependencyDiff { - latestCommitDate: string; + latestCommitDate?: string; base: string; // base branch head: string; // head branch retrievedAt: string; From 09cafe86a896ad12b4c0e475b46e1406c57ab08f Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 10:33:10 +0000 Subject: [PATCH 054/108] Fixed up Component Detection/Submission --- src/componentDetection.ts | 76 ++++++++++++++++++++------------------ src/componentSubmission.ts | 14 ++++--- 2 files changed, 50 insertions(+), 40 deletions(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 19eb2e1..76d7049 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -1,4 +1,4 @@ -import { Octokit } from "octokit" +import { Octokit } from "@octokit/core"; import { PackageCache, Package, @@ -11,19 +11,30 @@ import path from 'path'; import { tmpdir } from 'os'; import { StringDecoder } from 'node:string_decoder'; -export default class ComponentDetection { - public static componentDetectionPath = process.platform === "win32" ? './component-detection.exe' : './component-detection'; - public static outputPath = (() => { - const tmpDir = fs.mkdtempSync(path.join(tmpdir(), 'component-detection-')); - return path.join(tmpDir, 'output.json'); - })(); +export default class ComponentDetection { + public componentDetectionPath: string = process.platform === "win32" ? './component-detection.exe' : './component-detection'; + public outputPath: string; + octokit: Octokit; + baseUrl: string; - // This is the default entry point for this class. - // If executablePath is provided, use it directly and skip download. - static async scanAndGetManifests(path: string, executablePath?: string): Promise { + constructor(octokit: Octokit, baseUrl: string, executablePath?: string) { + this.octokit = octokit; + this.baseUrl = baseUrl; if (executablePath) { this.componentDetectionPath = executablePath; - } else { + } + + // Set the output path + this.outputPath = (() => { + const tmpDir = fs.mkdtempSync(path.join(tmpdir(), 'component-detection-')); + return path.join(tmpDir, 'output.json'); + })(); + } + + // This is the default entry point for this class. + // If executablePath is provided, use it directly and skip download. + async scanAndGetManifests(path: string): Promise { + if (!this.componentDetectionPath) { await this.downloadLatestRelease(); } @@ -37,7 +48,7 @@ export default class ComponentDetection { return await this.getManifestsFromResults(this.outputPath, path); } // Get the latest release from the component-detection repo, download the tarball, and extract it - public static async downloadLatestRelease() { + public async downloadLatestRelease() { try { const statResult = fs.statSync(this.componentDetectionPath); if (statResult && statResult.isFile()) { @@ -64,7 +75,7 @@ export default class ComponentDetection { } // Run the component-detection CLI on the path specified - public static runComponentDetection(path: string): Promise { + public runComponentDetection(path: string): Promise { console.debug(`Running component-detection on ${path}`); console.debug(`Writing to output file: ${this.outputPath}`); @@ -105,7 +116,7 @@ export default class ComponentDetection { }); } - public static async getManifestsFromResults(file: string, path: string): Promise { + public async getManifestsFromResults(file: string, path: string): Promise { console.debug(`Reading results from ${file}`); const results = await fs.readFileSync(file, 'utf8'); const json: any = JSON.parse(results); @@ -115,7 +126,7 @@ export default class ComponentDetection { return this.processComponentsToManifests(json.componentsFound, dependencyGraphs); } - public static processComponentsToManifests(componentsFound: any[], dependencyGraphs: DependencyGraphs): Manifest[] { + public processComponentsToManifests(componentsFound: any[], dependencyGraphs: DependencyGraphs): Manifest[] { // Parse the result file and add the packages to the package cache const packageCache = new PackageCache(); const packages: Array = []; @@ -196,7 +207,7 @@ export default class ComponentDetection { return manifests; } - private static addPackagesToManifests(packages: Array, manifests: Array, dependencyGraphs: DependencyGraphs): void { + private addPackagesToManifests(packages: Array, manifests: Array, dependencyGraphs: DependencyGraphs): void { packages.forEach((pkg: ComponentDetectionPackage) => { pkg.locationsFoundAt.forEach((location: any) => { // Use the normalized path (remove leading slash if present) @@ -277,28 +288,23 @@ export default class ComponentDetection { } } - private static async getLatestReleaseURL(): Promise { - let githubToken = process.env.GITHUB_TOKEN || ""; - - const githubAPIURL = process.env.GITHUB_API_URL || 'https://api.github.com'; - - let ghesMode = process.env.GITHUB_API_URL != githubAPIURL; - // If the we're running in GHES, then use an empty string as the token - if (ghesMode) { - githubToken = ""; + private async getLatestReleaseURL(): Promise { + let octokit: Octokit = this.octokit; + + if (this.baseUrl != 'https://api.github.com') { + octokit = new Octokit({ + auth: "", request: { fetch: fetch }, log: { + debug: console.debug, + info: console.info, + warn: console.warn, + error: console.error + }, + }); } - const octokit = new Octokit({ - auth: githubToken, baseUrl: githubAPIURL, request: { fetch: fetch }, log: { - debug: console.debug, - info: console.info, - warn: console.warn, - error: console.error - }, - }); const owner = "microsoft"; const repo = "component-detection"; - console.debug("Attempting to download latest release from " + githubAPIURL); + console.debug(`Attempting to download latest release from ${owner}/${repo}`); try { const latestRelease = await octokit.request("GET /repos/{owner}/{repo}/releases/latest", { owner, repo }); @@ -328,7 +334,7 @@ export default class ComponentDetection { * @param filePathInput The filePath input (relative or absolute) from the action configuration. * @returns A new DependencyGraphs object with relative path keys. */ - public static normalizeDependencyGraphPaths( + public normalizeDependencyGraphPaths( dependencyGraphs: DependencyGraphs, filePathInput: string ): DependencyGraphs { diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index b445deb..aeffe72 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -63,10 +63,12 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise Date: Fri, 5 Dec 2025 10:48:07 +0000 Subject: [PATCH 055/108] Fixed missing await --- src/componentSubmission.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index aeffe72..9a03bd1 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -178,7 +178,7 @@ export async function run(octokit: Octokit, tmpDir: string, owner: string, repo: snapshot.addManifest(manifest); }); - submitSnapshot(octokit, snapshot, { owner, repo }); + await submitSnapshot(octokit, snapshot, { owner, repo }); } /** From a67db56eeb5af60f5a0ec950fe213131ac0cc4aa Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 10:48:32 +0000 Subject: [PATCH 056/108] Updated packages --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7207bd6..1d51656 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@octokit/plugin-throttling": "^11.0.3", "chalk": "^5.6.2", "cross-fetch": "^4.1.0", - "inquirer": "^12.11.0", + "inquirer": "^12.11.1", "octokit": "^5.0.5", "p-limit": "^7.2.0", "packageurl-js": "^2.0.1", diff --git a/package.json b/package.json index 6f44367..c7dbd00 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "dev": "tsx src/cli.ts", "lint": "eslint . --ext .ts --max-warnings=0", "test": "node dist/test-fixture-match.js", - "test:branch-search": "node dist/test-branch-search.js" + "test:branch-search": "node dist/test-branch-search.js && node dist/test-branch-search-cache.js" }, "engines": { "node": ">=18.0.0" From 98dee3dd9f86e8bfe4a1256baa68775a2b06508a Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:12:50 +0000 Subject: [PATCH 057/108] Fixed test --- src/test-branch-search.ts | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/test-branch-search.ts b/src/test-branch-search.ts index f004bc1..059418c 100644 --- a/src/test-branch-search.ts +++ b/src/test-branch-search.ts @@ -21,18 +21,17 @@ async function main() { ]; const diffChanges = [ { changeType: 'added', name: 'lodash', ecosystem: 'npm', purl: 'pkg:npm/lodash@4.17.21', version: '4.17.21' }, - { changeType: 'updated', name: 'react', ecosystem: 'npm', purl: 'pkg:npm/react@18.3.0-beta', version: '18.2.0' } + { changeType: 'updated', name: 'react', ecosystem: 'npm', purl: 'pkg:npm/react@18.3.0', version: '18.3.0' }, + { changeType: 'removed', name: 'chalk', ecosystem: 'npm', purl: 'pkg:npm/chalk@5.6.1', version: '5.6.1' }, + { changeType: 'removed', name: 'react', ecosystem: 'npm', purl: 'pkg:npm/react@18.2.0', version: '18.2.0' } ]; - const synthetic: RepositorySbom = { + const synthetic = { repo: `${org}/${repo}`, - org, + org: org, retrievedAt: new Date().toISOString(), packages: basePackages, - // Use Map keyed by branch name per updated type - branchDiffs: new Map([ - [ - 'feature-x', + branchDiffs: [ { latestCommitDate: new Date().toISOString(), base: 'main', @@ -41,8 +40,7 @@ async function main() { changes: diffChanges } ] - ]) - } as RepositorySbom; + }; fs.writeFileSync(path.join(repoDir, 'sbom.json'), JSON.stringify(synthetic, null, 2), 'utf8'); @@ -57,9 +55,12 @@ async function main() { const queries = [ 'pkg:npm/react@>=18.2.0 <19.0.0', // should match base & branch updated version - 'pkg:npm/lodash@4.17.21', // should match added in branch diff & branch SBOM + 'pkg:npm/lodash@4.17.21', // should match added in branch diff 'pkg:npm/chalk@5.6.1' // base only ]; + + console.debug(JSON.stringify(collector.getAllSboms())); + const results = collector.searchByPurlsWithReasons(queries); if (!results.size) { From bc125d7c687b166ee68766f08762ef19ea4cd123 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:13:27 +0000 Subject: [PATCH 058/108] Updated test running --- package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index c7dbd00..96d51fc 100644 --- a/package.json +++ b/package.json @@ -12,8 +12,7 @@ "start": "node dist/cli.js", "dev": "tsx src/cli.ts", "lint": "eslint . --ext .ts --max-warnings=0", - "test": "node dist/test-fixture-match.js", - "test:branch-search": "node dist/test-branch-search.js && node dist/test-branch-search-cache.js" + "test": "node dist/test-fixture-match.js && node dist/test-branch-search.js" }, "engines": { "node": ">=18.0.0" From 8b51d9aa0716438077f4fbc494fe04676d5fa145 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:14:33 +0000 Subject: [PATCH 059/108] Remove test file and put in ignore file --- .gitignore | 3 +- .../example-org/demo-repo/sbom.json | 40 ------------------- 2 files changed, 2 insertions(+), 41 deletions(-) delete mode 100644 tmp-branch-search-cache/example-org/demo-repo/sbom.json diff --git a/.gitignore b/.gitignore index fc00982..2924918 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ dist/ data/ .vscode/ .DS_Store -component-detection \ No newline at end of file +component-detection +tmp-branch-search-cache/ \ No newline at end of file diff --git a/tmp-branch-search-cache/example-org/demo-repo/sbom.json b/tmp-branch-search-cache/example-org/demo-repo/sbom.json deleted file mode 100644 index b0927f4..0000000 --- a/tmp-branch-search-cache/example-org/demo-repo/sbom.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "repo": "example-org/demo-repo", - "org": "example-org", - "retrievedAt": "2025-11-26T15:02:48.754Z", - "packages": [ - { - "name": "chalk", - "version": "5.6.1", - "purl": "pkg:npm/chalk@5.6.1" - }, - { - "name": "react", - "version": "18.2.0", - "purl": "pkg:npm/react@18.2.0" - } - ], - "branchDiffs": [ - { - "base": "main", - "head": "feature-x", - "retrievedAt": "2025-11-26T15:02:48.755Z", - "changes": [ - { - "changeType": "added", - "name": "lodash", - "ecosystem": "npm", - "purl": "pkg:npm/lodash@4.17.21", - "version": "4.17.21" - }, - { - "changeType": "removed", - "name": "lodash", - "ecosystem": "npm", - "purl": "pkg:npm/lodash@3.0.0", - "version": "3.0.0" - } - ] - } - ] -} \ No newline at end of file From 95896ab8932b2e283f0d6a5508d2d125c4e4c29f Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:16:07 +0000 Subject: [PATCH 060/108] Update src/componentSubmission.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentSubmission.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index 9a03bd1..97708a3 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -127,8 +127,10 @@ function buildSparsePatterns(langs: string[]): string[] { add('**/*.sln'); } } - // Always include root lockfiles just in case - add('package.json'); add('package-lock.json'); add('yarn.lock'); add('pnpm-lock.yaml'); + // Include root lockfiles only if JavaScript/TypeScript is among selected languages + if (langs.some(l => ['javascript', 'typescript', 'node', 'js', 'ts'].includes(l.toLowerCase()))) { + add('package.json'); add('package-lock.json'); add('yarn.lock'); add('pnpm-lock.yaml'); + } return Array.from(set); } From 95a7fd0de798d9477bf5042972d335671411f81c Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:16:49 +0000 Subject: [PATCH 061/108] Update src/componentSubmission.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentSubmission.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index 97708a3..8d7486a 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -202,7 +202,7 @@ export async function submitSnapshot( 'POST /repos/{owner}/{repo}/dependency-graph/snapshots', { headers: { - accept: 'application/vnd.github.foo-bar-preview+json' + accept: 'application/vnd.github+json' }, owner: repo.owner, repo: repo.repo, From 93d441c560c2feb0c0396367e623482881696ac7 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:20:29 +0000 Subject: [PATCH 062/108] Update src/componentDetection.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentDetection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 76d7049..8020110 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -291,7 +291,7 @@ export default class ComponentDetection { private async getLatestReleaseURL(): Promise { let octokit: Octokit = this.octokit; - if (this.baseUrl != 'https://api.github.com') { + if (this.baseUrl !== 'https://api.github.com') { octokit = new Octokit({ auth: "", request: { fetch: fetch }, log: { debug: console.debug, From bd656c66d93b5d677292f1f8b1ea14249c065c32 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:20:54 +0000 Subject: [PATCH 063/108] Update README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 55baf48..87e435a 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,6 @@ Flags: ```bash --branch-scan # Fetch SBOMs for non-default branches --branch-limit # Max number of non-default branches per repo (default 10) ---dependency-review # Fetch dependency review diffs (enabled by default) --diff-base # Override base branch for diffs (default: repository default) ``` From 51bde4152923aaebb5a92822499a1a4f167ed2db Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:22:11 +0000 Subject: [PATCH 064/108] Update src/sbomCollector.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/sbomCollector.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 9a539a1..b7bc6d2 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -48,6 +48,16 @@ export class SbomCollector { if (!options.loadFromDir && !options.enterprise && !options.org && !options.repo) { throw new Error("One of enterprise/org/repo or loadFromDir must be specified"); } + // Validate repo format if provided + if (options.repo) { + if (typeof options.repo !== "string" || !options.repo.includes("/")) { + throw new Error('If specifying "repo", it must be in the format "org/repo".'); + } + const [orgPart, repoPart] = options.repo.split("/"); + if (!orgPart || !repoPart) { + throw new Error('If specifying "repo", it must be in the format "org/repo" with both parts non-empty.'); + } + } // Spread user options first then apply defaults via nullish coalescing so that // passing undefined does not erase defaults const o = { ...options }; From f909bdce92477e68203ef0865efbf97be97d5312 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:24:47 +0000 Subject: [PATCH 065/108] Update src/componentSubmission.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentSubmission.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index 8d7486a..8d3fca6 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -25,7 +25,7 @@ export interface SubmitOpts { componentDetectionBinPath?: string; // optional path to component-detection executable } -export async function getLanguageIntersection(octokit: any, owner: string, repo: string, languages: string[] | undefined, quiet: boolean = false): Promise { +export async function getLanguageIntersection(octokit: Octokit, owner: string, repo: string, languages: string[] | undefined, quiet: boolean = false): Promise { const langResp = await octokit.request('GET /repos/{owner}/{repo}/languages', { owner, repo }); const repoLangs = Object.keys(langResp.data || {}); const wanted = languages; From f3673afa9eb3e3b52239654246cd87da34709a44 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:28:01 +0000 Subject: [PATCH 066/108] Update src/componentDetection.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentDetection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 8020110..ecf6e41 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -68,7 +68,7 @@ export default class ComponentDetection { // Write the blob to a file console.debug(`Writing binary to file ${this.componentDetectionPath}`); - await fs.writeFileSync(this.componentDetectionPath, buffer, { mode: 0o777, flag: 'w' }); + await fs.writeFileSync(this.componentDetectionPath, buffer, { mode: 0o755, flag: 'w' }); } catch (error: any) { console.error(error); } From 938d9650bfacded02944e8a8e8a29e47dba49bca Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:30:08 +0000 Subject: [PATCH 067/108] Fix repo URL --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 87e435a..0da227c 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ If a branch SBOM or diff retrieval fails, the error is recorded but does not sto #### Handling Missing Dependency Review Snapshots -If the Dependency Review API returns a 404 for a branch diff (commonly due to a missing dependency snapshot on either the base or head commit), the toolkit can optionally attempt to generate and submit a snapshot using Component Detection and Dependency Submission. This is vendored-in and forked from the public [Component Detection Dependency Submission Action](https://github.com/your-org/component-detection-dependency-submission-action). +If the Dependency Review API returns a 404 for a branch diff (commonly due to a missing dependency snapshot on either the base or head commit), the toolkit can optionally attempt to generate and submit a snapshot using Component Detection and Dependency Submission. This is vendored-in and forked from the public [Component Detection Dependency Submission Action](https://github.com/advanced-security/component-detection-dependency-submission-action). Enable automatic submission + retry with: From 8e6e34122552498e9c414d323e4f90a0e1049832 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:30:56 +0000 Subject: [PATCH 068/108] Update src/componentSubmission.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentSubmission.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index 8d3fca6..f09a6c9 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -68,7 +68,10 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise Date: Fri, 5 Dec 2025 14:44:16 +0000 Subject: [PATCH 069/108] Made job id nearly impossible to clash --- src/componentSubmission.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index 8d3fca6..9e62c23 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -165,9 +165,11 @@ export async function run(octokit: Octokit, tmpDir: string, owner: string, repo: url: detectorUrl, }; + const date = new Date().toISOString(); + const job: Job = { correlator: 'github-sbom-toolkit', - id: Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString() + id: `${owner}-${repo}-${ref}-${date}-${Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString()}` }; let snapshot = new Snapshot(detector, undefined, job); From 5c8606bd108d544430d1bb8c846915316c01c514 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:52:52 +0000 Subject: [PATCH 070/108] Removed pointless cast to any --- src/sbomCollector.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index b7bc6d2..9e19ce7 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -637,7 +637,7 @@ export class SbomCollector { const candidatePurls: string[] = []; if ((change as { purl?: string }).purl) candidatePurls.push((change as { purl?: string }).purl as string); if (change.packageURL) candidatePurls.push(change.packageURL); - applyQueries(candidatePurls, queries, found, diff.head, (change as any).version); + applyQueries(candidatePurls, queries, found, diff.head, change.version); } } } From bdfe2f65b6b23bc14f34674eae41b9d30e2fad25 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:57:11 +0000 Subject: [PATCH 071/108] Move error count outside scope --- src/sbomCollector.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 9e19ce7..dbf3a75 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -310,9 +310,9 @@ export class SbomCollector { continue; } - if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); const base = this.opts.branchDiffBase || sbom?.defaultBranch; if (!base) { console.error(chalk.red(`Cannot compute branch diff for ${fullName} branch ${b.name} because base branch is undefined.`)); continue; } + if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); // Optionally perform dependency submission up front for the branch if (this.opts.forceSubmission) { @@ -334,7 +334,9 @@ export class SbomCollector { this.decisions[fullName] = (this.decisions[fullName] || "") + ` (branch scan error: ${(e as Error).message})`; console.debug((e as Error).message); } + if (sbom.error) this.summary.failedCount++; else this.summary.successCount++; + // Write freshly fetched SBOM immediately if a cache directory is configured if (this.opts.loadFromDir && this.opts.syncSboms && this.opts.loadFromDir.length) { try { writeOne(sbom, { outDir: this.opts.loadFromDir }); } catch { /* ignore write errors */ } From ca7d8b6551af6fd17d088a601ec32754acb116a6 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:00:39 +0000 Subject: [PATCH 072/108] Limit retries of diff --- src/sbomCollector.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index dbf3a75..d081e2b 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -325,7 +325,7 @@ export class SbomCollector { console.error(chalk.red(`Force submission failed for ${fullName} branch ${b.name}: ${(subErr as Error).message}`)); } } - const diff = await this.fetchDependencyReviewDiff(org, repo.name, base, b.name); + const diff = await this.fetchDependencyReviewDiff(org, repo.name, base, b.name, 1); branchDiffs.set(b.name, diff); } if (branchDiffs.size) sbom.branchDiffs = branchDiffs; @@ -495,7 +495,10 @@ export class SbomCollector { return branches; } - private async fetchDependencyReviewDiff(org: string, repo: string, base: string, head: string): Promise { + private async fetchDependencyReviewDiff(org: string, repo: string, base: string, head: string, retries: number): Promise { + if (retries <= 0) { + return { latestCommitDate: undefined, base, head, retrievedAt: new Date().toISOString(), changes: [], error: "Maximum retries exceeded" }; + } if (!this.octokit) throw new Error("No Octokit instance"); try { const basehead = `${base}...${head}`; @@ -532,7 +535,7 @@ export class SbomCollector { if (ok) { console.log(chalk.blue(`Snapshot submission attempted; waiting 3 seconds before retrying dependency review diff for ${org}/${repo} ${base}...${head}...`)); await new Promise(r => setTimeout(r, 3000)); - return await this.fetchDependencyReviewDiff(org, repo, base, head); + return await this.fetchDependencyReviewDiff(org, repo, base, head, retries--); } } catch (subErr) { console.error(chalk.red(`Snapshot submission failed for ${org}/${repo} branch ${head}: ${(subErr as Error).message}`)); From f2296bdf559c62e3152c681e9b5bbe2edcb15397 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:06:59 +0000 Subject: [PATCH 073/108] Initial plan From 22fde034e85df92fe668e6b2e2c89ca4ecaa4b25 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:15:27 +0000 Subject: [PATCH 074/108] Convert async forEach loops to for...of loops to fix race conditions Co-authored-by: aegilops <41705651+aegilops@users.noreply.github.com> --- src/componentDetection.ts | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index ecf6e41..8a83b58 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -131,7 +131,7 @@ export default class ComponentDetection { const packageCache = new PackageCache(); const packages: Array = []; - componentsFound.forEach(async (component: any) => { + for (const component of componentsFound) { // Skip components without packageUrl if (!component.component.packageUrl) { console.debug(`Skipping component detected without packageUrl: ${JSON.stringify({ @@ -139,7 +139,7 @@ export default class ComponentDetection { name: component.component.name || 'unnamed', type: component.component.type || 'unknown' }, null, 2)}`); - return; + continue; } console.debug(`Processing component: ${component.component.id}`); @@ -150,7 +150,7 @@ export default class ComponentDetection { // Skip if the packageUrl is empty (indicates an invalid or missing packageUrl) if (!packageUrl) { console.debug(`Skipping component with invalid packageUrl: ${component.component.id}`); - return; + continue; } if (!packageCache.hasPackage(packageUrl)) { @@ -159,16 +159,16 @@ export default class ComponentDetection { packageCache.addPackage(pkg); packages.push(pkg); } - }); + } // Set the transitive dependencies console.debug("Sorting out transitive dependencies"); - packages.forEach(async (pkg: ComponentDetectionPackage) => { - pkg.topLevelReferrers.forEach(async (referrer: any) => { + for (const pkg of packages) { + for (const referrer of pkg.topLevelReferrers) { // Skip if referrer doesn't have a valid packageUrl if (!referrer.packageUrl) { console.debug(`Skipping referrer without packageUrl for component: ${pkg.id}`); - return; + continue; } const referrerUrl = ComponentDetection.makePackageUrl(referrer.packageUrl); @@ -177,14 +177,14 @@ export default class ComponentDetection { // Skip if the generated packageUrl is empty if (!referrerUrl) { console.debug(`Skipping referrer with invalid packageUrl for component: ${pkg.id}`); - return; + continue; } try { const referrerPackage = packageCache.lookupPackage(referrerUrl); if (referrerPackage === pkg) { console.debug(`Found self-reference for package: ${pkg.id}`); - return; // Skip self-references + continue; // Skip self-references } if (referrerPackage) { referrerPackage.dependsOn(pkg); @@ -192,8 +192,8 @@ export default class ComponentDetection { } catch (error) { console.debug(`Error looking up referrer package: ${error}`); } - }); - }); + } + } // Create manifests const manifests: Array = []; @@ -367,8 +367,8 @@ export default class ComponentDetection { class ComponentDetectionPackage extends Package { public packageUrlString: string; - constructor(packageUrl: string, public id: string, public isDevelopmentDependency: boolean, public topLevelReferrers: [], - public locationsFoundAt: [], public containerDetailIds: [], public containerLayerIds: []) { + constructor(packageUrl: string, public id: string, public isDevelopmentDependency: boolean, public topLevelReferrers: any[], + public locationsFoundAt: any[], public containerDetailIds: any[], public containerLayerIds: any[]) { super(packageUrl); this.packageUrlString = packageUrl; } From f119054044555ca83678ddb1a816e59ee6b7a521 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:18:21 +0000 Subject: [PATCH 075/108] Add proper type definitions for ComponentDetectionPackage constructor Co-authored-by: aegilops <41705651+aegilops@users.noreply.github.com> --- src/componentDetection.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 8a83b58..0afa14b 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -364,11 +364,19 @@ export default class ComponentDetection { } } +/** + * Type for referrer objects in topLevelReferrers array + */ +type TopLevelReferrer = { + packageUrl?: any; + packageUrlString?: string; +}; + class ComponentDetectionPackage extends Package { public packageUrlString: string; - constructor(packageUrl: string, public id: string, public isDevelopmentDependency: boolean, public topLevelReferrers: any[], - public locationsFoundAt: any[], public containerDetailIds: any[], public containerLayerIds: any[]) { + constructor(packageUrl: string, public id: string, public isDevelopmentDependency: boolean, public topLevelReferrers: TopLevelReferrer[], + public locationsFoundAt: string[], public containerDetailIds: string[], public containerLayerIds: string[]) { super(packageUrl); this.packageUrlString = packageUrl; } From 8d93e5371ee9eae1e67be0c846be60ea3cc4e6db Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 8 Dec 2025 12:44:36 +0000 Subject: [PATCH 076/108] Better repo handling for --repo --- src/cli.ts | 1 + src/sbomCollector.ts | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/cli.ts b/src/cli.ts index ebaca1b..e6a3f7b 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -58,6 +58,7 @@ async function main() { if (!args.enterprise && !args.org && !args.repo) throw new Error("Provide --enterprise, --org or --repo with --sync-sboms"); if (args.enterprise && args.org) throw new Error("Specify only one of --enterprise or --org"); if (args.repo && (args.enterprise || args.org)) throw new Error("Specify only one of --enterprise, --org, or --repo"); + if (args.repo && !(args.repo as string).includes("/")) throw new Error("--repo must be in the format owner/repo"); if (syncing && !args.sbomCache) throw new Error("--sync-sboms requires --sbom-cache to write updated SBOMs to disk"); } else { const malwareOnly = !!args["sync-malware"] && !args.sbomCache && !args.purl && !args["purl-file"] && !args["match-malware"] && !args.uploadSarif && !args.interactive; diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index d081e2b..6a9e0e7 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -349,7 +349,11 @@ export class SbomCollector { renderBar(); })); await Promise.all(tasks); - newSboms = newSboms.filter(s => repoNames.has(s.repo) || repoNames.has(s.repo.split("/")[1])); + + newSboms = newSboms.filter(s => { + const repoToCheck = s.repo.includes("/") ? s.repo.split("/")[1] : s.repo; + return repoNames.has(repoToCheck); + }); this.sboms.push(...newSboms); } if (this.opts.showProgressBar) process.stdout.write("\n"); From 5b00aaf906ddb5155a229a577fa2c80d3804628d Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 8 Dec 2025 16:24:28 +0000 Subject: [PATCH 077/108] Sort out retries parameter and handling skipping submission --- src/componentSubmission.ts | 2 +- src/sbomCollector.ts | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index e4f6fec..079839d 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -70,7 +70,7 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise setTimeout(r, 1500)); + if (await submitSnapshotIfPossible({ octokit: this.octokit, owner: org, repo: repo.name, branch: b.name, languages: this.opts.submitLanguages, quiet: this.opts.quiet, componentDetectionBinPath: this.opts.componentDetectionBinPath })) { + // brief delay to allow snapshot ingestion + await new Promise(r => setTimeout(r, 1500)); + } } catch (subErr) { console.error(chalk.red(`Force submission failed for ${fullName} branch ${b.name}: ${(subErr as Error).message}`)); } } - const diff = await this.fetchDependencyReviewDiff(org, repo.name, base, b.name, latestCommit, 1); + const diff = await this.fetchDependencyReviewDiff(org, repo.name, base, b.name, 1, latestCommit); branchDiffs.set(b.name, diff); } if (branchDiffs.size) sbom.branchDiffs = branchDiffs; @@ -499,7 +500,7 @@ export class SbomCollector { return branches; } - private async fetchDependencyReviewDiff(org: string, repo: string, base: string, head: string, latestCommit?: { sha?: string; commitDate?: string, retries: number }): Promise { + private async fetchDependencyReviewDiff(org: string, repo: string, base: string, head: string, retries: number, latestCommit?: { sha?: string; commitDate?: string }): Promise { if (!this.octokit) throw new Error("No Octokit instance"); try { const basehead = `${base}...${head}`; @@ -536,7 +537,7 @@ export class SbomCollector { if (ok) { console.log(chalk.blue(`Snapshot submission attempted; waiting 3 seconds before retrying dependency review diff for ${org}/${repo} ${base}...${head}...`)); await new Promise(r => setTimeout(r, 3000)); - return await this.fetchDependencyReviewDiff(org, repo, base, head, latestCommit, retries--); + return await this.fetchDependencyReviewDiff(org, repo, base, head, retries--, latestCommit); } } catch (subErr) { console.error(chalk.red(`Snapshot submission failed for ${org}/${repo} branch ${head}: ${(subErr as Error).message}`)); From 546e33d95a6f1c5ad8c7edf6aa58b67bb20701da Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 8 Dec 2025 16:51:38 +0000 Subject: [PATCH 078/108] Fixed missing backticks --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0da227c..899289a 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,7 @@ Offline match with already-cached malware advisories (no network calls): ```bash npm run start -- --sbom-cache sboms --malware-cache malware-cache --match-malware +``` Malware-only advisory sync (no SBOM cache required): From 82b58dae766128dd19504b5ac934be7a81d9c3e0 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 8 Dec 2025 17:16:48 +0000 Subject: [PATCH 079/108] Added checking retries and made it pre-decrement --- src/sbomCollector.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 120420f..dcdc43c 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -502,6 +502,9 @@ export class SbomCollector { private async fetchDependencyReviewDiff(org: string, repo: string, base: string, head: string, retries: number, latestCommit?: { sha?: string; commitDate?: string }): Promise { if (!this.octokit) throw new Error("No Octokit instance"); + if (retries < 0) { + return { latestCommitDate: undefined, base, head, retrievedAt: new Date().toISOString(), changes: [], error: "Exceeded maximum retries for fetching dependency review diff" }; + } try { const basehead = `${base}...${head}`; const resp = await this.octokit.request("GET /repos/{owner}/{repo}/dependency-graph/compare/{basehead}", { owner: org, repo, basehead, headers: { Accept: "application/vnd.github+json" } }); @@ -537,7 +540,7 @@ export class SbomCollector { if (ok) { console.log(chalk.blue(`Snapshot submission attempted; waiting 3 seconds before retrying dependency review diff for ${org}/${repo} ${base}...${head}...`)); await new Promise(r => setTimeout(r, 3000)); - return await this.fetchDependencyReviewDiff(org, repo, base, head, retries--, latestCommit); + return await this.fetchDependencyReviewDiff(org, repo, base, head, --retries, latestCommit); } } catch (subErr) { console.error(chalk.red(`Snapshot submission failed for ${org}/${repo} branch ${head}: ${(subErr as Error).message}`)); From e6b11d9cd377ea41136ea738e0d96deab19fe184 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Mon, 8 Dec 2025 17:17:26 +0000 Subject: [PATCH 080/108] Update src/componentDetection.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentDetection.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 0afa14b..09853f9 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -272,9 +272,7 @@ export default class ComponentDetection { if (packageUrlJson.Version) { packageUrl += `@${packageUrlJson.Version}`; } - if (typeof packageUrlJson.Qualifiers === "object" - && packageUrlJson.Qualifiers !== null - && Object.keys(packageUrlJson.Qualifiers).length > 0) { + if (packageUrlJson.Qualifiers && Object.keys(packageUrlJson.Qualifiers).length > 0) { const qualifierString = Object.entries(packageUrlJson.Qualifiers) .map(([key, value]) => `${key}=${value}`) .join("&"); From f963d368a539665db2b7146c83f9fedb7a7176a8 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 8 Dec 2025 17:18:42 +0000 Subject: [PATCH 081/108] Use exact comparison --- src/componentDetection.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 0afa14b..968c71d 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -215,7 +215,7 @@ export default class ComponentDetection { // Unescape the path, as upstream ComponentDetection emits locationsFoundAt in URL-encoded form normalizedLocation = decodeURIComponent(normalizedLocation); - if (!manifests.find((manifest: Manifest) => manifest.name == normalizedLocation)) { + if (!manifests.find((manifest: Manifest) => manifest.name === normalizedLocation)) { const manifest = new Manifest(normalizedLocation, normalizedLocation); manifests.push(manifest); } @@ -229,14 +229,14 @@ export default class ComponentDetection { const directDependencies = depGraphEntry.explicitlyReferencedComponentIds; if (directDependencies.includes(pkg.id)) { manifests - .find((manifest: Manifest) => manifest.name == normalizedLocation) + .find((manifest: Manifest) => manifest.name === normalizedLocation) ?.addDirectDependency( pkg, ComponentDetection.getDependencyScope(pkg) ); } else { manifests - .find((manifest: Manifest) => manifest.name == normalizedLocation) + .find((manifest: Manifest) => manifest.name === normalizedLocation) ?.addIndirectDependency( pkg, ComponentDetection.getDependencyScope(pkg) From 1d04ee85400c2e4ec19e3bafe93f51e0ae7fd1cc Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Mon, 8 Dec 2025 17:23:08 +0000 Subject: [PATCH 082/108] Update src/sbomCollector.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/sbomCollector.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 120420f..18b2b87 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -260,7 +260,7 @@ export class SbomCollector { this.decisions[fullName] = `Fetching because error comparing pushed_at (${baseline.repoPushedAt} / ${repo.pushed_at})`; } } else { - this.decisions[fullName] = baseline ? `Fetching because missing pushed_at (${baseline.repoPushedAt} / ${repo.pushed_at})` : "Fetching because no baseline"; + this.decisions[fullName] = baseline ? `Fetching because of missing pushed_at (${baseline.repoPushedAt} / ${repo.pushed_at})` : "Fetching because no baseline"; } let sbom: RepositorySbom | undefined = undefined; From 46f8f8c8dfb69d37e136bcb9fc035b191ce8505c Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 8 Dec 2025 17:40:14 +0000 Subject: [PATCH 083/108] Clarified comment --- src/componentSubmission.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index 079839d..a15d42d 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -189,7 +189,7 @@ export async function run(octokit: Octokit, tmpDir: string, owner: string, repo: } /** - * submitSnapshot submits a snapshot to the Dependency Submission API - vendored in from @github/dependency-submission-toolkit, to make it work at the CLI, vs in Actions. + * submitSnapshot submits a snapshot to the Dependency Submission API - vendored in and modified from @github/dependency-submission-toolkit, to make it work at the CLI, vs in Actions. * * @param {Octokit} octokit - The Octokit instance for GitHub API requests * @param {Snapshot} snapshot - The dependency snapshot to submit From fb52fb915bb39996572ffcbd20b882c494b86886 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 8 Dec 2025 17:49:44 +0000 Subject: [PATCH 084/108] Removed type ignore --- src/cli.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index e6a3f7b..7763e36 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -140,9 +140,6 @@ async function main() { submitOnMissingSnapshot: argv["submit-on-missing-snapshot"] as boolean, forceSubmission: argv["force-submission"] as boolean, submitLanguages: (argv["submit-languages"] as string[] | undefined) || undefined, - // Pass through as part of options bag used by submission helper via collector - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore componentDetectionBinPath: argv["component-detection-bin"] as string | undefined, }) : undefined; From 49970785e6bb595c7b6eb36bc1901646911af87a Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Mon, 8 Dec 2025 17:51:59 +0000 Subject: [PATCH 085/108] Update src/componentDetection.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentDetection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 09853f9..2ea0ff3 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -34,7 +34,7 @@ export default class ComponentDetection { // This is the default entry point for this class. // If executablePath is provided, use it directly and skip download. async scanAndGetManifests(path: string): Promise { - if (!this.componentDetectionPath) { + if (!fs.existsSync(this.componentDetectionPath)) { await this.downloadLatestRelease(); } From e3aaf6f18c3c63875be818326dc37124f15f3f85 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Mon, 8 Dec 2025 17:55:35 +0000 Subject: [PATCH 086/108] Formatted --- src/cli.ts | 4 ++-- src/componentDetection.ts | 2 +- src/sbomCollector.ts | 2 +- src/test-branch-search.ts | 16 ++++++++-------- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index 7763e36..b023dc3 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -94,7 +94,7 @@ async function main() { if (debug) { console.debug(chalk.blue("Debug logging enabled")); } else { - console.debug = () => {}; + console.debug = () => { }; } const token = argv.token as string | undefined || process.env.GITHUB_TOKEN; @@ -113,7 +113,7 @@ async function main() { const wantCsv = !!argv.csv; const hasOutputFile = !!argv.outputFile; const wantCli = !!argv.cli && hasOutputFile; // only allow CLI alongside machine output when writing file - + let sboms: RepositorySbom[] = []; let summary: CollectionSummary | undefined; diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 968c71d..6b664e3 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -11,7 +11,7 @@ import path from 'path'; import { tmpdir } from 'os'; import { StringDecoder } from 'node:string_decoder'; -export default class ComponentDetection { +export default class ComponentDetection { public componentDetectionPath: string = process.platform === "win32" ? './component-detection.exe' : './component-detection'; public outputPath: string; octokit: Octokit; diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index dcdc43c..6bc6697 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -350,7 +350,7 @@ export class SbomCollector { renderBar(); })); await Promise.all(tasks); - + newSboms = newSboms.filter(s => { const repoToCheck = s.repo.includes("/") ? s.repo.split("/")[1] : s.repo; return repoNames.has(repoToCheck); diff --git a/src/test-branch-search.ts b/src/test-branch-search.ts index 059418c..208e797 100644 --- a/src/test-branch-search.ts +++ b/src/test-branch-search.ts @@ -32,14 +32,14 @@ async function main() { retrievedAt: new Date().toISOString(), packages: basePackages, branchDiffs: [ - { - latestCommitDate: new Date().toISOString(), - base: 'main', - head: 'feature-x', - retrievedAt: new Date().toISOString(), - changes: diffChanges - } - ] + { + latestCommitDate: new Date().toISOString(), + base: 'main', + head: 'feature-x', + retrievedAt: new Date().toISOString(), + changes: diffChanges + } + ] }; fs.writeFileSync(path.join(repoDir, 'sbom.json'), JSON.stringify(synthetic, null, 2), 'utf8'); From 4dcf2b30b8a633014fdde529ec9ff3b8194eb757 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Mon, 8 Dec 2025 18:07:49 +0000 Subject: [PATCH 087/108] Update src/sbomCollector.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/sbomCollector.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 18b2b87..2e99a2d 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -537,7 +537,7 @@ export class SbomCollector { if (ok) { console.log(chalk.blue(`Snapshot submission attempted; waiting 3 seconds before retrying dependency review diff for ${org}/${repo} ${base}...${head}...`)); await new Promise(r => setTimeout(r, 3000)); - return await this.fetchDependencyReviewDiff(org, repo, base, head, retries--, latestCommit); + return await this.fetchDependencyReviewDiff(org, repo, base, head, retries - 1, latestCommit); } } catch (subErr) { console.error(chalk.red(`Snapshot submission failed for ${org}/${repo} branch ${head}: ${(subErr as Error).message}`)); From ba8ff05ad36967eb75793afe86725f5425d4fcf3 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Mon, 8 Dec 2025 18:16:43 +0000 Subject: [PATCH 088/108] Update src/componentDetection.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentDetection.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 6745a89..85add7b 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -119,7 +119,12 @@ export default class ComponentDetection { public async getManifestsFromResults(file: string, path: string): Promise { console.debug(`Reading results from ${file}`); const results = await fs.readFileSync(file, 'utf8'); - const json: any = JSON.parse(results); + let json: any; + try { + json = JSON.parse(results); + } catch (err: any) { + throw new Error(`Failed to parse JSON results from component-detection output file "${file}": ${err instanceof Error ? err.message : String(err)}`); + } let dependencyGraphs: DependencyGraphs = this.normalizeDependencyGraphPaths(json.dependencyGraphs, path); From 54315569abe9916d85d7a61fff6b3516f6f1ce19 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Mon, 8 Dec 2025 18:18:54 +0000 Subject: [PATCH 089/108] Update src/sbomCollector.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/sbomCollector.ts | 72 ++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 00decb8..a3ebd06 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -591,42 +591,48 @@ export class SbomCollector { const queries: ParsedQuery[] = purls.map(parseQuery).filter((q): q is ParsedQuery => !!q); const results = new Map(); if (!queries.length) return results; - const applyQueries = (candidatePurls: string[], queries: ParsedQuery[], found: Map, branchTag?: string, fallbackVersion?: string) => { - const unique = Array.from(new Set(candidatePurls)); - for (const p of unique) { - const pLower = p.toLowerCase(); - const outKey = branchTag ? `${p}@${branchTag}` : p; - for (const q of queries) { - if (q.isPrefixWildcard) { - const prefix = q.lower.slice(0, -1); - if (pLower.startsWith(prefix)) { if (!found.has(outKey)) found.set(outKey, q.raw); } - continue; - } - if (q.versionConstraint && q.type && q.name) { - if (!pLower.startsWith("pkg:")) continue; - const body = p.slice(4); - const atIdx = body.indexOf("@"); - const main = atIdx >= 0 ? body.slice(0, atIdx) : body; - const ver = atIdx >= 0 ? body.slice(atIdx + 1) : fallbackVersion; - const slashIdx = main.indexOf("/"); - if (slashIdx < 0) continue; - const pType = main.slice(0, slashIdx).toLowerCase(); - const pName = main.slice(slashIdx + 1); - if (pType === q.type && pName.toLowerCase() === q.name.toLowerCase() && ver) { - try { - const coerced = semver.coerce(ver)?.version || ver; - if (semver.valid(coerced) && semver.satisfies(coerced, q.versionConstraint, { includePrerelease: true })) { - if (!found.has(outKey)) found.set(outKey, q.raw); - } - } catch { /* ignore */ } +// Move applyQueries to module scope +function applyQueries( + candidatePurls: string[], + queries: ParsedQuery[], + found: Map, + branchTag?: string, + fallbackVersion?: string +) { + const unique = Array.from(new Set(candidatePurls)); + for (const p of unique) { + const pLower = p.toLowerCase(); + const outKey = branchTag ? `${p}@${branchTag}` : p; + for (const q of queries) { + if (q.isPrefixWildcard) { + const prefix = q.lower.slice(0, -1); + if (pLower.startsWith(prefix)) { if (!found.has(outKey)) found.set(outKey, q.raw); } + continue; + } + if (q.versionConstraint && q.type && q.name) { + if (!pLower.startsWith("pkg:")) continue; + const body = p.slice(4); + const atIdx = body.indexOf("@"); + const main = atIdx >= 0 ? body.slice(0, atIdx) : body; + const ver = atIdx >= 0 ? body.slice(atIdx + 1) : fallbackVersion; + const slashIdx = main.indexOf("/"); + if (slashIdx < 0) continue; + const pType = main.slice(0, slashIdx).toLowerCase(); + const pName = main.slice(slashIdx + 1); + if (pType === q.type && pName.toLowerCase() === q.name.toLowerCase() && ver) { + try { + const coerced = semver.coerce(ver)?.version || ver; + if (semver.valid(coerced) && semver.satisfies(coerced, q.versionConstraint, { includePrerelease: true })) { + if (!found.has(outKey)) found.set(outKey, q.raw); } - } else if (q.exact) { - if (pLower === q.exact) { if (!found.has(outKey)) found.set(outKey, q.raw); } - } + } catch { /* ignore */ } } + } else if (q.exact) { + if (pLower === q.exact) { if (!found.has(outKey)) found.set(outKey, q.raw); } } - }; - + } + } +} for (const repoSbom of this.sboms) { if (repoSbom.error) continue; interface ExtRef { referenceType: string; referenceLocator: string } From 4ad422a5fd6e0e1e78fa8bce3e10b0120af9d499 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 8 Dec 2025 18:21:12 +0000 Subject: [PATCH 090/108] Initial plan From 395c925839ea11786e828df3ade70cf9f7d06e37 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 8 Dec 2025 18:29:35 +0000 Subject: [PATCH 091/108] Make snapshot ingestion delays configurable Co-authored-by: aegilops <41705651+aegilops@users.noreply.github.com> --- src/sbomCollector.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index a3ebd06..b42cb38 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -34,6 +34,8 @@ export interface CollectorOptions { forceSubmission?: boolean; // always submit snapshot for branches prior to diff submitLanguages?: string[]; // limit submission to these languages componentDetectionBinPath?: string; // optional path to component-detection executable + snapshotIngestionDelayMs?: number; // delay after snapshot submission to allow ingestion before dependency review (default: 1500ms) + retryIngestionDelayMs?: number; // delay after snapshot submission before retrying dependency review on 404 (default: 3000ms) } export class SbomCollector { @@ -85,7 +87,9 @@ export class SbomCollector { submitOnMissingSnapshot: o.submitOnMissingSnapshot ?? false, forceSubmission: o.forceSubmission ?? false, submitLanguages: o.submitLanguages ?? undefined, - componentDetectionBinPath: o.componentDetectionBinPath + componentDetectionBinPath: o.componentDetectionBinPath, + snapshotIngestionDelayMs: o.snapshotIngestionDelayMs ?? 1500, + retryIngestionDelayMs: o.retryIngestionDelayMs ?? 3000 } as Required; if (this.opts.token) { @@ -319,8 +323,9 @@ export class SbomCollector { try { console.debug(chalk.blue(`Force-submission enabled: submitting component snapshot for ${fullName} branch ${b.name}...`)); if (await submitSnapshotIfPossible({ octokit: this.octokit, owner: org, repo: repo.name, branch: b.name, languages: this.opts.submitLanguages, quiet: this.opts.quiet, componentDetectionBinPath: this.opts.componentDetectionBinPath })) { - // brief delay to allow snapshot ingestion - await new Promise(r => setTimeout(r, 1500)); + // Brief delay to allow GitHub to ingest the submitted snapshot before attempting dependency review. + // This prevents race conditions where the review diff is requested before the snapshot is available. + await new Promise(r => setTimeout(r, this.opts.snapshotIngestionDelayMs)); } } catch (subErr) { console.error(chalk.red(`Force submission failed for ${fullName} branch ${b.name}: ${(subErr as Error).message}`)); @@ -538,8 +543,10 @@ export class SbomCollector { try { const ok = await submitSnapshotIfPossible({ octokit: this.octokit, owner: org, repo: repo, branch: head, languages: this.opts.submitLanguages, quiet: this.opts.quiet, componentDetectionBinPath: this.opts.componentDetectionBinPath }); if (ok) { - console.log(chalk.blue(`Snapshot submission attempted; waiting 3 seconds before retrying dependency review diff for ${org}/${repo} ${base}...${head}...`)); - await new Promise(r => setTimeout(r, 3000)); + // Delay after snapshot submission to allow GitHub to ingest and process the snapshot + // before retrying the dependency review API. This helps avoid 404 errors on retry. + console.log(chalk.blue(`Snapshot submission attempted; waiting ${this.opts.retryIngestionDelayMs}ms before retrying dependency review diff for ${org}/${repo} ${base}...${head}...`)); + await new Promise(r => setTimeout(r, this.opts.retryIngestionDelayMs)); return await this.fetchDependencyReviewDiff(org, repo, base, head, retries - 1, latestCommit); } } catch (subErr) { From 4b2e75c2d46eb2d20b4d4935c983588eadf74225 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Mon, 8 Dec 2025 18:30:05 +0000 Subject: [PATCH 092/108] Update src/malwareMatcher.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/malwareMatcher.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/malwareMatcher.ts b/src/malwareMatcher.ts index 232dbd0..b4a28a4 100644 --- a/src/malwareMatcher.ts +++ b/src/malwareMatcher.ts @@ -159,8 +159,13 @@ export function matchMalware(advisories: MalwareAdvisoryNode[], sboms: Repositor } } // Annotate with default branch for reporting (if known) - return list.map(p => ({ ...p, __branch: repo.defaultBranch || undefined })); - }; + // Some ecosystems require a namespace (e.g., Maven, NuGet) + const ecosystemsWithNamespace = new Set(['maven', 'nuget', 'composer', 'golang']); + if (ecosystemsWithNamespace.has(change.ecosystem) && change.namespace) { + p = `pkg:${change.ecosystem}/${change.namespace}/${change.name}${change.version ? '@' + change.version : ''}`; + } else { + p = `pkg:${change.ecosystem}/${change.name}${change.version ? '@' + change.version : ''}`; + } // Enumerate packages implied by branch diffs (added/updated head-side versions) const enumerateDiffPackages = (repo: RepositorySbom): Array<{ purl: string; name?: string; ecosystem?: string; version?: string; __branch: string }> => { From 3d5f9ad1b18af2e569236e34d79e931843c4979b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 8 Dec 2025 18:31:49 +0000 Subject: [PATCH 093/108] Remove redundant 'ms' from log message Co-authored-by: aegilops <41705651+aegilops@users.noreply.github.com> --- src/sbomCollector.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index b42cb38..a67ec57 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -545,7 +545,7 @@ export class SbomCollector { if (ok) { // Delay after snapshot submission to allow GitHub to ingest and process the snapshot // before retrying the dependency review API. This helps avoid 404 errors on retry. - console.log(chalk.blue(`Snapshot submission attempted; waiting ${this.opts.retryIngestionDelayMs}ms before retrying dependency review diff for ${org}/${repo} ${base}...${head}...`)); + console.log(chalk.blue(`Snapshot submission attempted; waiting ${this.opts.retryIngestionDelayMs / 1000} seconds before retrying dependency review diff for ${org}/${repo} ${base}...${head}...`)); await new Promise(r => setTimeout(r, this.opts.retryIngestionDelayMs)); return await this.fetchDependencyReviewDiff(org, repo, base, head, retries - 1, latestCommit); } From e7f34a6311b460ea1766814f7ef272498a916aa4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 8 Dec 2025 18:43:10 +0000 Subject: [PATCH 094/108] Add CLI arguments for snapshot ingestion delays Co-authored-by: aegilops <41705651+aegilops@users.noreply.github.com> --- src/cli.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cli.ts b/src/cli.ts index b023dc3..dc68ae0 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -51,6 +51,8 @@ async function main() { .option("submit-languages", { type: "array", describe: "Limit snapshot submission to these languages (e.g., JavaScript,TypeScript,Python,Maven)." }) .option("component-detection-bin", { type: "string", describe: "Path to a local component-detection executable to use for snapshot submission (skips download)." }) .option("force-submission", { type: "boolean", default: false, describe: "Always run Dependency Submission for scanned branches before fetching diffs." }) + .option("snapshot-ingestion-delay", { type: "number", default: 1500, describe: "Delay (ms) after snapshot submission to allow ingestion before dependency review (default: 1500ms)" }) + .option("retry-ingestion-delay", { type: "number", default: 3000, describe: "Delay (ms) after snapshot submission before retrying dependency review on 404 (default: 3000ms)" }) .option("debug", { type: "boolean", default: false, describe: "Enable debug logging" }) .check(args => { const syncing = !!args.syncSboms; @@ -141,6 +143,8 @@ async function main() { forceSubmission: argv["force-submission"] as boolean, submitLanguages: (argv["submit-languages"] as string[] | undefined) || undefined, componentDetectionBinPath: argv["component-detection-bin"] as string | undefined, + snapshotIngestionDelayMs: argv["snapshot-ingestion-delay"] as number | undefined, + retryIngestionDelayMs: argv["retry-ingestion-delay"] as number | undefined, }) : undefined; if (collector && (argv.sbomCache || argv.syncSboms)) { From bd2e92e092bdebda8bcbe225e25e738dda942a62 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Tue, 9 Dec 2025 09:20:03 +0000 Subject: [PATCH 095/108] Update src/componentDetection.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/componentDetection.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/componentDetection.ts b/src/componentDetection.ts index 85add7b..669d5fe 100644 --- a/src/componentDetection.ts +++ b/src/componentDetection.ts @@ -315,7 +315,16 @@ export default class ComponentDetection { let downloadURL: string = ""; // TODO: do we need to handle different architectures here? // can we allow x64 on MacOS? We could allow an input parameter to override? - const assetName = process.platform === "win32" ? "component-detection-win-x64.exe" : process.platform === "linux" ? "component-detection-linux-x64" : "component-detection-osx-arm64"; + let assetName: string; + if (process.platform === "win32") { + assetName = "component-detection-win-x64.exe"; + } else if (process.platform === "linux") { + assetName = "component-detection-linux-x64"; + } else if (process.platform === "darwin") { + assetName = "component-detection-osx-arm64"; + } else { + throw new Error(`Unsupported platform: ${process.platform}`); + } latestRelease.data.assets.forEach((asset: any) => { if (asset.name === assetName) { downloadURL = asset.browser_download_url; From a79846097685be5ac99244171b38ae11402c9dd4 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 9 Dec 2025 11:07:11 +0000 Subject: [PATCH 096/108] Fix mismerged code --- src/malwareMatcher.ts | 19 +++++----- src/sbomCollector.ts | 81 ++++++++++++++++++++++--------------------- 2 files changed, 52 insertions(+), 48 deletions(-) diff --git a/src/malwareMatcher.ts b/src/malwareMatcher.ts index b4a28a4..9d3711b 100644 --- a/src/malwareMatcher.ts +++ b/src/malwareMatcher.ts @@ -106,6 +106,8 @@ export interface MatchMalwareOptions { advisoryDateCutoff?: string; } +const ecosystemsWithNamespace = new Set(['maven', 'nuget', 'composer', 'golang']); + export function matchMalware(advisories: MalwareAdvisoryNode[], sboms: RepositorySbom[], opts?: MatchMalwareOptions): MalwareMatch[] { const matches: MalwareMatch[] = []; @@ -159,13 +161,9 @@ export function matchMalware(advisories: MalwareAdvisoryNode[], sboms: Repositor } } // Annotate with default branch for reporting (if known) - // Some ecosystems require a namespace (e.g., Maven, NuGet) - const ecosystemsWithNamespace = new Set(['maven', 'nuget', 'composer', 'golang']); - if (ecosystemsWithNamespace.has(change.ecosystem) && change.namespace) { - p = `pkg:${change.ecosystem}/${change.namespace}/${change.name}${change.version ? '@' + change.version : ''}`; - } else { - p = `pkg:${change.ecosystem}/${change.name}${change.version ? '@' + change.version : ''}`; - } + const branchName = repo.defaultBranch || undefined; + return list.map(p => ({ ...p, __branch: branchName })); + } // Enumerate packages implied by branch diffs (added/updated head-side versions) const enumerateDiffPackages = (repo: RepositorySbom): Array<{ purl: string; name?: string; ecosystem?: string; version?: string; __branch: string }> => { @@ -178,8 +176,11 @@ export function matchMalware(advisories: MalwareAdvisoryNode[], sboms: Repositor let p: string | undefined = (change as { purl?: string }).purl; if (!p && change.packageURL && change.packageURL.startsWith('pkg:')) p = change.packageURL; if (!p && change.ecosystem && change.name && change.version) { - // Dependency review ecosystems are lower-case purl types already (e.g. npm, maven, pip, gem) - p = `pkg:${change.ecosystem}/${change.name}${change.version ? '@' + change.version : ''}`; + if (ecosystemsWithNamespace.has(change.ecosystem) && change.namespace) { + p = `pkg:${change.ecosystem}/${change.namespace}/${change.name}${change.version ? '@' + change.version : ''}`; + } else { + p = `pkg:${change.ecosystem}/${change.name}${change.version ? '@' + change.version : ''}`; + } } if (!p) continue; out.push({ purl: p, name: change.name, ecosystem: change.ecosystem, version: change.version, __branch: branchName }); diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index a67ec57..e9fd0ce 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -38,6 +38,16 @@ export interface CollectorOptions { retryIngestionDelayMs?: number; // delay after snapshot submission before retrying dependency review on 404 (default: 3000ms) } +interface ParsedQuery { + raw: string; + lower: string; + isPrefixWildcard: boolean; + exact?: string; + type?: string; + name?: string; + versionConstraint?: string; +} + export class SbomCollector { private octokit: ReturnType | undefined; // explicit type private opts: Required; @@ -562,15 +572,7 @@ export class SbomCollector { // New method including the query that produced each match searchByPurlsWithReasons(purls: string[]): Map { purls = purls.map(q => q.startsWith("pkg:") ? q : `pkg:${q}`); - interface ParsedQuery { - raw: string; - lower: string; - isPrefixWildcard: boolean; - exact?: string; - type?: string; - name?: string; - versionConstraint?: string; - } + const looksLikeSemverRange = (v: string) => /[\^~><=]|\|\|/.test(v.trim()); const parseQuery = (raw: string): ParsedQuery | null => { const trimmed = raw.trim(); @@ -598,7 +600,37 @@ export class SbomCollector { const queries: ParsedQuery[] = purls.map(parseQuery).filter((q): q is ParsedQuery => !!q); const results = new Map(); if (!queries.length) return results; -// Move applyQueries to module scope + + for (const repoSbom of this.sboms) { + if (repoSbom.error) continue; + interface ExtRef { referenceType: string; referenceLocator: string } + const found = new Map(); // purl -> query + for (const pkg of repoSbom.packages as Array) { + const refs = pkg.externalRefs; + const candidatePurls: string[] = []; + if (refs) for (const r of refs) if (r.referenceType === "purl" && r.referenceLocator) candidatePurls.push(r.referenceLocator); + if ((pkg as { purl?: string }).purl) candidatePurls.push((pkg as { purl?: string }).purl as string); + applyQueries(candidatePurls, queries, found, undefined, (pkg.version as string | undefined) || undefined); + } + // Include dependency review diff additions/updates (head packages only) + if (repoSbom.branchDiffs) { + const diffs = repoSbom.branchDiffs.values(); + for (const diff of diffs) { + for (const change of diff.changes) { + if (change.changeType !== "added" && change.changeType !== "updated") continue; + const candidatePurls: string[] = []; + if ((change as { purl?: string }).purl) candidatePurls.push((change as { purl?: string }).purl as string); + if (change.packageURL) candidatePurls.push(change.packageURL); + applyQueries(candidatePurls, queries, found, diff.head, change.version); + } + } + } + if (found.size) results.set(repoSbom.repo, Array.from(found.entries()).map(([purl, reason]) => ({ purl, reason }))); + } + return results; + } +} + function applyQueries( candidatePurls: string[], queries: ParsedQuery[], @@ -639,33 +671,4 @@ function applyQueries( } } } -} - for (const repoSbom of this.sboms) { - if (repoSbom.error) continue; - interface ExtRef { referenceType: string; referenceLocator: string } - const found = new Map(); // purl -> query - for (const pkg of repoSbom.packages as Array) { - const refs = pkg.externalRefs; - const candidatePurls: string[] = []; - if (refs) for (const r of refs) if (r.referenceType === "purl" && r.referenceLocator) candidatePurls.push(r.referenceLocator); - if ((pkg as { purl?: string }).purl) candidatePurls.push((pkg as { purl?: string }).purl as string); - applyQueries(candidatePurls, queries, found, undefined, (pkg.version as string | undefined) || undefined); - } - // Include dependency review diff additions/updates (head packages only) - if (repoSbom.branchDiffs) { - const diffs = repoSbom.branchDiffs.values(); - for (const diff of diffs) { - for (const change of diff.changes) { - if (change.changeType !== "added" && change.changeType !== "updated") continue; - const candidatePurls: string[] = []; - if ((change as { purl?: string }).purl) candidatePurls.push((change as { purl?: string }).purl as string); - if (change.packageURL) candidatePurls.push(change.packageURL); - applyQueries(candidatePurls, queries, found, diff.head, change.version); - } - } - } - if (found.size) results.set(repoSbom.repo, Array.from(found.entries()).map(([purl, reason]) => ({ purl, reason }))); - } - return results; - } } From bfa1e6a678d95a4bc275aa9ae97462ac7f44992c Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 9 Dec 2025 11:09:47 +0000 Subject: [PATCH 097/108] Remove redundant assignment --- src/sbomCollector.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index e9fd0ce..4333474 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -194,7 +194,6 @@ export class SbomCollector { totalRepos = 1; const [org, repoName] = this.opts.repo.split("/"); orgRepoMap[org] = [await this.getRepo(org, repoName)]; - this.summary.orgs = orgs; } this.summary.repositoryCount = totalRepos; From 5607399816b6c43bff6c02f1e9651947ddaaab50 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 9 Dec 2025 11:14:32 +0000 Subject: [PATCH 098/108] Fix SBOM counts and caching --- src/sbomCollector.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 4333474..584df54 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -349,14 +349,15 @@ export class SbomCollector { this.decisions[fullName] = (this.decisions[fullName] || "") + ` (branch scan error: ${(e as Error).message})`; console.debug((e as Error).message); } + } - if (sbom.error) this.summary.failedCount++; else this.summary.successCount++; + if (!sbom || sbom.error) this.summary.failedCount++; else this.summary.successCount++; - // Write freshly fetched SBOM immediately if a cache directory is configured - if (this.opts.loadFromDir && this.opts.syncSboms && this.opts.loadFromDir.length) { - try { writeOne(sbom, { outDir: this.opts.loadFromDir }); } catch { /* ignore write errors */ } - } + // Write freshly fetched SBOM immediately if a cache directory is configured + if (sbom && !sbom.error && this.opts.loadFromDir && this.opts.syncSboms && this.opts.loadFromDir.length) { + try { writeOne(sbom, { outDir: this.opts.loadFromDir }); } catch { /* ignore write errors */ } } + if (sbom) { newSboms.push(sbom); } From 9a78c60f88bb7b05fa3f98f0b4a9a079378a7dc0 Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Tue, 9 Dec 2025 15:55:52 +0000 Subject: [PATCH 099/108] Update src/malwareMatcher.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/malwareMatcher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/malwareMatcher.ts b/src/malwareMatcher.ts index 9d3711b..90d0fd2 100644 --- a/src/malwareMatcher.ts +++ b/src/malwareMatcher.ts @@ -163,7 +163,7 @@ export function matchMalware(advisories: MalwareAdvisoryNode[], sboms: Repositor // Annotate with default branch for reporting (if known) const branchName = repo.defaultBranch || undefined; return list.map(p => ({ ...p, __branch: branchName })); - } + }; // Enumerate packages implied by branch diffs (added/updated head-side versions) const enumerateDiffPackages = (repo: RepositorySbom): Array<{ purl: string; name?: string; ecosystem?: string; version?: string; __branch: string }> => { From 6be40f1e2014d9168f3a13db37e81948b965847a Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 9 Dec 2025 16:01:47 +0000 Subject: [PATCH 100/108] Remove default for malware-cache --- src/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli.ts b/src/cli.ts index dc68ae0..db1e87e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -30,7 +30,7 @@ async function main() { .option("quiet", { type: "boolean", default: false, describe: "Suppress all non-error output (does not suppress progress bar or JSON)" }) .option("interactive", { type: "boolean", default: false, describe: "Enter interactive PURL search mode after collection" }) .option("sync-malware", { type: "boolean", default: false, describe: "Sync malware advisories (MALWARE classification) to local cache" }) - .option("malware-cache", { type: "string", default: "malware-cache", describe: "Directory to store malware advisory cache" }) + .option("malware-cache", { type: "string", describe: "Directory to store malware advisory cache" }) .option("malware-since", { type: "string", describe: "Override last sync timestamp (ISO) for malware advisory incremental sync" }) .option("ca-bundle", { type: "string", describe: "Path to PEM file with additional CA certificate(s) (self-signed/internal)" }) .option("match-malware", { type: "boolean", default: false, describe: "After sync/load, match SBOM packages against malware advisories" }) From d5bad5abf1287330a25f1767f10febbd921e1972 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 9 Dec 2025 16:02:02 +0000 Subject: [PATCH 101/108] Added namespace to type --- src/types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/types.ts b/src/types.ts index cb26664..c90dd87 100644 --- a/src/types.ts +++ b/src/types.ts @@ -112,6 +112,7 @@ export interface DependencyReviewPackageChange { changeType: string; // added | removed | updated name?: string; // package name ecosystem?: string; // e.g. npm, maven, pip + namespace?: string; // e.g. groupId for maven packageURL?: string; // raw package URL (may be purl-like) purl?: string; // normalized purl (if derivable) license?: string; From b2d1d036a61b2a2a452dfcc242515711cb42c24f Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 9 Dec 2025 16:23:02 +0000 Subject: [PATCH 102/108] Minor changes to logging --- src/sbomCollector.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 584df54..171d9b3 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -546,7 +546,7 @@ export class SbomCollector { const status = (e as { status?: number })?.status; let reason = e instanceof Error ? e.message : String(e); if (status === 404) { - reason = "Dependency review unavailable (missing snapshot or feature disabled)"; + reason = "Dependency review unavailable (missing snapshot, feature disabled, or repo not found)"; // Optional retry path: submit snapshot then retry once if (this.opts.submitOnMissingSnapshot) { console.log(chalk.blue(`Attempting to submit component snapshot for ${org}/${repo} branch ${head} before retrying dependency review diff...`)); @@ -555,7 +555,7 @@ export class SbomCollector { if (ok) { // Delay after snapshot submission to allow GitHub to ingest and process the snapshot // before retrying the dependency review API. This helps avoid 404 errors on retry. - console.log(chalk.blue(`Snapshot submission attempted; waiting ${this.opts.retryIngestionDelayMs / 1000} seconds before retrying dependency review diff for ${org}/${repo} ${base}...${head}...`)); + console.debug(chalk.blue(`Snapshot submission attempted; waiting ${this.opts.retryIngestionDelayMs / 1000} seconds before retrying dependency review diff for ${org}/${repo} ${base}...${head}...`)); await new Promise(r => setTimeout(r, this.opts.retryIngestionDelayMs)); return await this.fetchDependencyReviewDiff(org, repo, base, head, retries - 1, latestCommit); } From a5023770d86260a7a103989bf135a1a54caa4844 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 9 Dec 2025 16:24:19 +0000 Subject: [PATCH 103/108] Remove unused type --- src/types.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/types.ts b/src/types.ts index c90dd87..8af5472 100644 --- a/src/types.ts +++ b/src/types.ts @@ -113,8 +113,7 @@ export interface DependencyReviewPackageChange { name?: string; // package name ecosystem?: string; // e.g. npm, maven, pip namespace?: string; // e.g. groupId for maven - packageURL?: string; // raw package URL (may be purl-like) - purl?: string; // normalized purl (if derivable) + packageURL?: string; // raw package URL license?: string; manifest?: string; // manifest path scope?: string; // e.g. runtime, development From 4efe35e69b329d789c43bc23ce5c273240093203 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 9 Dec 2025 16:44:09 +0000 Subject: [PATCH 104/108] Add more delays to processing --- src/componentSubmission.ts | 18 ++++++++++++------ src/sbomCollector.ts | 24 +++++++++++++++++++++--- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/componentSubmission.ts b/src/componentSubmission.ts index a15d42d..1ceaaaf 100644 --- a/src/componentSubmission.ts +++ b/src/componentSubmission.ts @@ -23,10 +23,14 @@ export interface SubmitOpts { quiet?: boolean; languages?: string[]; componentDetectionBinPath?: string; // optional path to component-detection executable + lightDelayMs?: number; } -export async function getLanguageIntersection(octokit: Octokit, owner: string, repo: string, languages: string[] | undefined, quiet: boolean = false): Promise { +export async function getLanguageIntersection(octokit: Octokit, owner: string, repo: string, languages: string[] | undefined, quiet: boolean = false, lightDelayMs: number = 0): Promise { const langResp = await octokit.request('GET /repos/{owner}/{repo}/languages', { owner, repo }); + + await new Promise(r => setTimeout(r, lightDelayMs)); + const repoLangs = Object.keys(langResp.data || {}); const wanted = languages; const intersect = wanted ? repoLangs.filter(l => wanted.some(w => w.toLowerCase() === l.toLowerCase())) : repoLangs; @@ -37,7 +41,7 @@ export async function getLanguageIntersection(octokit: Octokit, owner: string, r return intersect; } -export async function sparseCheckout(owner: string, repo: string, branch: string, destDir: string, intersect: string[], baseUrl?: string) { +export async function sparseCheckout(owner: string, repo: string, branch: string, destDir: string, intersect: string[], baseUrl?: string, lightDelayMs?: number) { const cwd = destDir; const repoUrl = (baseUrl && baseUrl.includes('api/v3')) ? baseUrl.replace(/\/api\/v3$/, '') + `/${owner}/${repo}.git` @@ -52,6 +56,8 @@ export async function sparseCheckout(owner: string, repo: string, branch: string await execGit(['fetch', '--depth=1', 'origin', branch], { cwd }); await execGit(['checkout', 'FETCH_HEAD'], { cwd }); + await new Promise(r => setTimeout(r, lightDelayMs)); + const { stdout: shaOut } = await execGit(['rev-parse', 'HEAD'], { cwd: destDir }); const sha = shaOut.trim(); console.debug(`Checked out ${owner}/${repo}@${branch} to ${destDir} at commit ${sha}`); @@ -66,7 +72,7 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise { +export async function runComponentDetectionAndSubmit(octokit: Octokit, tmpDir: string, owner: string, repo: string, sha: string, ref: string, componentDetectionBinPath?: string): Promise { const componentDetection = new ComponentDetection(octokit, '', componentDetectionBinPath); diff --git a/src/sbomCollector.ts b/src/sbomCollector.ts index 171d9b3..0e1a684 100644 --- a/src/sbomCollector.ts +++ b/src/sbomCollector.ts @@ -300,6 +300,9 @@ export class SbomCollector { try { const branches = await this.listBranches(org, repo.name); + + if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); + const nonDefault = branches.filter(b => b.name !== sbom.defaultBranch); const limited = this.opts.branchLimit && this.opts.branchLimit > 0 ? nonDefault.slice(0, this.opts.branchLimit) : nonDefault; const branchDiffs: Map = new Map(); @@ -307,10 +310,14 @@ export class SbomCollector { // get the commits, compare to the stored diff info. If the latest commit is newer, then fetch diff, otherwise skip const latestCommit = await this.getLatestCommit(org, repo.name, b.name); + + if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); + if (!latestCommit) { console.error(chalk.red(`Failed to get latest commit for ${fullName} branch ${b.name}.`)); continue; } + const existing = sbom.branchDiffs instanceof Map ? sbom.branchDiffs.get(b.name) : undefined; if (await this.isCommitNewer(latestCommit, existing) || this.opts.forceSubmission) { console.debug(chalk.green(`Fetching branch diff for ${fullName} branch ${b.name}...`)); @@ -326,7 +333,6 @@ export class SbomCollector { const base = this.opts.branchDiffBase || sbom?.defaultBranch; if (!base) { console.error(chalk.red(`Cannot compute branch diff for ${fullName} branch ${b.name} because base branch is undefined.`)); continue; } - if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); // Optionally perform dependency submission up front for the branch if (this.opts.forceSubmission) { try { @@ -357,7 +363,7 @@ export class SbomCollector { if (sbom && !sbom.error && this.opts.loadFromDir && this.opts.syncSboms && this.opts.loadFromDir.length) { try { writeOne(sbom, { outDir: this.opts.loadFromDir }); } catch { /* ignore write errors */ } } - + if (sbom) { newSboms.push(sbom); } @@ -382,6 +388,9 @@ export class SbomCollector { if (!this.octokit) throw new Error("No Octokit instance"); try { const resp = await this.octokit.request("GET /repos/{owner}/{repo}/commits", { owner: org, repo, sha: branch }); + + await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); + const commitSha = resp.data?.[0]?.sha; const commitDate = resp.data?.[0]?.commit?.author?.date; return { sha: commitSha, commitDate }; @@ -434,6 +443,9 @@ export class SbomCollector { while (!done) { try { const resp = await this.octokit.request("GET /orgs/{org}/repos", { org, per_page, page, type: this.opts.includePrivate ? "all" : "public" }); + + await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); + const items = resp.data as Array<{ name: string; pushed_at?: string; updated_at?: string; default_branch?: string }>; for (const r of items) { repos.push({ name: r.name, pushed_at: r.pushed_at, updated_at: r.updated_at, default_branch: r.default_branch }); @@ -453,6 +465,9 @@ export class SbomCollector { try { const resp = await this.octokit.request("GET /repos/{owner}/{repo}", { owner: org, repo }); + + await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); + const data = resp.data as { name: string; pushed_at?: string; updated_at?: string; default_branch?: string }; return data; } catch (e) { @@ -466,7 +481,6 @@ export class SbomCollector { const fullName = `${org}/${repo}`; try { - // TODO: Ensure dependency graph is enabled before requesting SBOM const resp = await this.octokit.request("GET /repos/{owner}/{repo}/dependency-graph/sbom", { owner: org, repo, headers: { Accept: "application/vnd.github+json" } }); const sbomWrapper = resp.data as { sbom?: Sbom }; const packages: SbomPackage[] = sbomWrapper?.sbom?.packages ?? []; @@ -505,6 +519,8 @@ export class SbomCollector { while (!done) { try { const resp = await this.octokit.request("GET /repos/{owner}/{repo}/branches", { owner: org, repo, per_page, page }); + await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); + const data = resp.data as Array<{ name: string; protected?: boolean; commit?: { sha?: string } }>; branches.push(...data); if (data.length < per_page) done = true; else page++; @@ -523,6 +539,8 @@ export class SbomCollector { try { const basehead = `${base}...${head}`; const resp = await this.octokit.request("GET /repos/{owner}/{repo}/dependency-graph/compare/{basehead}", { owner: org, repo, basehead, headers: { Accept: "application/vnd.github+json" } }); + await new Promise(r => setTimeout(r, this.opts.lightDelayMs)); + // Response shape includes change_set array (per docs). We normalize to DependencyReviewPackageChange[] const raw = resp.data; From abe838a992d60d5dcc765f84f389bffee8c52174 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 9 Dec 2025 16:44:20 +0000 Subject: [PATCH 105/108] Update CHANGELOG --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af7bc12..0cda4a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [2025-12-04] – 0.2.0 - Branch scanning and dependency submission +## [2025-12-09] – 0.2.0 - Branch scanning and dependency submission Added: @@ -11,6 +11,7 @@ Added: - Automatically submits dependency snapshots for branches being scanned, if not already present, using Component Detection. - Language-aware sparse checkout. - Use a pre-downloaded binary (`--component-detection-bin`) or an auto-downloaded release. + - Allows forcing submission, even if a snapshot already exists. - Search and matching: - Refactored search to de-duplicate logic and include branch diffs (added/updated packages only). - Malware matching enhanced to enumerate packages from diffs; matches annotated with branch. @@ -21,7 +22,7 @@ Added: - JSON/CLI/CSV interaction clarified and documented. - Added examples for malware-only sync and branch scanning. - Advisory sync robustness: - - GraphQL advisory sync now implements adaptive retries with exponential backoff and `Retry-After` support; respects `--quiet`. + - GraphQL advisory sync implements adaptive retries with exponential backoff and `Retry-After` support. Fixed: From 2339190175718cd70fa6fac2c69839cd0a27ce13 Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 9 Dec 2025 16:59:46 +0000 Subject: [PATCH 106/108] Minor README updates --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 899289a..52e5bea 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Supports human-readable, JSON, CSV and SARIF output. SARIF alerts can be uploade - includes Code Scanning upload† - Works with GitHub.com, GitHub Enterprise Server, GitHub Enterprise Managed Users and GitHub Enterprise Cloud with Data Residency (custom base URL) -† GitHub Advanced Security or GitHub Code Security required for this feature +† GitHub Advanced Security/GitHub Code Security required for this feature ## Usage @@ -388,12 +388,12 @@ Then type one PURL query per line. Entering a blank line or using Ctrl+C on a bl | `--token ` | GitHub token; required for `--sync-sboms`, `--sync-malware`, and `--upload-sarif` (or use `GITHUB_TOKEN`) | | `--enterprise ` | Collect across all orgs in an Enterprise (mutually exclusive with `--org`/`--repo` when syncing) | | `--org ` | Single organization scope (mutually exclusive with `--enterprise`/`--repo` when syncing) | -| `--repo ` | Single repository scope (mutually exclusive with `--enterprise`/`--org` when syncing) | +| `--repo ` | Single repository scope in the form `owner/name` (mutually exclusive with `--enterprise`/`--org` when syncing) | | `--base-url ` | GitHub Enterprise Server REST base URL (e.g. `https://ghe.example.com/api/v3`) | | `--concurrency ` | Parallel SBOM fetches (default 5) | | `--sbom-delay ` | Delay between SBOM fetch requests (default 3000) | | `--light-delay ` | Delay between lightweight metadata requests (default 100) | -| `--sbom-cache ` | Directory to read/write per‑repo SBOM JSON; required for offline mode | +| `--sbom-cache ` | Directory to read/write per‑repo SBOM JSON; required for SBOM syncing and offline use | | `--sync-sboms` | Perform API calls to collect SBOMs; without it the CLI runs offline using `--sbom-cache` | | `--progress` | Show a progress bar during SBOM collection | | `--suppress-secondary-rate-limit-logs` | Suppress secondary rate limit warning logs (useful with `--progress`) | @@ -465,7 +465,7 @@ npm run start -- --sbom-cache fixtures/sboms --malware-cache fixtures/malware-ca Standard & secondary rate limits trigger an automatic retry (up to 2 times). -You can tune concurrency and increase the delay to reduce the chance of hitting rate limits. +You can tune concurrency and increase the various delays to reduce the chance of hitting rate limits, if you find that you have hit rate limits. Each time a secondary rate limit is hit, the delay between fetching SBOMs is increased by 10%, to provide a way to adaptively respond to that rate limit. From 810fda3dccf50e67fbb5e42f879d03d4a53f01eb Mon Sep 17 00:00:00 2001 From: aegilops <41705651+aegilops@users.noreply.github.com> Date: Tue, 9 Dec 2025 17:20:45 +0000 Subject: [PATCH 107/108] Encoding URI parts in PURL, removed unneeded code --- src/malwareAdvisories.ts | 4 ++-- src/malwareMatcher.ts | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/malwareAdvisories.ts b/src/malwareAdvisories.ts index 544b01b..2f64679 100644 --- a/src/malwareAdvisories.ts +++ b/src/malwareAdvisories.ts @@ -181,13 +181,13 @@ export class MalwareAdvisorySync { request: agent ? { agent } : undefined }); - this.cachePath = path.join(this.opts.cacheDir, CACHE_FILENAME); + this.cachePath = this.opts.cacheDir ? path.join(this.opts.cacheDir, CACHE_FILENAME) : undefined; this.cache = this.loadCache(); } private loadCache(): MalwareAdvisoryCacheFile { if (!this.cachePath) { - console.warn("No cache path defined; cannot load malware advisory cache."); + console.debug("No cache path defined; cannot load malware advisory cache."); return { schemaVersion: 1, lastSync: new Date(0).toISOString(), advisories: [] }; } try { diff --git a/src/malwareMatcher.ts b/src/malwareMatcher.ts index 90d0fd2..22e9753 100644 --- a/src/malwareMatcher.ts +++ b/src/malwareMatcher.ts @@ -173,13 +173,12 @@ export function matchMalware(advisories: MalwareAdvisoryNode[], sboms: Repositor const branchName = diff.head; for (const change of diff.changes) { if (change.changeType !== 'added' && change.changeType !== 'updated') continue; - let p: string | undefined = (change as { purl?: string }).purl; - if (!p && change.packageURL && change.packageURL.startsWith('pkg:')) p = change.packageURL; + let p = change.packageURL; if (!p && change.ecosystem && change.name && change.version) { if (ecosystemsWithNamespace.has(change.ecosystem) && change.namespace) { - p = `pkg:${change.ecosystem}/${change.namespace}/${change.name}${change.version ? '@' + change.version : ''}`; + p = `pkg:${change.ecosystem}/${encodeURIComponent(change.namespace)}/${encodeURIComponent(change.name)}${change.version ? '@' + change.version : ''}`; } else { - p = `pkg:${change.ecosystem}/${change.name}${change.version ? '@' + change.version : ''}`; + p = `pkg:${change.ecosystem}/${encodeURIComponent(change.name)}${change.version ? '@' + change.version : ''}`; } } if (!p) continue; From 2d3fe41101538b24757001e5424ce84569b2b61a Mon Sep 17 00:00:00 2001 From: Paul Hodgkinson <41705651+aegilops@users.noreply.github.com> Date: Tue, 9 Dec 2025 17:32:31 +0000 Subject: [PATCH 108/108] Update src/cli.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli.ts b/src/cli.ts index db1e87e..831b22c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -301,7 +301,7 @@ async function main() { } } const malwareRows: Array<{ repo: string; purl: string; advisory: string; range: string | null; updatedAt: string; branch: string | undefined }> = []; - if (malwareMatches) { + if (malwareMatches.length) { for (const m of malwareMatches) { malwareRows.push({ repo: m.repo, purl: m.purl, advisory: m.advisoryGhsaId, range: m.vulnerableVersionRange, updatedAt: m.advisoryUpdatedAt, branch: m.branch }); }