92 lines · 4.6 KB
import { loadRepoById } from "../repos";
import { diffTreesBounded } from "../git/diff";
import { ZERO_SHA, withStore } from "../git/store";
import type { GitStore } from "../git/store";
import { listRefs } from "../git/refs";
import { compareCommits } from "../git/history";
import { redactFindings, scannableText, scanText } from "./detectors";
import { recordSecretAlerts } from "./secret-alerts";
import type { LocatedFinding } from "./secret-alerts";
import { loadSecuritySettings } from "./settings";

const MAX_SCAN_FILES = 1000;
const MAX_SCAN_COMMITS = 50;
const RISKY_NAMES =
  /(^|\/)(\.env[^/]*|[^/]*\.(pem|key|p12|pfx|json|ya?ml|toml|ini|cfg|conf|properties|tf|tfvars|sh|py|js|ts|rb|go)|[^/]*(secret|credential|token|password|config)[^/]*)$/i;
const MAX_BLOB_BYTES = 1024 * 1024;
const MAX_TOTAL_BYTES = 16 * 1024 * 1024;
const BLOB_CONCURRENCY = 8;
const SKIPPED_EXTENSIONS = /\.(png|jpe?g|gif|webp|ico|pdf|zip|gz|tgz|jar|woff2?|ttf|otf|mp[34]|mov|wasm)$/i;

type Candidate = { path: string; sha: string };

async function candidatesFor(store: GitStore, baseTree: string | null, headTree: string): Promise<Candidate[]> {
  if (baseTree) {
    const { changes } = await diffTreesBounded(store, baseTree, headTree);
    return changes
      .filter((change) => change.newSha && change.status !== "deleted")
      .map((change) => ({ path: change.path, sha: change.newSha ?? "" }));
  }
  const { entries } = await store.walk(headTree, MAX_SCAN_FILES * 4);
  return entries.filter((entry) => entry.type === "blob").map((entry) => ({ path: entry.path, sha: entry.sha }));
}

/**
 * Blobs introduced by every commit in `base..head` (bounded), not just the tip diff, so a secret added and
 * removed within one push is still seen. Deduplicated by blob, with likely secret-bearing paths first.
 */
async function pushCandidates(store: GitStore, baseSha: string | null, headSha: string, headTree: string) {
  const commits = baseSha
    ? (await compareCommits(store, baseSha, store, headSha)).commits.slice(0, MAX_SCAN_COMMITS)