419 lines · 16.7 KB
import { DurableObject, env } from "cloudflare:workers";
import { loadRepoById } from "../repos";
import { branchSha, listRefs } from "../git/refs";
import { withStore } from "../git/store";
import type { GitStore } from "../git/store";
import { diffTreesBounded } from "../git/diff";
import { run } from "../db";
import { MAX_POINTER_BYTES, parseLfsPointer } from "../lfs/pointer";
import { parseAttributes } from "~/lib/gitattributes";
import { languageOf, summarizeLanguages } from "~/lib/languages";
import { CODE_PAGE_SIZE, matchingLines } from "~/lib/code-search";
import type { CodeSearchLine } from "~/lib/code-intel";
import { compileQuery, matchLines, requiredLiteral } from "~/lib/text-search";
import type { LineMatch, TextQuery } from "~/lib/text-search";

const FILES_PER_PASS = 100;
const MAX_INDEXED_FILES = 5000;
const MAX_INDEXED_BYTES = 128 * 1024;
/** Files whose sizes feed the language breakdown (any text file, indexed or not). */
const MAX_SIZED_FILES = 20_000;
const MAX_ATTRIBUTES_BYTES = 64 * 1024;
const MAX_ATTRIBUTE_RULES = 500;
/** Bumped when the index gains data (2: file sizes for languages); an older index is rebuilt once. */
const INDEX_SCHEMA = 2;
const BINARY_SNIFF = 8000;
const READ_CONCURRENCY = 8;
const RETRY_ON_ERROR_MS = 60_000;
/** Counting stops here so a broad query stays cheap. */
const COUNT_CAP = 1000;
const LINES_PER_HIT = 3;
const SNIPPET_TOKENS = 24;
const SNIPPET = `snippet(code_fts, 1, char(1), char(2), '…', ${SNIPPET_TOKENS})`;
const SKIPPED_MODES = new Set(["120000", "160000"]);
const decoder = new TextDecoder("utf-8", { fatal: true });

const STATE_KEY = "state";
/** The language summary last written to the repository row, so unchanged ones skip the write. */
const LANGUAGES_KEY = "languages";
const JOB_KEY = "job";