From cdaba26fa5d2c600d4c226d1c8a5f2837c16bf8e Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:29:17 -0500 Subject: [PATCH] perf(core): coalesce snapshot captures and skip repeat ignore work (#59419) --- packages/core/src/git.ts | 117 +++++++++++++++++++++++++++++---- packages/core/test/git.test.ts | 68 +++++++++++++++++++ 2 files changed, 171 insertions(+), 14 deletions(-) diff --git a/packages/core/src/git.ts b/packages/core/src/git.ts index 8a5a1b0d344f..15ba8ada7dcb 100644 --- a/packages/core/src/git.ts +++ b/packages/core/src/git.ts @@ -2,7 +2,7 @@ export * as Git from "./git.js" import os from "os" import path from "path" -import { Cause, Context, Effect, Exit, Layer, Option, Schedule, Schema } from "effect" +import { Cause, Context, Deferred, Effect, Exit, Layer, Option, Schedule, Schema } from "effect" import { ChildProcess } from "effect/process" import { AbsolutePath, RelativePath } from "./schema.js" import { FSUtil } from "@opencode/util/fs-util" @@ -531,7 +531,12 @@ const layer = Layer.effect( * Both commands only read the index, so they never take `index.lock`. Two parallel processes beat one combined * `ls-files -m -o`, whose lstat pass is not threaded like diff-files'. */ - const listChanges = Effect.fnUntraced(function* (repository: Repository, scope: RelativePath, index?: string) { + const listChanges = Effect.fnUntraced(function* ( + repository: Repository, + scope: RelativePath, + index?: string, + excludeFrom?: string, + ) { const list = (args: string[]) => repositoryOperation("refresh", repository, args, { index }).pipe( // Embedded repositories are listed as `dir/`; update-index records them as gitlinks. @@ -540,7 +545,15 @@ const layer = Layer.effect( const [tracked, untracked] = yield* Effect.all( [ list(["diff-files", "--name-only", "-z", "--", literalPathspec(scope)]), - list(["ls-files", "--others", "--exclude-standard", "-z", "--", literalPathspec(scope)]), + list([ + "ls-files", + "--others", + ...(excludeFrom ? [`--exclude-from=${excludeFrom}`] : []), + "--exclude-standard", + "-z", + "--", + literalPathspec(scope), + ]), ], { concurrency: 2 }, ) @@ -745,6 +758,9 @@ const layer = Layer.effect( // A clean capture returns the tree last written from the exact index it scanned. const lastCaptures = new Map() const preparedStores = new Set() + const pendingCaptures = new Map>() + // Untracked files skipped for size, by path, with the stamp they had when skipped. + const oversizedFiles = new Map>() /** * Changes are listed against the temporary index they are written to, never the @@ -752,25 +768,56 @@ const layer = Layer.effect( * therefore exactly its own index plus the worktree changes against it. */ const attemptCapture = Effect.fnUntraced(function* (input: CaptureInput) { + const sourceExclude = input.ignores ? path.join(input.ignores.commonDirectory, "info", "exclude") : undefined + const excludeFrom = sourceExclude && (yield* fs.existsSafe(sourceExclude)) ? sourceExclude : undefined const result = yield* withTemporaryIndex(input.repository, (index) => Effect.gen(function* () { const scanned = yield* indexFingerprint(index) - const changes = yield* Effect.forEach(input.scopes, (scope) => listChanges(input.repository, scope, index), { - concurrency: "unbounded", - }) + const changes = yield* Effect.forEach( + input.scopes, + (scope) => listChanges(input.repository, scope, index, excludeFrom), + { concurrency: "unbounded" }, + ) const last = lastCaptures.get(input.repository.gitDirectory) const untracked = changes.flatMap((change) => change.untracked) + const oversized = oversizedFiles.get(input.repository.gitDirectory) + const stillOversized = new Set( + oversized + ? (yield* Effect.forEach( + untracked.filter((file) => oversized.has(file)), + (file) => + oversizedStamp(input, file).pipe( + Effect.map((current) => (current && current === oversized.get(file) ? file : undefined)), + ), + { concurrency: 8 }, + )).filter((file): file is RelativePath => file !== undefined) + : [], + ) // Paths the source ignores are listed as untracked on every scan; they leave the index unchanged. const excluded = scanned && last?.fingerprint === scanned && changes.every((change) => !change.tracked.length) ? input.ignores - ? yield* ignored({ repository: input.ignores, paths: untracked }) + ? yield* ignored({ + repository: input.ignores, + paths: untracked.filter((file) => !stillOversized.has(file)), + }) : new Set() : undefined - if (last && excluded && untracked.every((file) => excluded.has(file))) return last.tree - yield* Effect.forEach(changes, (change) => updateIndex({ ...input, changes: change, index, excluded }), { - discard: true, - }) + if (last && excluded && untracked.every((file) => excluded.has(file) || stillOversized.has(file))) + return last.tree + const skipped = (yield* Effect.forEach(changes, (change) => + updateIndex({ ...input, changes: change, index, excluded }), + )).flatMap((update) => update.skipped) + oversizedFiles.set( + input.repository.gitDirectory, + new Map( + (yield* Effect.forEach( + skipped, + (file) => oversizedStamp(input, file).pipe(Effect.map((value) => [file, value] as const)), + { concurrency: 8 }, + )).filter((entry): entry is readonly [RelativePath, string] => entry[1] !== undefined), + ), + ) return yield* writeTree(input.repository, index) }), ) @@ -780,10 +827,53 @@ const layer = Layer.effect( return result.value }) - const captureTree = Effect.fn("Git.tree.capture")((input: CaptureInput) => + const oversizedStamp = (input: CaptureInput, file: RelativePath) => + fs.stat(path.join(input.repository.worktree, file)).pipe( + Effect.map((info) => + input.maximumUntrackedFileBytes && info.type === "File" && Number(info.size) > input.maximumUntrackedFileBytes + ? `${info.size}:${Option.getOrUndefined(info.mtime)?.getTime()}` + : undefined, + ), + Effect.orElseSucceed(() => undefined), + ) + + /** + * Callers queued behind a running capture share the next one, which starts after they all asked, so it still + * sees every change made before each call. + */ + const captureTree = Effect.fn("Git.tree.capture")(function* (input: CaptureInput) { + const key = JSON.stringify([ + input.repository.gitDirectory, + input.repository.worktree, + input.scopes, + input.ignores?.gitDirectory, + input.maximumUntrackedFileBytes, + ]) + const pending = pendingCaptures.get(key) + if (pending) { + const exit = yield* Effect.exit(Deferred.await(pending)) + if (Exit.isSuccess(exit) || !Cause.hasInterruptsOnly(exit.cause)) return yield* exit + } + const deferred = Deferred.makeUnsafe() + const release = () => { + if (pendingCaptures.get(key) === deferred) pendingCaptures.delete(key) + } + return yield* Effect.uninterruptibleMask((restore) => { + pendingCaptures.set(key, deferred) + return restore(captureLocked(input, release)).pipe( + Effect.onExit((exit) => { + release() + return Deferred.done(deferred, exit) + }), + ) + }) + }) + + const captureLocked = (input: CaptureInput, onStart: () => void) => locked( input.repository, Effect.gen(function* () { + onStart() if (!preparedStores.has(input.repository.gitDirectory)) { preparedStores.add(input.repository.gitDirectory) yield* sweepTemporaryIndexes(input.repository) @@ -803,8 +893,7 @@ const layer = Layer.effect( ), ) }), - ), - ) + ) /** * Only an index Git itself cannot read is rebuilt, decided by exit status rather diff --git a/packages/core/test/git.test.ts b/packages/core/test/git.test.ts index 4c5b3c40ed44..f74e9915ba1f 100644 --- a/packages/core/test/git.test.ts +++ b/packages/core/test/git.test.ts @@ -9,6 +9,7 @@ import { AbsolutePath, RelativePath } from "@opencode/core/schema" import { VcsPatch } from "@opencode/core/vcs/patch" import { branch, commit, initGit, initRepo, read, withRemote } from "./fixture/git" import { tmpdir } from "./fixture/tmpdir" +import { withEnv } from "./fixture/env" import { testEffect } from "./lib/effect" const it = testEffect(LayerNode.compile(Git.node)) @@ -519,8 +520,75 @@ describe("Git capture", () => { ) }), ) + + it.live("queued captures share one capture that still sees changes made before the call", () => + Effect.gen(function* () { + const { root, project, repository, service, capture } = yield* captureSetup + const before = yield* capture() + yield* Effect.promise(() => Bun.write(path.join(project, "a.txt"), "changed\n")) + const trace = path.join(root.path, "trace") + const trees = yield* withEnv({ GIT_TRACE: trace }, () => + Effect.all(Array.from({ length: 4 }, capture), { concurrency: "unbounded" }), + ) + expect(new Set(trees).size).toBe(1) + expect(yield* service.tree.files({ repository, from: before, to: trees[0]! })).toEqual([ + RelativePath.make("a.txt"), + ]) + expect((yield* traced(trace)).filter((command) => command === "ls-files").length).toBeLessThanOrEqual(2) + }), + ) + + it.live("skips repeat ignore checks for oversized and info/exclude-only untracked files", () => + Effect.gen(function* () { + const { root, project, capture } = yield* captureSetup + yield* Effect.promise(async () => { + await Bun.write(path.join(project, ".git", "info", "exclude"), "excluded.txt\n") + await Bun.write(path.join(project, "excluded.txt"), "x\n") + await Bun.write(path.join(project, "large.txt"), "too large\n") + }) + const first = yield* capture({ maximumUntrackedFileBytes: 4 }) + const trace = path.join(root.path, "trace") + expect(yield* withEnv({ GIT_TRACE: trace }, () => capture({ maximumUntrackedFileBytes: 4 }))).toBe(first) + expect((yield* traced(trace)).toSorted()).toEqual(["diff-files", "ls-files"]) + const entries = yield* Effect.promise(() => + $`git --git-dir ${root.path}/storage ls-tree --name-only ${first}`.text(), + ) + expect(entries.split("\n").filter(Boolean)).toEqual(["a.txt", "b.txt"]) + }), + ) }) +const captureSetup = Effect.gen(function* () { + const root = yield* Effect.acquireRelease( + Effect.promise(() => tmpdir()), + (dir) => Effect.promise(() => dir[Symbol.asyncDispose]()), + ) + const project = path.join(root.path, "project") + yield* Effect.promise(async () => { + await fs.mkdir(project) + await initRepo(project) + await Bun.write(path.join(project, "a.txt"), "a\n") + await Bun.write(path.join(project, "b.txt"), "b\n") + await $`git add . && git commit -q -m initial`.cwd(project).quiet() + }) + const service = yield* Git.Service + const source = yield* service.repo.discover(AbsolutePath.make(project)) + if (!source) throw new Error("Repository not found") + const repository = yield* service.repo.create({ + worktree: source.worktree, + gitDirectory: AbsolutePath.make(path.join(root.path, "storage")), + seed: source, + }) + const capture = (options?: { maximumUntrackedFileBytes?: number }) => + service.tree.capture({ repository, scopes: [RelativePath.make(".")], ignores: source, ...options }) + return { root, project, repository, service, capture } +}) + +const traced = (file: string) => + Effect.promise(async () => + (await Bun.file(file).text()).split("\n").flatMap((line) => line.match(/trace: built-in: git (\S+)/)?.[1] ?? []), + ) + describe("Git capture recovery", () => { const setup = Effect.gen(function* () { const root = yield* Effect.acquireRelease(