perf(core): coalesce snapshot captures and skip repeat ignore work (#59419)

This commit is contained in:
Aiden Cline 2026-10-27 12:29:17 -05:00 • committed by GitHub
parent b3de41a572
commit cdaba26fa5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 171 additions and 14 deletions

View file

@ -2,7 +2,7 @@ export * as Git from "./git.js"
import os from "os"
import path from "path"
import { Cause, Context, Effect, Exit, Layer, Option, Schedule, Schema } from "effect"
import { Cause, Context, Deferred, Effect, Exit, Layer, Option, Schedule, Schema } from "effect"
import { ChildProcess } from "effect/process"
import { AbsolutePath, RelativePath } from "./schema.js"
import { FSUtil } from "@opencode/util/fs-util"
@ -531,7 +531,12 @@ const layer = Layer.effect(
* Both commands only read the index, so they never take `index.lock`. Two parallel processes beat one combined
* `ls-files -m -o`, whose lstat pass is not threaded like diff-files'.
*/
const listChanges = Effect.fnUntraced(function* (repository: Repository, scope: RelativePath, index?: string) {
const listChanges = Effect.fnUntraced(function* (
repository: Repository,
scope: RelativePath,
index?: string,
excludeFrom?: string,
) {
const list = (args: string[]) =>
repositoryOperation("refresh", repository, args, { index }).pipe(
// Embedded repositories are listed as `dir/`; update-index records them as gitlinks.
@ -540,7 +545,15 @@ const layer = Layer.effect(
const [tracked, untracked] = yield* Effect.all(
[
list(["diff-files", "--name-only", "-z", "--", literalPathspec(scope)]),
list(["ls-files", "--others", "--exclude-standard", "-z", "--", literalPathspec(scope)]),
list([
"ls-files",
"--others",
...(excludeFrom ? [`--exclude-from=${excludeFrom}`] : []),
"--exclude-standard",
"-z",
"--",
literalPathspec(scope),
]),
],
{ concurrency: 2 },
)
@ -745,6 +758,9 @@ const layer = Layer.effect(
// A clean capture returns the tree last written from the exact index it scanned.
const lastCaptures = new Map<string, { readonly fingerprint: string; readonly tree: TreeID }>()
const preparedStores = new Set<string>()
const pendingCaptures = new Map<string, Deferred.Deferred<TreeID, OperationError>>()
// Untracked files skipped for size, by path, with the stamp they had when skipped.
const oversizedFiles = new Map<string, Map<RelativePath, string>>()
/**
* Changes are listed against the temporary index they are written to, never the
@ -752,25 +768,56 @@ const layer = Layer.effect(
* therefore exactly its own index plus the worktree changes against it.
*/
const attemptCapture = Effect.fnUntraced(function* (input: CaptureInput) {
const sourceExclude = input.ignores ? path.join(input.ignores.commonDirectory, "info", "exclude") : undefined
const excludeFrom = sourceExclude && (yield* fs.existsSafe(sourceExclude)) ? sourceExclude : undefined
const result = yield* withTemporaryIndex(input.repository, (index) =>
Effect.gen(function* () {
const scanned = yield* indexFingerprint(index)
const changes = yield* Effect.forEach(input.scopes, (scope) => listChanges(input.repository, scope, index), {
concurrency: "unbounded",
})
const changes = yield* Effect.forEach(
input.scopes,
(scope) => listChanges(input.repository, scope, index, excludeFrom),
{ concurrency: "unbounded" },
)
const last = lastCaptures.get(input.repository.gitDirectory)
const untracked = changes.flatMap((change) => change.untracked)
const oversized = oversizedFiles.get(input.repository.gitDirectory)
const stillOversized = new Set(
oversized
? (yield* Effect.forEach(
untracked.filter((file) => oversized.has(file)),
(file) =>
oversizedStamp(input, file).pipe(
Effect.map((current) => (current && current === oversized.get(file) ? file : undefined)),
),
{ concurrency: 8 },
)).filter((file): file is RelativePath => file !== undefined)
: [],
)
// Paths the source ignores are listed as untracked on every scan; they leave the index unchanged.
const excluded =
scanned && last?.fingerprint === scanned && changes.every((change) => !change.tracked.length)
? input.ignores
? yield* ignored({ repository: input.ignores, paths: untracked })
? yield* ignored({
repository: input.ignores,
paths: untracked.filter((file) => !stillOversized.has(file)),
})
: new Set<RelativePath>()
: undefined
if (last && excluded && untracked.every((file) => excluded.has(file))) return last.tree
yield* Effect.forEach(changes, (change) => updateIndex({ ...input, changes: change, index, excluded }), {
discard: true,
})
if (last && excluded && untracked.every((file) => excluded.has(file) || stillOversized.has(file)))
return last.tree
const skipped = (yield* Effect.forEach(changes, (change) =>
updateIndex({ ...input, changes: change, index, excluded }),
)).flatMap((update) => update.skipped)
oversizedFiles.set(
input.repository.gitDirectory,
new Map(
(yield* Effect.forEach(
skipped,
(file) => oversizedStamp(input, file).pipe(Effect.map((value) => [file, value] as const)),
{ concurrency: 8 },
)).filter((entry): entry is readonly [RelativePath, string] => entry[1] !== undefined),
),
)
return yield* writeTree(input.repository, index)
}),
)
@ -780,10 +827,53 @@ const layer = Layer.effect(
return result.value
})
const captureTree = Effect.fn("Git.tree.capture")((input: CaptureInput) =>
const oversizedStamp = (input: CaptureInput, file: RelativePath) =>
fs.stat(path.join(input.repository.worktree, file)).pipe(
Effect.map((info) =>
input.maximumUntrackedFileBytes && info.type === "File" && Number(info.size) > input.maximumUntrackedFileBytes
? `${info.size}:${Option.getOrUndefined(info.mtime)?.getTime()}`
: undefined,
),
Effect.orElseSucceed(() => undefined),
)
/**
* Callers queued behind a running capture share the next one, which starts after they all asked, so it still
* sees every change made before each call.
*/
const captureTree = Effect.fn("Git.tree.capture")(function* (input: CaptureInput) {
const key = JSON.stringify([
input.repository.gitDirectory,
input.repository.worktree,
input.scopes,
input.ignores?.gitDirectory,
input.maximumUntrackedFileBytes,
])
const pending = pendingCaptures.get(key)
if (pending) {
const exit = yield* Effect.exit(Deferred.await(pending))
if (Exit.isSuccess(exit) || !Cause.hasInterruptsOnly(exit.cause)) return yield* exit
}
const deferred = Deferred.makeUnsafe<TreeID, OperationError>()
const release = () => {
if (pendingCaptures.get(key) === deferred) pendingCaptures.delete(key)
}
return yield* Effect.uninterruptibleMask((restore) => {
pendingCaptures.set(key, deferred)
return restore(captureLocked(input, release)).pipe(
Effect.onExit((exit) => {
release()
return Deferred.done(deferred, exit)
}),
)
})
})
const captureLocked = (input: CaptureInput, onStart: () => void) =>
locked(
input.repository,
Effect.gen(function* () {
onStart()
if (!preparedStores.has(input.repository.gitDirectory)) {
preparedStores.add(input.repository.gitDirectory)
yield* sweepTemporaryIndexes(input.repository)
@ -803,8 +893,7 @@ const layer = Layer.effect(
),
)
}),
),
)
)
/**
* Only an index Git itself cannot read is rebuilt, decided by exit status rather

View file

@ -9,6 +9,7 @@ import { AbsolutePath, RelativePath } from "@opencode/core/schema"
import { VcsPatch } from "@opencode/core/vcs/patch"
import { branch, commit, initGit, initRepo, read, withRemote } from "./fixture/git"
import { tmpdir } from "./fixture/tmpdir"
import { withEnv } from "./fixture/env"
import { testEffect } from "./lib/effect"
const it = testEffect(LayerNode.compile(Git.node))
@ -519,8 +520,75 @@ describe("Git capture", () => {
)
}),
)
it.live("queued captures share one capture that still sees changes made before the call", () =>
Effect.gen(function* () {
const { root, project, repository, service, capture } = yield* captureSetup
const before = yield* capture()
yield* Effect.promise(() => Bun.write(path.join(project, "a.txt"), "changed\n"))
const trace = path.join(root.path, "trace")
const trees = yield* withEnv({ GIT_TRACE: trace }, () =>
Effect.all(Array.from({ length: 4 }, capture), { concurrency: "unbounded" }),
)
expect(new Set(trees).size).toBe(1)
expect(yield* service.tree.files({ repository, from: before, to: trees[0]! })).toEqual([
RelativePath.make("a.txt"),
])
expect((yield* traced(trace)).filter((command) => command === "ls-files").length).toBeLessThanOrEqual(2)
}),
)
it.live("skips repeat ignore checks for oversized and info/exclude-only untracked files", () =>
Effect.gen(function* () {
const { root, project, capture } = yield* captureSetup
yield* Effect.promise(async () => {
await Bun.write(path.join(project, ".git", "info", "exclude"), "excluded.txt\n")
await Bun.write(path.join(project, "excluded.txt"), "x\n")
await Bun.write(path.join(project, "large.txt"), "too large\n")
})
const first = yield* capture({ maximumUntrackedFileBytes: 4 })
const trace = path.join(root.path, "trace")
expect(yield* withEnv({ GIT_TRACE: trace }, () => capture({ maximumUntrackedFileBytes: 4 }))).toBe(first)
expect((yield* traced(trace)).toSorted()).toEqual(["diff-files", "ls-files"])
const entries = yield* Effect.promise(() =>
$`git --git-dir ${root.path}/storage ls-tree --name-only ${first}`.text(),
)
expect(entries.split("\n").filter(Boolean)).toEqual(["a.txt", "b.txt"])
}),
)
})
const captureSetup = Effect.gen(function* () {
const root = yield* Effect.acquireRelease(
Effect.promise(() => tmpdir()),
(dir) => Effect.promise(() => dir[Symbol.asyncDispose]()),
)
const project = path.join(root.path, "project")
yield* Effect.promise(async () => {
await fs.mkdir(project)
await initRepo(project)
await Bun.write(path.join(project, "a.txt"), "a\n")
await Bun.write(path.join(project, "b.txt"), "b\n")
await $`git add . && git commit -q -m initial`.cwd(project).quiet()
})
const service = yield* Git.Service
const source = yield* service.repo.discover(AbsolutePath.make(project))
if (!source) throw new Error("Repository not found")
const repository = yield* service.repo.create({
worktree: source.worktree,
gitDirectory: AbsolutePath.make(path.join(root.path, "storage")),
seed: source,
})
const capture = (options?: { maximumUntrackedFileBytes?: number }) =>
service.tree.capture({ repository, scopes: [RelativePath.make(".")], ignores: source, ...options })
return { root, project, repository, service, capture }
})
const traced = (file: string) =>
Effect.promise(async () =>
(await Bun.file(file).text()).split("\n").flatMap((line) => line.match(/trace: built-in: git (\S+)/)?.[1] ?? []),
)
describe("Git capture recovery", () => {
const setup = Effect.gen(function* () {
const root = yield* Effect.acquireRelease(