mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-16 00:03:32 +00:00
perf(core): coalesce snapshot captures and skip repeat ignore work (#59419)
This commit is contained in:
parent
b3de41a572
commit
cdaba26fa5
2 changed files with 171 additions and 14 deletions
|
|
@ -2,7 +2,7 @@ export * as Git from "./git.js"
|
|||
|
||||
import os from "os"
|
||||
import path from "path"
|
||||
import { Cause, Context, Effect, Exit, Layer, Option, Schedule, Schema } from "effect"
|
||||
import { Cause, Context, Deferred, Effect, Exit, Layer, Option, Schedule, Schema } from "effect"
|
||||
import { ChildProcess } from "effect/process"
|
||||
import { AbsolutePath, RelativePath } from "./schema.js"
|
||||
import { FSUtil } from "@opencode/util/fs-util"
|
||||
|
|
@ -531,7 +531,12 @@ const layer = Layer.effect(
|
|||
* Both commands only read the index, so they never take `index.lock`. Two parallel processes beat one combined
|
||||
* `ls-files -m -o`, whose lstat pass is not threaded like diff-files'.
|
||||
*/
|
||||
const listChanges = Effect.fnUntraced(function* (repository: Repository, scope: RelativePath, index?: string) {
|
||||
const listChanges = Effect.fnUntraced(function* (
|
||||
repository: Repository,
|
||||
scope: RelativePath,
|
||||
index?: string,
|
||||
excludeFrom?: string,
|
||||
) {
|
||||
const list = (args: string[]) =>
|
||||
repositoryOperation("refresh", repository, args, { index }).pipe(
|
||||
// Embedded repositories are listed as `dir/`; update-index records them as gitlinks.
|
||||
|
|
@ -540,7 +545,15 @@ const layer = Layer.effect(
|
|||
const [tracked, untracked] = yield* Effect.all(
|
||||
[
|
||||
list(["diff-files", "--name-only", "-z", "--", literalPathspec(scope)]),
|
||||
list(["ls-files", "--others", "--exclude-standard", "-z", "--", literalPathspec(scope)]),
|
||||
list([
|
||||
"ls-files",
|
||||
"--others",
|
||||
...(excludeFrom ? [`--exclude-from=${excludeFrom}`] : []),
|
||||
"--exclude-standard",
|
||||
"-z",
|
||||
"--",
|
||||
literalPathspec(scope),
|
||||
]),
|
||||
],
|
||||
{ concurrency: 2 },
|
||||
)
|
||||
|
|
@ -745,6 +758,9 @@ const layer = Layer.effect(
|
|||
// A clean capture returns the tree last written from the exact index it scanned.
|
||||
const lastCaptures = new Map<string, { readonly fingerprint: string; readonly tree: TreeID }>()
|
||||
const preparedStores = new Set<string>()
|
||||
const pendingCaptures = new Map<string, Deferred.Deferred<TreeID, OperationError>>()
|
||||
// Untracked files skipped for size, by path, with the stamp they had when skipped.
|
||||
const oversizedFiles = new Map<string, Map<RelativePath, string>>()
|
||||
|
||||
/**
|
||||
* Changes are listed against the temporary index they are written to, never the
|
||||
|
|
@ -752,25 +768,56 @@ const layer = Layer.effect(
|
|||
* therefore exactly its own index plus the worktree changes against it.
|
||||
*/
|
||||
const attemptCapture = Effect.fnUntraced(function* (input: CaptureInput) {
|
||||
const sourceExclude = input.ignores ? path.join(input.ignores.commonDirectory, "info", "exclude") : undefined
|
||||
const excludeFrom = sourceExclude && (yield* fs.existsSafe(sourceExclude)) ? sourceExclude : undefined
|
||||
const result = yield* withTemporaryIndex(input.repository, (index) =>
|
||||
Effect.gen(function* () {
|
||||
const scanned = yield* indexFingerprint(index)
|
||||
const changes = yield* Effect.forEach(input.scopes, (scope) => listChanges(input.repository, scope, index), {
|
||||
concurrency: "unbounded",
|
||||
})
|
||||
const changes = yield* Effect.forEach(
|
||||
input.scopes,
|
||||
(scope) => listChanges(input.repository, scope, index, excludeFrom),
|
||||
{ concurrency: "unbounded" },
|
||||
)
|
||||
const last = lastCaptures.get(input.repository.gitDirectory)
|
||||
const untracked = changes.flatMap((change) => change.untracked)
|
||||
const oversized = oversizedFiles.get(input.repository.gitDirectory)
|
||||
const stillOversized = new Set(
|
||||
oversized
|
||||
? (yield* Effect.forEach(
|
||||
untracked.filter((file) => oversized.has(file)),
|
||||
(file) =>
|
||||
oversizedStamp(input, file).pipe(
|
||||
Effect.map((current) => (current && current === oversized.get(file) ? file : undefined)),
|
||||
),
|
||||
{ concurrency: 8 },
|
||||
)).filter((file): file is RelativePath => file !== undefined)
|
||||
: [],
|
||||
)
|
||||
// Paths the source ignores are listed as untracked on every scan; they leave the index unchanged.
|
||||
const excluded =
|
||||
scanned && last?.fingerprint === scanned && changes.every((change) => !change.tracked.length)
|
||||
? input.ignores
|
||||
? yield* ignored({ repository: input.ignores, paths: untracked })
|
||||
? yield* ignored({
|
||||
repository: input.ignores,
|
||||
paths: untracked.filter((file) => !stillOversized.has(file)),
|
||||
})
|
||||
: new Set<RelativePath>()
|
||||
: undefined
|
||||
if (last && excluded && untracked.every((file) => excluded.has(file))) return last.tree
|
||||
yield* Effect.forEach(changes, (change) => updateIndex({ ...input, changes: change, index, excluded }), {
|
||||
discard: true,
|
||||
})
|
||||
if (last && excluded && untracked.every((file) => excluded.has(file) || stillOversized.has(file)))
|
||||
return last.tree
|
||||
const skipped = (yield* Effect.forEach(changes, (change) =>
|
||||
updateIndex({ ...input, changes: change, index, excluded }),
|
||||
)).flatMap((update) => update.skipped)
|
||||
oversizedFiles.set(
|
||||
input.repository.gitDirectory,
|
||||
new Map(
|
||||
(yield* Effect.forEach(
|
||||
skipped,
|
||||
(file) => oversizedStamp(input, file).pipe(Effect.map((value) => [file, value] as const)),
|
||||
{ concurrency: 8 },
|
||||
)).filter((entry): entry is readonly [RelativePath, string] => entry[1] !== undefined),
|
||||
),
|
||||
)
|
||||
return yield* writeTree(input.repository, index)
|
||||
}),
|
||||
)
|
||||
|
|
@ -780,10 +827,53 @@ const layer = Layer.effect(
|
|||
return result.value
|
||||
})
|
||||
|
||||
const captureTree = Effect.fn("Git.tree.capture")((input: CaptureInput) =>
|
||||
const oversizedStamp = (input: CaptureInput, file: RelativePath) =>
|
||||
fs.stat(path.join(input.repository.worktree, file)).pipe(
|
||||
Effect.map((info) =>
|
||||
input.maximumUntrackedFileBytes && info.type === "File" && Number(info.size) > input.maximumUntrackedFileBytes
|
||||
? `${info.size}:${Option.getOrUndefined(info.mtime)?.getTime()}`
|
||||
: undefined,
|
||||
),
|
||||
Effect.orElseSucceed(() => undefined),
|
||||
)
|
||||
|
||||
/**
|
||||
* Callers queued behind a running capture share the next one, which starts after they all asked, so it still
|
||||
* sees every change made before each call.
|
||||
*/
|
||||
const captureTree = Effect.fn("Git.tree.capture")(function* (input: CaptureInput) {
|
||||
const key = JSON.stringify([
|
||||
input.repository.gitDirectory,
|
||||
input.repository.worktree,
|
||||
input.scopes,
|
||||
input.ignores?.gitDirectory,
|
||||
input.maximumUntrackedFileBytes,
|
||||
])
|
||||
const pending = pendingCaptures.get(key)
|
||||
if (pending) {
|
||||
const exit = yield* Effect.exit(Deferred.await(pending))
|
||||
if (Exit.isSuccess(exit) || !Cause.hasInterruptsOnly(exit.cause)) return yield* exit
|
||||
}
|
||||
const deferred = Deferred.makeUnsafe<TreeID, OperationError>()
|
||||
const release = () => {
|
||||
if (pendingCaptures.get(key) === deferred) pendingCaptures.delete(key)
|
||||
}
|
||||
return yield* Effect.uninterruptibleMask((restore) => {
|
||||
pendingCaptures.set(key, deferred)
|
||||
return restore(captureLocked(input, release)).pipe(
|
||||
Effect.onExit((exit) => {
|
||||
release()
|
||||
return Deferred.done(deferred, exit)
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
const captureLocked = (input: CaptureInput, onStart: () => void) =>
|
||||
locked(
|
||||
input.repository,
|
||||
Effect.gen(function* () {
|
||||
onStart()
|
||||
if (!preparedStores.has(input.repository.gitDirectory)) {
|
||||
preparedStores.add(input.repository.gitDirectory)
|
||||
yield* sweepTemporaryIndexes(input.repository)
|
||||
|
|
@ -803,8 +893,7 @@ const layer = Layer.effect(
|
|||
),
|
||||
)
|
||||
}),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
/**
|
||||
* Only an index Git itself cannot read is rebuilt, decided by exit status rather
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import { AbsolutePath, RelativePath } from "@opencode/core/schema"
|
|||
import { VcsPatch } from "@opencode/core/vcs/patch"
|
||||
import { branch, commit, initGit, initRepo, read, withRemote } from "./fixture/git"
|
||||
import { tmpdir } from "./fixture/tmpdir"
|
||||
import { withEnv } from "./fixture/env"
|
||||
import { testEffect } from "./lib/effect"
|
||||
|
||||
const it = testEffect(LayerNode.compile(Git.node))
|
||||
|
|
@ -519,8 +520,75 @@ describe("Git capture", () => {
|
|||
)
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("queued captures share one capture that still sees changes made before the call", () =>
|
||||
Effect.gen(function* () {
|
||||
const { root, project, repository, service, capture } = yield* captureSetup
|
||||
const before = yield* capture()
|
||||
yield* Effect.promise(() => Bun.write(path.join(project, "a.txt"), "changed\n"))
|
||||
const trace = path.join(root.path, "trace")
|
||||
const trees = yield* withEnv({ GIT_TRACE: trace }, () =>
|
||||
Effect.all(Array.from({ length: 4 }, capture), { concurrency: "unbounded" }),
|
||||
)
|
||||
expect(new Set(trees).size).toBe(1)
|
||||
expect(yield* service.tree.files({ repository, from: before, to: trees[0]! })).toEqual([
|
||||
RelativePath.make("a.txt"),
|
||||
])
|
||||
expect((yield* traced(trace)).filter((command) => command === "ls-files").length).toBeLessThanOrEqual(2)
|
||||
}),
|
||||
)
|
||||
|
||||
it.live("skips repeat ignore checks for oversized and info/exclude-only untracked files", () =>
|
||||
Effect.gen(function* () {
|
||||
const { root, project, capture } = yield* captureSetup
|
||||
yield* Effect.promise(async () => {
|
||||
await Bun.write(path.join(project, ".git", "info", "exclude"), "excluded.txt\n")
|
||||
await Bun.write(path.join(project, "excluded.txt"), "x\n")
|
||||
await Bun.write(path.join(project, "large.txt"), "too large\n")
|
||||
})
|
||||
const first = yield* capture({ maximumUntrackedFileBytes: 4 })
|
||||
const trace = path.join(root.path, "trace")
|
||||
expect(yield* withEnv({ GIT_TRACE: trace }, () => capture({ maximumUntrackedFileBytes: 4 }))).toBe(first)
|
||||
expect((yield* traced(trace)).toSorted()).toEqual(["diff-files", "ls-files"])
|
||||
const entries = yield* Effect.promise(() =>
|
||||
$`git --git-dir ${root.path}/storage ls-tree --name-only ${first}`.text(),
|
||||
)
|
||||
expect(entries.split("\n").filter(Boolean)).toEqual(["a.txt", "b.txt"])
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
const captureSetup = Effect.gen(function* () {
|
||||
const root = yield* Effect.acquireRelease(
|
||||
Effect.promise(() => tmpdir()),
|
||||
(dir) => Effect.promise(() => dir[Symbol.asyncDispose]()),
|
||||
)
|
||||
const project = path.join(root.path, "project")
|
||||
yield* Effect.promise(async () => {
|
||||
await fs.mkdir(project)
|
||||
await initRepo(project)
|
||||
await Bun.write(path.join(project, "a.txt"), "a\n")
|
||||
await Bun.write(path.join(project, "b.txt"), "b\n")
|
||||
await $`git add . && git commit -q -m initial`.cwd(project).quiet()
|
||||
})
|
||||
const service = yield* Git.Service
|
||||
const source = yield* service.repo.discover(AbsolutePath.make(project))
|
||||
if (!source) throw new Error("Repository not found")
|
||||
const repository = yield* service.repo.create({
|
||||
worktree: source.worktree,
|
||||
gitDirectory: AbsolutePath.make(path.join(root.path, "storage")),
|
||||
seed: source,
|
||||
})
|
||||
const capture = (options?: { maximumUntrackedFileBytes?: number }) =>
|
||||
service.tree.capture({ repository, scopes: [RelativePath.make(".")], ignores: source, ...options })
|
||||
return { root, project, repository, service, capture }
|
||||
})
|
||||
|
||||
const traced = (file: string) =>
|
||||
Effect.promise(async () =>
|
||||
(await Bun.file(file).text()).split("\n").flatMap((line) => line.match(/trace: built-in: git (\S+)/)?.[1] ?? []),
|
||||
)
|
||||
|
||||
describe("Git capture recovery", () => {
|
||||
const setup = Effect.gen(function* () {
|
||||
const root = yield* Effect.acquireRelease(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue