Improve performance with very long diffs (#6121)

Parsing long diffs was very slow, especially for commits that add or
remove files with hundreds of thousands of lines. This caused very
noticeable lags for certain operations such as scrolling down in such a
diff using `>`, or bringing up the `<ctrl-g>` files menu.
This commit is contained in:
Stefan Haller
2026-10-14 09:48:15 +02:00
committed by GitHub
13 changed files with 271 additions and 33 deletions
+29
View File
@@ -183,6 +183,35 @@ func (self *Patch) OldLineNumberOfLine(idx int) int {
return hunk.oldStart + offset
}
// LineNumbersOfLines returns what LineNumberOfLine and OldLineNumberOfLine return for
// each line of the patch. It numbers all of them in one pass, whereas calling those
// two for every line takes time quadratic in the length of a hunk.
func (self *Patch) LineNumbersOfLines() (newLineNumbers []int, oldLineNumbers []int) {
newLineNumbers = make([]int, 0, self.LineCount())
oldLineNumbers = make([]int, 0, self.LineCount())
for range self.header {
newLineNumbers = append(newLineNumbers, 1)
oldLineNumbers = append(oldLineNumbers, 1)
}
for _, hunk := range self.hunks {
newLineNumber, oldLineNumber := hunk.newStart, hunk.oldStart
// The hunk's header line has the numbers of the hunk's first line.
newLineNumbers = append(newLineNumbers, newLineNumber)
oldLineNumbers = append(oldLineNumbers, oldLineNumber)
for _, line := range hunk.bodyLines {
newLineNumbers = append(newLineNumbers, newLineNumber)
oldLineNumbers = append(oldLineNumbers, oldLineNumber)
if line.Kind == ADDITION || line.Kind == CONTEXT {
newLineNumber++
}
if line.Kind == DELETION || line.Kind == CONTEXT {
oldLineNumber++
}
}
}
return newLineNumbers, oldLineNumbers
}
// Returns hunk index containing the line at the given patch line index
func (self *Patch) HunkContainingLine(idx int) int {
for hunkIdx, hunk := range self.hunks {
+3 -2
View File
@@ -416,12 +416,13 @@ type LineIdentity struct {
// old file keeps them apart.
func ChangeLineIndexByIdentity(parsed *Patch) map[LineIdentity]int {
byIdentity := map[LineIdentity]int{}
newLineNumbers, oldLineNumbers := parsed.LineNumbersOfLines()
for idx, line := range parsed.Lines() {
switch {
case line.IsAddition():
byIdentity[LineIdentity{parsed.LineNumberOfLine(idx), false}] = idx
byIdentity[LineIdentity{newLineNumbers[idx], false}] = idx
case line.IsDeletion():
byIdentity[LineIdentity{parsed.OldLineNumberOfLine(idx), true}] = idx
byIdentity[LineIdentity{oldLineNumbers[idx], true}] = idx
}
}
return byIdentity
+31
View File
@@ -838,6 +838,37 @@ func TestOldLineNumberOfLine(t *testing.T) {
}
}
func TestLineNumbersOfLines(t *testing.T) {
scenarios := []struct {
testName string
patchStr string
}{
{"simpleDiff", simpleDiff},
{"twoHunks", twoHunks},
{"twoHunksWithMoreAdditionsThanRemovals", twoHunksWithMoreAdditionsThanRemovals},
{"twoChangesInOneHunk", twoChangesInOneHunk},
{"consecutiveDeletions", consecutiveDeletions},
{"renameWithModificationDiff", renameWithModificationDiff},
{"addNewlineToEndOfFile", addNewlineToEndOfFile},
{"newFile", newFile},
{"deletedFile", deletedFile},
}
for _, s := range scenarios {
t.Run(s.testName, func(t *testing.T) {
patch := Parse(s.patchStr)
newLineNumbers, oldLineNumbers := patch.LineNumbersOfLines()
assert.Len(t, newLineNumbers, patch.LineCount())
assert.Len(t, oldLineNumbers, patch.LineCount())
for idx := range patch.LineCount() {
assert.Equal(t, patch.LineNumberOfLine(idx), newLineNumbers[idx], "new line number of line %d", idx)
assert.Equal(t, patch.OldLineNumberOfLine(idx), oldLineNumbers[idx], "old line number of line %d", idx)
}
})
}
}
func TestGetNextStageableLineIndex(t *testing.T) {
type scenario struct {
testName string
+54 -7
View File
@@ -754,6 +754,15 @@ type lineType struct {
// which they hold as the overwrite it caused. nil while the cells spell the
// line as it was written, as they do for most lines.
asWritten []byte
// metadata holds the OSC 1717 records that the line's cells carry, each once, in
// the order they first appear. They are collected when the write cursor moves on
// to the next line, so that a reader can take them from here instead of going
// through every cell of every line. metadataCollected is false until then, and
// again once anything is written to the line; a reader then takes them from the
// cells.
metadata []string
metadataCollected bool
}
// textAsWritten returns the line's text as its writer wrote it, escape sequences
@@ -766,6 +775,34 @@ func (l *lineType) textAsWritten() string {
return l.cells.String()
}
// diffLineMetadata returns the OSC 1717 records that the line's cells carry (see
// lineType.metadata).
func (l *lineType) diffLineMetadata() []string {
if l.metadataCollected {
return l.metadata
}
return metadataOfCells(l.cells)
}
// collectMetadata keeps the records that the line's cells carry, for readers to take
// without going through the cells (see lineType.metadata).
func (l *lineType) collectMetadata() {
l.metadata = metadataOfCells(l.cells)
l.metadataCollected = true
}
// metadataOfCells returns the OSC 1717 records that the given cells carry, each once,
// in the order they first appear.
func metadataOfCells(cells []cell) []string {
var metadata []string
for _, c := range cells {
if c.metadata != "" && !slices.Contains(metadata, c.metadata) {
metadata = append(metadata, c.metadata)
}
}
return metadata
}
// trailingFillAttributes describes the fg/bg colors that draw() should
// use for cells past the end of a wrapped segment's content. On a source
// line this records what the writer asked for via '\x1b[K' (and so opts
@@ -1153,6 +1190,7 @@ func (b *viewBuffer) writeCells(cells []cell) {
}
}
b.lines[b.wy].cells = line[:newLen]
b.lines[b.wy].metadataCollected = false
b.wx += len(cells)
}
@@ -1244,6 +1282,7 @@ func (b *viewBuffer) write(v *View, p []byte) {
}
advanceToNextLine := func() {
b.lines[b.wy].collectMetadata()
b.seekWrite(0, b.wy+1)
if b.wy >= len(b.lines) {
b.lines = append(b.lines, lineType{})
@@ -2119,6 +2158,20 @@ func (v *View) DiffLineContents() []DiffLineContent {
return diffLineContentsFrom(v.buf, 0)
}
// DiffLineMetadata is DiffLineContents without the text of the lines, for a reader
// that only needs the records. Taking the text is most of the cost of
// DiffLineContents.
func (v *View) DiffLineMetadata() [][]string {
v.writeMutex.Lock()
defer v.writeMutex.Unlock()
metadata := make([][]string, len(v.buf.lines))
for i := range v.buf.lines {
metadata[i] = v.buf.lines[i].diffLineMetadata()
}
return metadata
}
// OffscreenDiffLineContents is DiffLineContents for the content of a re-render in
// progress (see BeginOffscreenRender). A reader deciding where the new content
// should be shown has to work from this: it has to answer before the swap, since
@@ -2170,13 +2223,7 @@ func diffLineContentsFrom(buf *viewBuffer, from int) []DiffLineContent {
contents := make([]DiffLineContent, len(lines))
for i := range lines {
line := &lines[i]
var metadata []string
for _, c := range line.cells {
if c.metadata != "" && !slices.Contains(metadata, c.metadata) {
metadata = append(metadata, c.metadata)
}
}
contents[i] = DiffLineContent{Text: line.textAsWritten(), Metadata: metadata}
contents[i] = DiffLineContent{Text: line.textAsWritten(), Metadata: line.diffLineMetadata()}
}
return contents
}
+34
View File
@@ -335,6 +335,40 @@ func TestDiffLineContentsWithRecordsCoveringNoCell(t *testing.T) {
}, v.DiffLineContents())
}
func TestDiffLineContentsOfARewrittenLine(t *testing.T) {
v := NewView("name", 0, 0, 80, 10, OutputNormal)
v.writeString(strings.Join([]string{
osc1717("1;c;1;;foo.txt") + "line1",
osc1717("1;c;2;;foo.txt") + "line2",
"last line",
}, "\n"))
// The records of a line that is written again are the ones it is written with,
// even when the line's earlier records were taken from it already.
v.OverwriteLines(0, osc1717("1;a;7;;bar.txt")+"new1")
assert.Equal(t, []DiffLineContent{
{Text: "new1", Metadata: []string{"1;a;7;;bar.txt"}},
{Text: "line2", Metadata: []string{"1;c;2;;foo.txt"}},
{Text: "last line"},
}, v.DiffLineContents())
}
func TestDiffLineMetadata(t *testing.T) {
v := NewView("name", 0, 0, 80, 10, OutputNormal)
v.writeString(strings.Join([]string{
osc1717("1;d;2;2;foo.txt") + "old2 " + osc1717("1;a;2;;foo.txt") + "new2",
"@@ a hunk header, which carries no record @@",
osc1717("1;c;3;;foo.txt") + "line3",
}, "\n"))
assert.Equal(t, [][]string{
{"1;d;2;2;foo.txt", "1;a;2;;foo.txt"},
nil,
{"1;c;3;;foo.txt"},
}, v.DiffLineMetadata())
}
func TestDiffLineContentsSwallowsHandshake(t *testing.T) {
v := NewView("name", 0, 0, 80, 10, OutputNormal)
+36 -11
View File
@@ -5,6 +5,7 @@ import (
"github.com/jesseduffield/lazygit/pkg/gocui"
"github.com/jesseduffield/lazygit/pkg/gui/types"
"github.com/samber/lo"
)
type DiffLineHelper struct {
@@ -59,7 +60,7 @@ func (self *DiffLineHelper) diffLineIdentitiesAt(
return nil, false
}
contents := view.DiffLineContents()
contents := diffLineContents(view)
if bufferLineIdx >= len(contents) {
return nil, false
}
@@ -134,10 +135,13 @@ func (self *DiffLineHelper) resolveDiffLines(contents []gocui.DiffLineContent) [
// side-by-side row leads with the deletion whose addition was what got remembered
// under a unified one.
func (self *DiffLineHelper) resolveDiffLineIdentities(contents []gocui.DiffLineContent) [][]types.DiffLineInfo {
toDiffLineInfo := self.diffLineInfoConverter()
identities := make([][]types.DiffLineInfo, len(contents))
for i, parsed := range parseDiffLineIdentities(contents) {
if len(parsed) > 0 {
identities[i] = self.diffLineInfos(parsed)
identities[i] = lo.Map(parsed, func(line parsedDiffLine, _ int) types.DiffLineInfo {
return toDiffLineInfo(line)
})
}
}
return identities
@@ -159,19 +163,40 @@ func (self *DiffLineHelper) diffLineInfos(parsed []parsedDiffLine) []types.DiffL
return infos
}
// diffLineInfoConverter returns diffLineInfo as a function, for the callers that turn
// every line of a diff into an identity (see diffLineInfoConverterIn).
func (self *DiffLineHelper) diffLineInfoConverter() func(parsedDiffLine) types.DiffLineInfo {
return diffLineInfoConverterIn(self.c.Git().RepoPaths.WorktreePath())
}
// diffLineInfoIn is diffLineInfo against a given worktree, for the callers that can't
// ask which repo we are in where they run: a repo switch replaces it, so only the UI
// thread may read it.
func diffLineInfoIn(worktreePath string, parsed parsedDiffLine) types.DiffLineInfo {
path := parsed.Path
if !filepath.IsAbs(path) {
path = filepath.Join(worktreePath, path)
}
return diffLineInfoConverterIn(worktreePath)(parsed)
}
return types.DiffLineInfo{
Path: path,
Type: parsed.Type,
NewLine: parsed.NewLine,
OldLine: parsed.OldLine,
// diffLineInfoConverterIn returns diffLineInfoIn for the given worktree as a function,
// for the callers that turn every line of a diff into an identity. The lines of a diff
// come file by file, so the path of a line is nearly always the path of the line
// before it. The function makes a path absolute only when it differs from the one
// before.
func diffLineInfoConverterIn(worktreePath string) func(parsedDiffLine) types.DiffLineInfo {
path, absolutePath := "", ""
return func(parsed parsedDiffLine) types.DiffLineInfo {
if absolutePath == "" || parsed.Path != path {
path = parsed.Path
absolutePath = parsed.Path
if !filepath.IsAbs(absolutePath) {
absolutePath = filepath.Join(worktreePath, absolutePath)
}
}
return types.DiffLineInfo{
Path: absolutePath,
Type: parsed.Type,
NewLine: parsed.NewLine,
OldLine: parsed.OldLine,
}
}
}
@@ -0,0 +1,34 @@
package helpers
import (
"path/filepath"
"testing"
"github.com/jesseduffield/lazygit/pkg/gui/types"
"github.com/stretchr/testify/assert"
)
func TestDiffLineInfoConverterIn(t *testing.T) {
worktreePath := t.TempDir()
absolutePath := filepath.Join(t.TempDir(), "d.txt")
toDiffLineInfo := diffLineInfoConverterIn(worktreePath)
lines := []parsedDiffLine{
{Path: "a.txt", Type: types.DiffLineAdded, NewLine: 1},
{Path: "a.txt", Type: types.DiffLineDeleted, NewLine: 2, OldLine: 1},
{Path: "dir/b.txt", Type: types.DiffLineContext, NewLine: 3},
{Path: absolutePath, Type: types.DiffLineAdded, NewLine: 4},
{Path: "a.txt", Type: types.DiffLineAdded, NewLine: 5},
}
expected := []types.DiffLineInfo{
{Path: filepath.Join(worktreePath, "a.txt"), Type: types.DiffLineAdded, NewLine: 1},
{Path: filepath.Join(worktreePath, "a.txt"), Type: types.DiffLineDeleted, NewLine: 2, OldLine: 1},
{Path: filepath.Join(worktreePath, "dir", "b.txt"), Type: types.DiffLineContext, NewLine: 3},
{Path: absolutePath, Type: types.DiffLineAdded, NewLine: 4},
{Path: filepath.Join(worktreePath, "a.txt"), Type: types.DiffLineAdded, NewLine: 5},
}
for i, line := range lines {
assert.Equal(t, expected[i], toDiffLineInfo(line), "line %d", i)
}
}
@@ -9,6 +9,7 @@ import (
"github.com/jesseduffield/lazygit/pkg/commands/patch"
"github.com/jesseduffield/lazygit/pkg/gocui"
"github.com/jesseduffield/lazygit/pkg/gui/types"
"github.com/samber/lo"
)
// diffFilePrefix marks the start of a file's section in a (possibly multi-file)
@@ -125,6 +126,20 @@ func renderingStatesDiffLines(contents []gocui.DiffLineContent) bool {
})
}
// diffLineContents returns the contents of view's rows, as the queries about the lines
// of a diff read them. A rendering that states which diff line each row shows is read
// by those records alone (see renderingStatesDiffLines), so the text of its rows is
// left out then. Taking the text is most of the cost of reading a long diff.
func diffLineContents(view *gocui.View) []gocui.DiffLineContent {
contents := lo.Map(view.DiffLineMetadata(), func(metadata []string, _ int) gocui.DiffLineContent {
return gocui.DiffLineContent{Metadata: metadata}
})
if renderingStatesDiffLines(contents) {
return contents
}
return view.DiffLineContents()
}
// parseDiffLineRecords parses the records a row carries, left to right, leaving out
// the ones we don't understand. A row carries more than one when the rendering puts
// two diff lines on it, as a side-by-side row does with a deletion and the addition
@@ -272,6 +287,7 @@ func parseFileSection(fileLines []string, endsTheBuffer bool) []bufferLineParse
return result
}
patchLines := p.Lines()
newLineNumbers, oldLineNumbers := p.LineNumbersOfLines()
for i := range fileLines {
if i >= len(patchLines) {
break
@@ -279,10 +295,10 @@ func parseFileSection(fileLines []string, endsTheBuffer bool) []bufferLineParse
parsed := parsedDiffLine{
Path: relPath,
Type: diffLineTypeForKind(patchLines[i].Kind),
NewLine: p.LineNumberOfLine(i),
NewLine: newLineNumbers[i],
}
if parsed.Type == types.DiffLineDeleted {
parsed.OldLine = p.OldLineNumberOfLine(i)
parsed.OldLine = oldLineNumbers[i]
}
result[i] = bufferLineParse{parsed, true}
}
@@ -544,3 +544,22 @@ func TestParseDiffLineIdentities(t *testing.T) {
}
})
}
func TestDiffLineContentsLeaveOutTheTextOfARenderingWithRecords(t *testing.T) {
record := func(payload string) string { return "\x1b]1717;" + payload + "\x1b\\" }
rendering := gocui.NewView("rendering", 0, 0, 80, 10, gocui.OutputNormal)
rendering.WriteString(record("1;a;1;;foo.txt") + "added\n" + "a divider\n")
assert.Equal(t, []gocui.DiffLineContent{
{Metadata: []string{"1;a;1;;foo.txt"}},
{},
}, diffLineContents(rendering))
// A diff without records is read by its text, so it keeps it.
diff := gocui.NewView("diff", 0, 0, 80, 10, gocui.OutputNormal)
diff.WriteString("diff --git a/foo.txt b/foo.txt\n+added\n")
assert.Equal(t, []gocui.DiffLineContent{
{Text: "diff --git a/foo.txt b/foo.txt"},
{Text: "+added"},
}, diffLineContents(diff))
}
@@ -72,12 +72,13 @@ func (self *DiffLineHelper) PlainDiffOfSelection(
}
diffLines := strings.Split(strings.TrimSuffix(plainDiff(relPaths), "\n"), "\n")
toDiffLineInfo := diffLineInfoConverterIn(worktreePath)
runs := map[string][2]int{}
for i, parsed := range parseAllDiffLinesFromBuffer(diffLines) {
if !parsed.ok {
continue
}
info := diffLineInfoIn(worktreePath, parsed.parsed)
info := toDiffLineInfo(parsed.parsed)
selected := selection[info.Path]
if selected == nil || !selected.covers(info) {
continue
@@ -28,7 +28,7 @@ import (
// beside the addition replacing it — and all of them are reported: what the user
// pointed at is the row, so everything on it is selected.
func (self *DiffLineHelper) DiffLinesInBufferRange(view *gocui.View, first int, last int) []types.DiffLineInfo {
identities := self.resolveDiffLineIdentities(view.DiffLineContents())
identities := self.resolveDiffLineIdentities(diffLineContents(view))
infos := []types.DiffLineInfo{}
for bufferLine := first; bufferLine <= min(last, len(identities)-1); bufferLine++ {
@@ -47,13 +47,14 @@ func (self *DiffLineHelper) DiffLinesInBufferRange(view *gocui.View, first int,
func (self *DiffLineHelper) ChangeLineOrdinals(
diff string, infos []types.DiffLineInfo,
) map[string][]int {
toDiffLineInfo := self.diffLineInfoConverter()
ordinals := map[patchLine]int{}
counts := map[string]int{}
for _, parsed := range parseAllDiffLinesFromBuffer(strings.Split(diff, "\n")) {
if !parsed.ok {
continue
}
info := self.diffLineInfo(parsed.parsed)
info := toDiffLineInfo(parsed.parsed)
if !info.IsChange() {
continue
}
@@ -135,7 +136,7 @@ func (self *DiffLineHelper) ChangeLinesInBufferRange(view *gocui.View, first int
// header, or a row whose identity couldn't be recovered. Those are the rows a
// selection is anchored on and navigation moves between.
func (self *DiffLineHelper) changeLines(view *gocui.View) []bool {
resolved := self.resolveDiffLines(view.DiffLineContents())
resolved := self.resolveDiffLines(diffLineContents(view))
isChange := make([]bool, len(resolved))
for i, r := range resolved {
isChange[i] = r.ok && r.info.IsChange()
@@ -243,7 +244,7 @@ func (self *DiffLineHelper) IsSingleHunkForWholeFile(view *gocui.View, changeVie
if !ok {
return false
}
resolved := self.resolveDiffLines(view.DiffLineContents())
resolved := self.resolveDiffLines(diffLineContents(view))
if anchor >= len(resolved) || !resolved[anchor].ok {
return false
}
@@ -374,7 +375,7 @@ func (self *DiffLineHelper) StartOfFileInDiff(view *gocui.View, path string) (in
// was materialized into, so that both halves of a change belong to the same file however
// the diff renderer states them.
func (self *DiffLineHelper) filePaths(view *gocui.View) []string {
resolved := self.resolveDiffLines(view.DiffLineContents())
resolved := self.resolveDiffLines(diffLineContents(view))
toRepoTerms := self.repoTermsMapper(view)
paths := make([]string, len(resolved))
for i, row := range resolved {
@@ -151,7 +151,7 @@ func (self *DiffLineHelper) selectionLine(
if len(candidates) == 0 {
return anchorViewLine
}
_, bufferLine := self.nearestSurvivingCandidate(view.DiffLineContents(), candidates)
_, bufferLine := self.nearestSurvivingCandidate(diffLineContents(view), candidates)
if bufferLine == -1 {
return anchorViewLine
}
@@ -208,7 +208,7 @@ func selectionFarEndViewLine(view *gocui.View) (int, bool, bool) {
// findDiffLine returns the view line showing the given diff line in what view is
// displaying now, for placing a remembered line once the re-render is on screen.
func (self *DiffLineHelper) findDiffLine(view *gocui.View, identity types.DiffLineInfo) (int, bool) {
bufferLine, ok := self.patchLineRows(view.DiffLineContents())[patchLineOf(identity)]
bufferLine, ok := self.patchLineRows(diffLineContents(view))[patchLineOf(identity)]
if !ok {
return 0, false
}
@@ -273,7 +273,7 @@ func (self *DiffLineHelper) restoreDiffLinePositionOnRerender(
// place it left behind is the same one that identity used to have.
func (self *DiffLineHelper) ChangeLineOrdinal(view *gocui.View, bufferLine int) int {
ordinal := 0
for i, row := range self.resolveDiffLines(view.DiffLineContents()) {
for i, row := range self.resolveDiffLines(diffLineContents(view)) {
if i >= bufferLine {
break
}
@@ -421,7 +421,7 @@ func (self *DiffLineHelper) nearbyDiffLines(view *gocui.View, anchorViewLine int
if !ok {
return nil
}
resolved := self.resolveDiffLines(view.DiffLineContents())
resolved := self.resolveDiffLines(diffLineContents(view))
if anchor >= len(resolved) {
return nil
}
@@ -222,7 +222,7 @@ func (self *DiffLineHelper) RefreshInclusionGutter() {
return
}
resolved := self.resolveDiffLines(view.DiffLineContents())
resolved := self.resolveDiffLines(diffLineContents(view))
marks := make([]bool, len(resolved))
for i, row := range resolved {
if row.ok && row.info.IsChange() {