Skip to content

Commit 66caa25

Browse files
authored
Reduce memory consumption of graph (pipe sets) (#4498)
- **PR Description** As a followup to #2533, reduce the memory consumption some more by optimizing the storage of the pipe sets used for the commit graph. Some coarse measurements (taken by opening the respective repo, typing `4 >`, and waiting for all commits to be loaded, and then reading the Memory column in Activity Monitor): | | master | this PR | | ------------- | ------- | ------- | | git | 2.5 GB | 1.0 GB | | our work repo | 2.3 GB | 1.3 GB | | linux kernel | 94.8 GB | 38.0 GB | It's still not really usable for the linux kernel, but for all other repos that I come across in my daily use of lazygit, it works quite well now.
2 parents d87bdaf + 6ca627d commit 66caa25

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+641
-528
lines changed

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ require (
3333
github.com/sirupsen/logrus v1.9.3
3434
github.com/spf13/afero v1.9.5
3535
github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad
36-
github.com/stefanhaller/git-todo-parser v0.0.7-0.20240406123903-fd957137b6e2
36+
github.com/stefanhaller/git-todo-parser v0.0.7-0.20250429125209-dcf39e4641f5
3737
github.com/stretchr/testify v1.10.0
3838
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778
3939
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -286,8 +286,8 @@ github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM=
286286
github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ=
287287
github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad h1:fiWzISvDn0Csy5H0iwgAuJGQTUpVfEMJJd4nRFXogbc=
288288
github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0=
289-
github.com/stefanhaller/git-todo-parser v0.0.7-0.20240406123903-fd957137b6e2 h1:RTNWemd/9z9A5L/AggEP3OdhRz5dXETB/wdAlYF0SuM=
290-
github.com/stefanhaller/git-todo-parser v0.0.7-0.20240406123903-fd957137b6e2/go.mod h1:HFt9hGqMzgQ+gVxMKcvTvGaFz4Y0yYycqqAp2V3wcJY=
289+
github.com/stefanhaller/git-todo-parser v0.0.7-0.20250429125209-dcf39e4641f5 h1:ZCI0NPs0xXd00Ej9lX+wwbHjQDkstJa3kUbX7WCOF8I=
290+
github.com/stefanhaller/git-todo-parser v0.0.7-0.20250429125209-dcf39e4641f5/go.mod h1:HFt9hGqMzgQ+gVxMKcvTvGaFz4Y0yYycqqAp2V3wcJY=
291291
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
292292
github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
293293
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=

pkg/app/daemon/rebase.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ func (self *TodoLine) ToString() string {
2222
if self.Action == "break" {
2323
return self.Action + "\n"
2424
} else {
25-
return self.Action + " " + self.Commit.Hash + " " + self.Commit.Name + "\n"
25+
return self.Action + " " + self.Commit.Hash() + " " + self.Commit.Name + "\n"
2626
}
2727
}
2828

pkg/commands/git_commands/commit_loader.go

Lines changed: 39 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ type GetCommitsOptions struct {
6666
// If non-empty, show divergence from this ref (left-right log)
6767
RefToShowDivergenceFrom string
6868
MainBranches *MainBranches
69+
HashPool *utils.StringPool
6970
}
7071

7172
// GetCommits obtains the commits of the current branch
@@ -74,7 +75,7 @@ func (self *CommitLoader) GetCommits(opts GetCommitsOptions) ([]*models.Commit,
7475

7576
if opts.IncludeRebaseCommits && opts.FilterPath == "" {
7677
var err error
77-
commits, err = self.MergeRebasingCommits(commits)
78+
commits, err = self.MergeRebasingCommits(opts.HashPool, commits)
7879
if err != nil {
7980
return nil, err
8081
}
@@ -89,7 +90,7 @@ func (self *CommitLoader) GetCommits(opts GetCommitsOptions) ([]*models.Commit,
8990
defer wg.Done()
9091

9192
logErr = self.getLogCmd(opts).RunAndProcessLines(func(line string) (bool, error) {
92-
commit := self.extractCommitFromLine(line, opts.RefToShowDivergenceFrom != "")
93+
commit := self.extractCommitFromLine(opts.HashPool, line, opts.RefToShowDivergenceFrom != "")
9394
commits = append(commits, commit)
9495
return false, nil
9596
})
@@ -121,7 +122,7 @@ func (self *CommitLoader) GetCommits(opts GetCommitsOptions) ([]*models.Commit,
121122
}
122123

123124
for _, commit := range commits {
124-
if commit.Hash == firstPushedCommit {
125+
if commit.Hash() == firstPushedCommit {
125126
passedFirstPushedCommit = true
126127
}
127128
if !commit.IsTODO() {
@@ -159,7 +160,7 @@ func (self *CommitLoader) GetCommits(opts GetCommitsOptions) ([]*models.Commit,
159160
return commits, nil
160161
}
161162

162-
func (self *CommitLoader) MergeRebasingCommits(commits []*models.Commit) ([]*models.Commit, error) {
163+
func (self *CommitLoader) MergeRebasingCommits(hashPool *utils.StringPool, commits []*models.Commit) ([]*models.Commit, error) {
163164
// chances are we have as many commits as last time so we'll set the capacity to be the old length
164165
result := make([]*models.Commit, 0, len(commits))
165166
for i, commit := range commits {
@@ -172,7 +173,7 @@ func (self *CommitLoader) MergeRebasingCommits(commits []*models.Commit) ([]*mod
172173
workingTreeState := self.getWorkingTreeState()
173174
addConflictedRebasingCommit := true
174175
if workingTreeState.CherryPicking || workingTreeState.Reverting {
175-
sequencerCommits, err := self.getHydratedSequencerCommits(workingTreeState)
176+
sequencerCommits, err := self.getHydratedSequencerCommits(hashPool, workingTreeState)
176177
if err != nil {
177178
return nil, err
178179
}
@@ -181,7 +182,7 @@ func (self *CommitLoader) MergeRebasingCommits(commits []*models.Commit) ([]*mod
181182
}
182183

183184
if workingTreeState.Rebasing {
184-
rebasingCommits, err := self.getHydratedRebasingCommits(addConflictedRebasingCommit)
185+
rebasingCommits, err := self.getHydratedRebasingCommits(hashPool, addConflictedRebasingCommit)
185186
if err != nil {
186187
return nil, err
187188
}
@@ -196,7 +197,7 @@ func (self *CommitLoader) MergeRebasingCommits(commits []*models.Commit) ([]*mod
196197
// then puts them into a commit object
197198
// example input:
198199
// 8ad01fe32fcc20f07bc6693f87aa4977c327f1e1|10 hours ago|Jesse Duffield| (HEAD -> master, tag: v0.15.2)|refresh commits when adding a tag
199-
func (self *CommitLoader) extractCommitFromLine(line string, showDivergence bool) *models.Commit {
200+
func (self *CommitLoader) extractCommitFromLine(hashPool *utils.StringPool, line string, showDivergence bool) *models.Commit {
200201
split := strings.SplitN(line, "\x00", 8)
201202

202203
hash := split[0]
@@ -234,7 +235,7 @@ func (self *CommitLoader) extractCommitFromLine(line string, showDivergence bool
234235
parents = strings.Split(parentHashes, " ")
235236
}
236237

237-
return &models.Commit{
238+
return models.NewCommit(hashPool, models.NewCommitOpts{
238239
Hash: hash,
239240
Name: message,
240241
Tags: tags,
@@ -244,16 +245,16 @@ func (self *CommitLoader) extractCommitFromLine(line string, showDivergence bool
244245
AuthorEmail: authorEmail,
245246
Parents: parents,
246247
Divergence: divergence,
247-
}
248+
})
248249
}
249250

250-
func (self *CommitLoader) getHydratedRebasingCommits(addConflictingCommit bool) ([]*models.Commit, error) {
251+
func (self *CommitLoader) getHydratedRebasingCommits(hashPool *utils.StringPool, addConflictingCommit bool) ([]*models.Commit, error) {
251252
todoFileHasShortHashes := self.version.IsOlderThan(2, 25, 2)
252-
return self.getHydratedTodoCommits(self.getRebasingCommits(addConflictingCommit), todoFileHasShortHashes)
253+
return self.getHydratedTodoCommits(hashPool, self.getRebasingCommits(hashPool, addConflictingCommit), todoFileHasShortHashes)
253254
}
254255

255-
func (self *CommitLoader) getHydratedSequencerCommits(workingTreeState models.WorkingTreeState) ([]*models.Commit, error) {
256-
commits := self.getSequencerCommits()
256+
func (self *CommitLoader) getHydratedSequencerCommits(hashPool *utils.StringPool, workingTreeState models.WorkingTreeState) ([]*models.Commit, error) {
257+
commits := self.getSequencerCommits(hashPool)
257258
if len(commits) > 0 {
258259
// If we have any commits in .git/sequencer/todo, then the last one of
259260
// those is the conflicting one.
@@ -262,22 +263,22 @@ func (self *CommitLoader) getHydratedSequencerCommits(workingTreeState models.Wo
262263
// For single-commit cherry-picks and reverts, git apparently doesn't
263264
// use the sequencer; in that case, CHERRY_PICK_HEAD or REVERT_HEAD is
264265
// our conflicting commit, so synthesize it here.
265-
conflicedCommit := self.getConflictedSequencerCommit(workingTreeState)
266+
conflicedCommit := self.getConflictedSequencerCommit(hashPool, workingTreeState)
266267
if conflicedCommit != nil {
267268
commits = append(commits, conflicedCommit)
268269
}
269270
}
270271

271-
return self.getHydratedTodoCommits(commits, true)
272+
return self.getHydratedTodoCommits(hashPool, commits, true)
272273
}
273274

274-
func (self *CommitLoader) getHydratedTodoCommits(todoCommits []*models.Commit, todoFileHasShortHashes bool) ([]*models.Commit, error) {
275+
func (self *CommitLoader) getHydratedTodoCommits(hashPool *utils.StringPool, todoCommits []*models.Commit, todoFileHasShortHashes bool) ([]*models.Commit, error) {
275276
if len(todoCommits) == 0 {
276277
return nil, nil
277278
}
278279

279280
commitHashes := lo.FilterMap(todoCommits, func(commit *models.Commit, _ int) (string, bool) {
280-
return commit.Hash, commit.Hash != ""
281+
return commit.Hash(), commit.Hash() != ""
281282
})
282283

283284
// note that we're not filtering these as we do non-rebasing commits just because
@@ -292,8 +293,8 @@ func (self *CommitLoader) getHydratedTodoCommits(todoCommits []*models.Commit, t
292293

293294
fullCommits := map[string]*models.Commit{}
294295
err := cmdObj.RunAndProcessLines(func(line string) (bool, error) {
295-
commit := self.extractCommitFromLine(line, false)
296-
fullCommits[commit.Hash] = commit
296+
commit := self.extractCommitFromLine(hashPool, line, false)
297+
fullCommits[commit.Hash()] = commit
297298
return false, nil
298299
})
299300
if err != nil {
@@ -315,9 +316,9 @@ func (self *CommitLoader) getHydratedTodoCommits(todoCommits []*models.Commit, t
315316

316317
hydratedCommits := make([]*models.Commit, 0, len(todoCommits))
317318
for _, rebasingCommit := range todoCommits {
318-
if rebasingCommit.Hash == "" {
319+
if rebasingCommit.Hash() == "" {
319320
hydratedCommits = append(hydratedCommits, rebasingCommit)
320-
} else if commit := findFullCommit(rebasingCommit.Hash); commit != nil {
321+
} else if commit := findFullCommit(rebasingCommit.Hash()); commit != nil {
321322
commit.Action = rebasingCommit.Action
322323
commit.Status = rebasingCommit.Status
323324
hydratedCommits = append(hydratedCommits, commit)
@@ -331,7 +332,7 @@ func (self *CommitLoader) getHydratedTodoCommits(todoCommits []*models.Commit, t
331332
// git-rebase-todo example:
332333
// pick ac446ae94ee560bdb8d1d057278657b251aaef17 ac446ae
333334
// pick afb893148791a2fbd8091aeb81deba4930c73031 afb8931
334-
func (self *CommitLoader) getRebasingCommits(addConflictingCommit bool) []*models.Commit {
335+
func (self *CommitLoader) getRebasingCommits(hashPool *utils.StringPool, addConflictingCommit bool) []*models.Commit {
335336
bytesContent, err := self.readFile(filepath.Join(self.repoPaths.WorktreeGitDirPath(), "rebase-merge/git-rebase-todo"))
336337
if err != nil {
337338
self.Log.Error(fmt.Sprintf("error occurred reading git-rebase-todo: %s", err.Error()))
@@ -350,7 +351,7 @@ func (self *CommitLoader) getRebasingCommits(addConflictingCommit bool) []*model
350351
// See if the current commit couldn't be applied because it conflicted; if
351352
// so, add a fake entry for it
352353
if addConflictingCommit {
353-
if conflictedCommit := self.getConflictedCommit(todos); conflictedCommit != nil {
354+
if conflictedCommit := self.getConflictedCommit(hashPool, todos); conflictedCommit != nil {
354355
commits = append(commits, conflictedCommit)
355356
}
356357
}
@@ -364,18 +365,18 @@ func (self *CommitLoader) getRebasingCommits(addConflictingCommit bool) []*model
364365
// Command does not have a commit associated, skip
365366
continue
366367
}
367-
commits = utils.Prepend(commits, &models.Commit{
368+
commits = utils.Prepend(commits, models.NewCommit(hashPool, models.NewCommitOpts{
368369
Hash: t.Commit,
369370
Name: t.Msg,
370371
Status: models.StatusRebasing,
371372
Action: t.Command,
372-
})
373+
}))
373374
}
374375

375376
return commits
376377
}
377378

378-
func (self *CommitLoader) getConflictedCommit(todos []todo.Todo) *models.Commit {
379+
func (self *CommitLoader) getConflictedCommit(hashPool *utils.StringPool, todos []todo.Todo) *models.Commit {
379380
bytesContent, err := self.readFile(filepath.Join(self.repoPaths.WorktreeGitDirPath(), "rebase-merge/done"))
380381
if err != nil {
381382
self.Log.Error(fmt.Sprintf("error occurred reading rebase-merge/done: %s", err.Error()))
@@ -391,10 +392,10 @@ func (self *CommitLoader) getConflictedCommit(todos []todo.Todo) *models.Commit
391392
amendFileExists, _ := self.os.FileExists(filepath.Join(self.repoPaths.WorktreeGitDirPath(), "rebase-merge/amend"))
392393
messageFileExists, _ := self.os.FileExists(filepath.Join(self.repoPaths.WorktreeGitDirPath(), "rebase-merge/message"))
393394

394-
return self.getConflictedCommitImpl(todos, doneTodos, amendFileExists, messageFileExists)
395+
return self.getConflictedCommitImpl(hashPool, todos, doneTodos, amendFileExists, messageFileExists)
395396
}
396397

397-
func (self *CommitLoader) getConflictedCommitImpl(todos []todo.Todo, doneTodos []todo.Todo, amendFileExists bool, messageFileExists bool) *models.Commit {
398+
func (self *CommitLoader) getConflictedCommitImpl(hashPool *utils.StringPool, todos []todo.Todo, doneTodos []todo.Todo, amendFileExists bool, messageFileExists bool) *models.Commit {
398399
// Should never be possible, but just to be safe:
399400
if len(doneTodos) == 0 {
400401
self.Log.Error("no done entries in rebase-merge/done file")
@@ -465,14 +466,14 @@ func (self *CommitLoader) getConflictedCommitImpl(todos []todo.Todo, doneTodos [
465466

466467
// Any other todo that has a commit associated with it must have failed with
467468
// a conflict, otherwise we wouldn't have stopped the rebase:
468-
return &models.Commit{
469+
return models.NewCommit(hashPool, models.NewCommitOpts{
469470
Hash: lastTodo.Commit,
470471
Action: lastTodo.Command,
471472
Status: models.StatusConflicted,
472-
}
473+
})
473474
}
474475

475-
func (self *CommitLoader) getSequencerCommits() []*models.Commit {
476+
func (self *CommitLoader) getSequencerCommits(hashPool *utils.StringPool) []*models.Commit {
476477
bytesContent, err := self.readFile(filepath.Join(self.repoPaths.WorktreeGitDirPath(), "sequencer/todo"))
477478
if err != nil {
478479
self.Log.Error(fmt.Sprintf("error occurred reading sequencer/todo: %s", err.Error()))
@@ -493,18 +494,18 @@ func (self *CommitLoader) getSequencerCommits() []*models.Commit {
493494
// Command does not have a commit associated, skip
494495
continue
495496
}
496-
commits = utils.Prepend(commits, &models.Commit{
497+
commits = utils.Prepend(commits, models.NewCommit(hashPool, models.NewCommitOpts{
497498
Hash: t.Commit,
498499
Name: t.Msg,
499500
Status: models.StatusCherryPickingOrReverting,
500501
Action: t.Command,
501-
})
502+
}))
502503
}
503504

504505
return commits
505506
}
506507

507-
func (self *CommitLoader) getConflictedSequencerCommit(workingTreeState models.WorkingTreeState) *models.Commit {
508+
func (self *CommitLoader) getConflictedSequencerCommit(hashPool *utils.StringPool, workingTreeState models.WorkingTreeState) *models.Commit {
508509
var shaFile string
509510
var action todo.TodoCommand
510511
if workingTreeState.CherryPicking {
@@ -526,11 +527,11 @@ func (self *CommitLoader) getConflictedSequencerCommit(workingTreeState models.W
526527
if len(lines) == 0 {
527528
return nil
528529
}
529-
return &models.Commit{
530+
return models.NewCommit(hashPool, models.NewCommitOpts{
530531
Hash: lines[0],
531532
Status: models.StatusConflicted,
532533
Action: action,
533-
}
534+
})
534535
}
535536

536537
func setCommitMergedStatuses(ancestor string, commits []*models.Commit) {
@@ -541,7 +542,7 @@ func setCommitMergedStatuses(ancestor string, commits []*models.Commit) {
541542
passedAncestor := false
542543
for i, commit := range commits {
543544
// some commits aren't really commits and don't have hashes, such as the update-ref todo
544-
if commit.Hash != "" && strings.HasPrefix(ancestor, commit.Hash) {
545+
if commit.Hash() != "" && strings.HasPrefix(ancestor, commit.Hash()) {
545546
passedAncestor = true
546547
}
547548
if commit.Status != models.StatusPushed && commit.Status != models.StatusUnpushed {
@@ -609,4 +610,4 @@ func (self *CommitLoader) getLogCmd(opts GetCommitsOptions) oscommands.ICmdObj {
609610
return self.cmd.New(cmdArgs).DontLog()
610611
}
611612

612-
const prettyFormat = `--pretty=format:%H%x00%at%x00%aN%x00%ae%x00%D%x00%p%x00%m%x00%s`
613+
const prettyFormat = `--pretty=format:%H%x00%at%x00%aN%x00%ae%x00%D%x00%P%x00%m%x00%s`

0 commit comments

Comments
 (0)