Skip to content

Commit 861d712

Browse files
charlieviethcharlievieth
committed
fastwalk: add optional sorting and improve documentation
This commit adds the new SortMode and Config.Sort setting to sort a directory's entries before they are processed. This does not make the global order that directories and entries are visited non-deterministic but it does help make the output a bit saner compared to the default directory order. This was added to make the output of FZF a bit nicer. This commit also improves documentation and comments of exported functions.
1 parent 875daa3 commit 861d712

14 files changed

+1020
-280
lines changed

‎adapters.go

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,17 @@ func isDir(path string, d fs.DirEntry) bool {
1818
return false
1919
}
2020

21-
// IgnoreDuplicateDirs wraps fs.WalkDirFunc walkFn to make it follow symbolic
21+
// IgnoreDuplicateDirs wraps [fs.WalkDirFunc] walkFn to make it follow symbolic
2222
// links and ignore duplicate directories (if a symlink points to a directory
2323
// that has already been traversed it is skipped). The walkFn is called for
2424
// for skipped directories, but the directory is not traversed (this is
2525
// required for error handling).
2626
//
27-
// The Config.Follow setting has no effect on the behavior of Walk when
27+
// The Follow [Config] setting has no effect on the behavior of Walk when
2828
// this wrapper is used.
2929
//
30-
// In most use cases, the returned fs.WalkDirFunc should not be reused between
31-
// in another call to Walk. If it is reused, any previously visited file will
32-
// be skipped.
30+
// In most use cases, the returned [fs.WalkDirFunc] should not be reused.
31+
// If it is reused, any previously visited file will be skipped.
3332
//
3433
// NOTE: The order of traversal is undefined. Given an "example" directory
3534
// like the one below where "dir" is a directory and "smydir1" and "smydir2"
@@ -68,9 +67,8 @@ func IgnoreDuplicateDirs(walkFn fs.WalkDirFunc) fs.WalkDirFunc {
6867
// files are ignored. If a symlink resolves to a file that has already been
6968
// visited it will be skipped.
7069
//
71-
// In most use cases, the returned fs.WalkDirFunc should not be reused between
72-
// in another call to Walk. If it is reused, any previously visited file will
73-
// be skipped.
70+
// In most use cases, the returned [fs.WalkDirFunc] should not be reused.
71+
// If it is reused, any previously visited file will be skipped.
7472
//
7573
// This can significantly slow Walk as os.Stat() is called for each path
7674
// (on Windows, os.Stat() is only needed for symlinks).
@@ -92,8 +90,8 @@ func IgnoreDuplicateFiles(walkFn fs.WalkDirFunc) fs.WalkDirFunc {
9290
}
9391
}
9492

95-
// IgnorePermissionErrors wraps walkFn so that permission errors are ignored.
96-
// The returned fs.WalkDirFunc may be reused.
93+
// IgnorePermissionErrors wraps walkFn so that [fs.ErrPermission] permission
94+
// errors are ignored. The returned [fs.WalkDirFunc] may be reused.
9795
func IgnorePermissionErrors(walkFn fs.WalkDirFunc) fs.WalkDirFunc {
9896
return func(path string, d fs.DirEntry, err error) error {
9997
if err != nil && os.IsPermission(err) {

‎dirent.go

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,16 @@ func loadFileInfo(pinfo **fileInfo) *fileInfo {
3131
return fi
3232
}
3333

34-
// StatDirEntry returns the fs.FileInfo for the file or subdirectory described
35-
// by the entry. If the entry is a symbolic link, StatDirEntry returns the
36-
// fs.FileInfo for the file the line references (os.Stat).
37-
// If fs.DirEntry de is a fastwalk.DirEntry it's Stat() method is used and the
38-
// returned fs.FileInfo may be a previously cached result.
34+
// StatDirEntry returns a [fs.FileInfo] describing the named file ([os.Stat]).
35+
// If de is a [fastwalk.DirEntry] its Stat method is used and the returned
36+
// FileInfo may be cached from a prior call to Stat. If a cached result is not
37+
// desired, users should just call [os.Stat] directly.
38+
//
39+
// This is a helper function for calling Stat on the DirEntry passed to the
40+
// walkFn argument to [Walk].
41+
//
42+
// The path argument is only used if de is not of type [fastwalk.DirEntry].
43+
// Therefore, de should be the DirEntry describing path.
3944
func StatDirEntry(path string, de fs.DirEntry) (fs.FileInfo, error) {
4045
if de == nil {
4146
return nil, &os.PathError{Op: "stat", Path: path, Err: syscall.EINVAL}

‎dirent_portable.go

Lines changed: 93 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,123 @@
11
//go:build !darwin && !(aix || dragonfly || freebsd || (js && wasm) || linux || netbsd || openbsd || solaris)
22

3+
// TODO: add a "portable_dirent" build tag so that we can test this
4+
// on non-Windows platforms
5+
36
package fastwalk
47

58
import (
69
"io/fs"
710
"os"
11+
"slices"
12+
"strings"
13+
"sync"
814
)
915

16+
var _ DirEntry = (*portableDirent)(nil)
17+
1018
type portableDirent struct {
1119
fs.DirEntry
12-
path string
13-
stat *fileInfo
20+
parent string
21+
stat *fileInfo
22+
}
23+
24+
func (d *portableDirent) String() string {
25+
return fs.FormatDirEntry(d)
1426
}
1527

16-
// TODO: cache the result of Stat
1728
func (d *portableDirent) Stat() (fs.FileInfo, error) {
1829
if d.DirEntry.Type()&os.ModeSymlink == 0 {
1930
return d.DirEntry.Info()
2031
}
2132
stat := loadFileInfo(&d.stat)
2233
stat.once.Do(func() {
23-
stat.FileInfo, stat.err = os.Stat(d.path)
34+
stat.FileInfo, stat.err = os.Stat(d.parent + string(os.PathSeparator) + d.Name())
2435
})
2536
return stat.FileInfo, stat.err
2637
}
2738

28-
func newDirEntry(dirName string, info fs.DirEntry) fs.DirEntry {
39+
func newDirEntry(dirName string, info fs.DirEntry) DirEntry {
2940
return &portableDirent{
3041
DirEntry: info,
31-
path: dirName + string(os.PathSeparator) + info.Name(),
42+
parent: dirName,
3243
}
3344
}
3445

35-
func fileInfoToDirEntry(dirname string, fi fs.FileInfo) fs.DirEntry {
46+
func fileInfoToDirEntry(dirname string, fi fs.FileInfo) DirEntry {
3647
return newDirEntry(dirname, fs.FileInfoToDirEntry(fi))
3748
}
49+
50+
var direntSlicePool = sync.Pool{
51+
New: func() any {
52+
a := make([]DirEntry, 0, 32)
53+
return &a
54+
},
55+
}
56+
57+
func putDirentSlice(p *[]DirEntry) {
58+
// max is half as many as Unix because twice the size
59+
if p != nil && cap(*p) <= 16*1024 {
60+
a := *p
61+
for i := range a {
62+
a[i] = nil
63+
}
64+
*p = a[:0]
65+
direntSlicePool.Put(p)
66+
}
67+
}
68+
69+
func sortDirents(mode SortMode, dents []DirEntry) {
70+
if len(dents) <= 1 {
71+
return
72+
}
73+
switch mode {
74+
case SortLexical:
75+
slices.SortFunc(dents, func(d1, d2 DirEntry) int {
76+
return strings.Compare(d1.Name(), d2.Name())
77+
})
78+
case SortFilesFirst:
79+
slices.SortFunc(dents, func(d1, d2 DirEntry) int {
80+
r1 := d1.Type().IsRegular()
81+
r2 := d2.Type().IsRegular()
82+
switch {
83+
case r1 && !r2:
84+
return -1
85+
case !r1 && r2:
86+
return 1
87+
case !r1 && !r2:
88+
// Both are not regular files: sort directories last
89+
dd1 := d1.Type().IsDir()
90+
dd2 := d2.Type().IsDir()
91+
switch {
92+
case !dd1 && dd2:
93+
return -1
94+
case dd1 && !dd2:
95+
return 1
96+
}
97+
}
98+
return strings.Compare(d1.Name(), d2.Name())
99+
})
100+
case SortDirsFirst:
101+
slices.SortFunc(dents, func(d1, d2 DirEntry) int {
102+
dd1 := d1.Type().IsDir()
103+
dd2 := d2.Type().IsDir()
104+
switch {
105+
case dd1 && !dd2:
106+
return -1
107+
case !dd1 && dd2:
108+
return 1
109+
case !dd1 && !dd2:
110+
// Both are not directories: sort regular files first
111+
r1 := d1.Type().IsRegular()
112+
r2 := d2.Type().IsRegular()
113+
switch {
114+
case r1 && !r2:
115+
return -1
116+
case !r1 && r2:
117+
return 1
118+
}
119+
}
120+
return strings.Compare(d1.Name(), d2.Name())
121+
})
122+
}
123+
}

‎dirent_portable_test.go

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
//go:build !darwin && !(aix || dragonfly || freebsd || (js && wasm) || linux || netbsd || openbsd || solaris)
2+
3+
package fastwalk
4+
5+
import (
6+
"io/fs"
7+
"math/rand"
8+
"reflect"
9+
"testing"
10+
"time"
11+
)
12+
13+
var _ DirEntry = dirEntry{}
14+
15+
// Minimal DirEntry for testing
16+
type dirEntry struct {
17+
name string
18+
typ fs.FileMode
19+
}
20+
21+
func (de dirEntry) Name() string { return de.name }
22+
func (de dirEntry) IsDir() bool { return de.typ.IsDir() }
23+
func (de dirEntry) Type() fs.FileMode { return de.typ.Type() }
24+
func (de dirEntry) Info() (fs.FileInfo, error) { panic("not implemented") }
25+
func (de dirEntry) Stat() (fs.FileInfo, error) { panic("not implemented") }
26+
27+
func (de dirEntry) String() string {
28+
return fs.FormatDirEntry(de)
29+
}
30+
31+
// NB: this must be kept in sync with the
32+
// TestSortDirents in dirent_unix_test.go
33+
func TestSortDirents(t *testing.T) {
34+
direntNames := func(dents []DirEntry) []string {
35+
names := make([]string, len(dents))
36+
for i, d := range dents {
37+
names[i] = d.Name()
38+
}
39+
return names
40+
}
41+
42+
t.Run("None", func(t *testing.T) {
43+
dents := []DirEntry{
44+
dirEntry{name: "b"},
45+
dirEntry{name: "a"},
46+
dirEntry{name: "d"},
47+
dirEntry{name: "c"},
48+
}
49+
want := direntNames(dents)
50+
sortDirents(SortNone, dents)
51+
got := direntNames(dents)
52+
if !reflect.DeepEqual(got, want) {
53+
t.Errorf("got: %q want: %q", got, want)
54+
}
55+
})
56+
57+
rr := rand.New(rand.NewSource(time.Now().UnixNano()))
58+
shuffleDirents := func(dents []DirEntry) []DirEntry {
59+
rr.Shuffle(len(dents), func(i, j int) {
60+
dents[i], dents[j] = dents[j], dents[i]
61+
})
62+
return dents
63+
}
64+
65+
// dents needs to be in the expected order
66+
test := func(t *testing.T, dents []DirEntry, mode SortMode) {
67+
want := direntNames(dents)
68+
// Run multiple times with different shuffles
69+
for i := 0; i < 10; i++ {
70+
t.Run("", func(t *testing.T) {
71+
sortDirents(mode, shuffleDirents(dents))
72+
got := direntNames(dents)
73+
if !reflect.DeepEqual(got, want) {
74+
t.Errorf("got: %q want: %q", got, want)
75+
}
76+
})
77+
}
78+
}
79+
80+
t.Run("Lexical", func(t *testing.T) {
81+
dents := []DirEntry{
82+
dirEntry{name: "a"},
83+
dirEntry{name: "b"},
84+
dirEntry{name: "c"},
85+
dirEntry{name: "d"},
86+
}
87+
test(t, dents, SortLexical)
88+
})
89+
90+
t.Run("FilesFirst", func(t *testing.T) {
91+
dents := []DirEntry{
92+
// Files lexically
93+
dirEntry{name: "f1", typ: 0},
94+
dirEntry{name: "f2", typ: 0},
95+
dirEntry{name: "f3", typ: 0},
96+
// Non-dirs lexically
97+
dirEntry{name: "a1", typ: fs.ModeSymlink},
98+
dirEntry{name: "a2", typ: fs.ModeSymlink},
99+
dirEntry{name: "a3", typ: fs.ModeSymlink},
100+
dirEntry{name: "s1", typ: fs.ModeSocket},
101+
dirEntry{name: "s2", typ: fs.ModeSocket},
102+
dirEntry{name: "s3", typ: fs.ModeSocket},
103+
// Dirs lexically
104+
dirEntry{name: "d1", typ: fs.ModeDir},
105+
dirEntry{name: "d2", typ: fs.ModeDir},
106+
dirEntry{name: "d3", typ: fs.ModeDir},
107+
}
108+
test(t, dents, SortFilesFirst)
109+
})
110+
111+
t.Run("DirsFirst", func(t *testing.T) {
112+
dents := []DirEntry{
113+
// Dirs lexically
114+
dirEntry{name: "d1", typ: fs.ModeDir},
115+
dirEntry{name: "d2", typ: fs.ModeDir},
116+
dirEntry{name: "d3", typ: fs.ModeDir},
117+
// Files lexically
118+
dirEntry{name: "f1", typ: 0},
119+
dirEntry{name: "f2", typ: 0},
120+
dirEntry{name: "f3", typ: 0},
121+
// Non-dirs lexically
122+
dirEntry{name: "a1", typ: fs.ModeSymlink},
123+
dirEntry{name: "a2", typ: fs.ModeSymlink},
124+
dirEntry{name: "a3", typ: fs.ModeSymlink},
125+
dirEntry{name: "s1", typ: fs.ModeSocket},
126+
dirEntry{name: "s2", typ: fs.ModeSocket},
127+
dirEntry{name: "s3", typ: fs.ModeSocket},
128+
}
129+
test(t, dents, SortDirsFirst)
130+
})
131+
}

0 commit comments

Comments
 (0)