diff --git a/cmd/tsgo/lsp.go b/cmd/tsgo/lsp.go index a47b4a00c0..28d460b29f 100644 --- a/cmd/tsgo/lsp.go +++ b/cmd/tsgo/lsp.go @@ -4,11 +4,13 @@ import ( "flag" "fmt" "os" + "runtime" "github.com/microsoft/typescript-go/internal/bundled" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/lsp" "github.com/microsoft/typescript-go/internal/pprof" + "github.com/microsoft/typescript-go/internal/tspath" "github.com/microsoft/typescript-go/internal/vfs/osvfs" ) @@ -37,6 +39,7 @@ func runLSP(args []string) int { fs := bundled.WrapFS(osvfs.FS()) defaultLibraryPath := bundled.LibPath() + typingsLocation := getGlobalTypingsCacheLocation() s := lsp.NewServer(&lsp.ServerOptions{ In: os.Stdin, @@ -45,6 +48,7 @@ func runLSP(args []string) int { Cwd: core.Must(os.Getwd()), FS: fs, DefaultLibraryPath: defaultLibraryPath, + TypingsLocation: typingsLocation, }) if err := s.Run(); err != nil { @@ -52,3 +56,70 @@ func runLSP(args []string) int { } return 0 } + +func getGlobalTypingsCacheLocation() string { + switch runtime.GOOS { + case "windows": + return tspath.CombinePaths(tspath.CombinePaths(getWindowsCacheLocation(), "Microsoft/TypeScript"), core.VersionMajorMinor()) + case "openbsd", "freebsd", "netbsd", "darwin", "linux", "android": + return tspath.CombinePaths(tspath.CombinePaths(getNonWindowsCacheLocation(), "typescript"), core.VersionMajorMinor()) + default: + panic("unsupported platform: " + runtime.GOOS) + } +} + +func getWindowsCacheLocation() string { + basePath, err := os.UserCacheDir() + if err != nil { + if basePath, err = os.UserConfigDir(); err != nil { + if basePath, err = os.UserHomeDir(); err != nil { + if userProfile := os.Getenv("USERPROFILE"); userProfile != "" { + basePath = userProfile + } else if homeDrive, homePath := os.Getenv("HOMEDRIVE"), os.Getenv("HOMEPATH"); homeDrive != "" && homePath != "" { + basePath = homeDrive + homePath + } else { + basePath = os.TempDir() + } + } + } + } + return basePath +} + +func getNonWindowsCacheLocation() string { + if xdgCacheHome := os.Getenv("XDG_CACHE_HOME"); xdgCacheHome != "" { + return xdgCacheHome + } + const platformIsDarwin = runtime.GOOS == "darwin" + var usersDir string + if platformIsDarwin { + usersDir = "Users" + } else { + usersDir = "home" + } + homePath, err := os.UserHomeDir() + if err != nil { + if home := os.Getenv("HOME"); home != "" { + homePath = home + } else { + var userName string + if logName := os.Getenv("LOGNAME"); logName != "" { + userName = logName + } else if user := os.Getenv("USER"); user != "" { + userName = user + } + if userName != "" { + homePath = "/" + usersDir + "/" + userName + } else { + homePath = os.TempDir() + } + } + } + var cacheFolder string + if platformIsDarwin { + cacheFolder = "Library/Caches" + } else { + cacheFolder = ".cache" + } + return tspath.CombinePaths(homePath, cacheFolder) +} diff --git a/internal/api/api.go b/internal/api/api.go index 6f8e16d6bc..d03df66cd6 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -73,6 +73,11 @@ func (api *API) DefaultLibraryPath() string { return api.host.DefaultLibraryPath() } +// TypingsInstaller implements ProjectHost +func (api *API) TypingsInstaller() *project.TypingsInstaller { + return nil +} + // DocumentRegistry implements ProjectHost. func (api *API) DocumentRegistry() *project.DocumentRegistry { return api.documentRegistry diff --git a/internal/compiler/program.go b/internal/compiler/program.go index 090c937d4e..5bad0857c2 100644 --- a/internal/compiler/program.go +++ b/internal/compiler/program.go @@ -28,6 +28,9 @@ type ProgramOptions struct { ProjectReference []core.ProjectReference ConfigFileParsingDiagnostics []*ast.Diagnostic CreateCheckerPool func(*Program) CheckerPool + + TypingsLocation string + ProjectName string } type Program struct { @@ -189,7 +192,7 @@ func NewProgram(options ProgramOptions) *Program { // tracing?.push(tracing.Phase.Program, "createProgram", { configFilePath: options.configFilePath, rootDir: options.rootDir }, /*separateBeginAndEnd*/ true); // performance.mark("beforeProgram"); - p.resolver = module.NewResolver(p.host, p.compilerOptions) + p.resolver = module.NewResolver(p.host, p.compilerOptions, p.programOptions.TypingsLocation, p.programOptions.ProjectName) var libs []string diff --git a/internal/core/nodemodules.go b/internal/core/nodemodules.go new file mode 100644 index 0000000000..c103e4f837 --- /dev/null +++ b/internal/core/nodemodules.go @@ -0,0 +1,88 @@ +package core + +import ( + "maps" + "sync" +) + +var UnprefixedNodeCoreModules = map[string]bool{ + "assert": true, + "assert/strict": true, + "async_hooks": true, + "buffer": true, + "child_process": true, + "cluster": true, + "console": true, + "constants": true, + "crypto": true, + "dgram": true, + "diagnostics_channel": true, + "dns": true, + "dns/promises": true, + "domain": true, + "events": true, + "fs": true, + "fs/promises": true, + "http": true, + "http2": true, + "https": true, + "inspector": true, + "inspector/promises": true, + "module": true, + "net": true, + "os": true, + "path": true, + "path/posix": true, + "path/win32": true, + "perf_hooks": true, + "process": true, + "punycode": true, + "querystring": true, + "readline": true, + "readline/promises": true, + "repl": true, + "stream": true, + "stream/consumers": true, + "stream/promises": true, + "stream/web": true, + "string_decoder": true, + "sys": true, + "test/mock_loader": true, + "timers": true, + "timers/promises": true, + "tls": true, + "trace_events": true, + "tty": true, + "url": true, + "util": true, + "util/types": true, + "v8": true, + "vm": true, + "wasi": true, + "worker_threads": true, + "zlib": true, +} + +var ExclusivelyPrefixedNodeCoreModules = map[string]bool{ + "node:sea": true, + "node:sqlite": true, + "node:test": true, + "node:test/reporters": true, +} + +var nodeCoreModules = sync.OnceValue(func() map[string]bool { + nodeCoreModules := make(map[string]bool, len(UnprefixedNodeCoreModules)*2+len(ExclusivelyPrefixedNodeCoreModules)) + for unprefixed := range UnprefixedNodeCoreModules { + nodeCoreModules[unprefixed] = true + nodeCoreModules["node:"+unprefixed] = true + } + maps.Copy(nodeCoreModules, ExclusivelyPrefixedNodeCoreModules) + return nodeCoreModules +}) + +func NonRelativeModuleNameForTypingCache(moduleName string) string { + if nodeCoreModules()[moduleName] { + return "node" + } + return moduleName +} diff --git a/internal/ls/completions_test.go b/internal/ls/completions_test.go index a13a31535f..7fb8821a02 100644 --- a/internal/ls/completions_test.go +++ b/internal/ls/completions_test.go @@ -2029,7 +2029,7 @@ func assertIncludesItem(t *testing.T, actual *lsproto.CompletionList, expected * } func createLanguageService(ctx context.Context, fileName string, files map[string]any) (*ls.LanguageService, func()) { - projectService, _ := projecttestutil.Setup(files) + projectService, _ := projecttestutil.Setup(files, nil) projectService.OpenFile(fileName, files[fileName].(string), core.ScriptKindTS, "") project := projectService.Projects()[0] return project.GetLanguageServiceForRequest(ctx) diff --git a/internal/lsp/server.go b/internal/lsp/server.go index 77d4824241..a16a09abc8 100644 --- a/internal/lsp/server.go +++ b/internal/lsp/server.go @@ -30,6 +30,7 @@ type ServerOptions struct { NewLine core.NewLineKind FS vfs.FS DefaultLibraryPath string + TypingsLocation string } func NewServer(opts *ServerOptions) *Server { @@ -48,6 +49,7 @@ func NewServer(opts *ServerOptions) *Server { newLine: opts.NewLine, fs: opts.FS, defaultLibraryPath: opts.DefaultLibraryPath, + typingsLocation: opts.TypingsLocation, } } @@ -79,6 +81,7 @@ type Server struct { newLine core.NewLineKind fs vfs.FS defaultLibraryPath string + typingsLocation string initializeParams *lsproto.InitializeParams positionEncoding lsproto.PositionEncodingKind @@ -100,6 +103,11 @@ func (s *Server) DefaultLibraryPath() string { return s.defaultLibraryPath } +// TypingsLocation implements project.ServiceHost. +func (s *Server) TypingsLocation() string { + return s.typingsLocation +} + // GetCurrentDirectory implements project.ServiceHost. func (s *Server) GetCurrentDirectory() string { return s.cwd @@ -477,6 +485,10 @@ func (s *Server) handleInitialized(ctx context.Context, req *lsproto.RequestMess Logger: s.logger, WatchEnabled: s.watchEnabled, PositionEncoding: s.positionEncoding, + TypingsInstallerOptions: project.TypingsInstallerOptions{ + ThrottleLimit: 5, + NpmInstall: project.NpmInstall, + }, }) return nil diff --git a/internal/module/resolver.go b/internal/module/resolver.go index 320b84a967..0a8035ce65 100644 --- a/internal/module/resolver.go +++ b/internal/module/resolver.go @@ -109,17 +109,23 @@ type Resolver struct { caches host ResolutionHost compilerOptions *core.CompilerOptions + typingsLocation string + projectName string // reportDiagnostic: DiagnosticReporter } func NewResolver( host ResolutionHost, options *core.CompilerOptions, + typingsLocation string, + projectName string, ) *Resolver { return &Resolver{ host: host, caches: newCaches(host.GetCurrentDirectory(), host.FS().UseCaseSensitiveFileNames(), options), compilerOptions: options, + typingsLocation: typingsLocation, + projectName: projectName, } } @@ -229,6 +235,36 @@ func (r *Resolver) ResolveModuleName(moduleName string, containingFile string, r } } + return r.tryResolveFromTypingsLocation(moduleName, containingDirectory, result) +} + +func (r *Resolver) tryResolveFromTypingsLocation(moduleName string, containingDirectory string, originalResult *ResolvedModule) *ResolvedModule { + if r.typingsLocation == "" || + tspath.IsExternalModuleNameRelative(moduleName) || + (originalResult.ResolvedFileName != "" && tspath.ExtensionIsOneOf(originalResult.Extension, tspath.SupportedTSExtensionsWithJsonFlat)) { + return originalResult + } + + state := newResolutionState( + moduleName, + containingDirectory, + false, /*isTypeReferenceDirective*/ + core.ModuleKindNone, // resolutionMode, + r.compilerOptions, + nil, // redirectedReference, + r, + ) + if r.traceEnabled() { + r.host.Trace(diagnostics.Auto_discovery_for_typings_is_enabled_in_project_0_Running_extra_resolution_pass_for_module_1_using_cache_location_2.Format(r.projectName, moduleName, r.typingsLocation)) + } + globalResolved := state.loadModuleFromImmediateNodeModulesDirectory(extensionsDeclaration, r.typingsLocation, false) + if globalResolved == nil { + return originalResult + } + result := state.createResolvedModule(globalResolved, true) + result.FailedLookupLocations = append(originalResult.FailedLookupLocations, result.FailedLookupLocations...) + result.AffectingLocations = append(originalResult.AffectingLocations, result.AffectingLocations...) + result.ResolutionDiagnostics = append(originalResult.ResolutionDiagnostics, result.ResolutionDiagnostics...) return result } @@ -1718,7 +1754,7 @@ func extensionIsOk(extensions extensions, extension string) bool { } func ResolveConfig(moduleName string, containingFile string, host ResolutionHost) *ResolvedModule { - resolver := NewResolver(host, &core.CompilerOptions{ModuleResolution: core.ModuleResolutionKindNodeNext}) + resolver := NewResolver(host, &core.CompilerOptions{ModuleResolution: core.ModuleResolutionKindNodeNext}, "", "") return resolver.resolveConfig(moduleName, containingFile) } diff --git a/internal/module/resolver_test.go b/internal/module/resolver_test.go index 74502764a3..6274422887 100644 --- a/internal/module/resolver_test.go +++ b/internal/module/resolver_test.go @@ -280,7 +280,7 @@ func runTraceBaseline(t *testing.T, test traceTestCase) { t.Parallel() host := newVFSModuleResolutionHost(test.files, test.currentDirectory) - resolver := module.NewResolver(host, test.compilerOptions) + resolver := module.NewResolver(host, test.compilerOptions, "", "") for _, call := range test.calls { doCall(t, resolver, call, false /*skipLocations*/) @@ -291,7 +291,7 @@ func runTraceBaseline(t *testing.T, test traceTestCase) { t.Run("concurrent", func(t *testing.T) { concurrentHost := newVFSModuleResolutionHost(test.files, test.currentDirectory) - concurrentResolver := module.NewResolver(concurrentHost, test.compilerOptions) + concurrentResolver := module.NewResolver(concurrentHost, test.compilerOptions, "", "") var wg sync.WaitGroup for _, call := range test.calls { diff --git a/internal/packagejson/packagejson.go b/internal/packagejson/packagejson.go index 41c994ad88..07dd14f8b5 100644 --- a/internal/packagejson/packagejson.go +++ b/internal/packagejson/packagejson.go @@ -22,6 +22,7 @@ type PathFields struct { type DependencyFields struct { Dependencies Expected[map[string]string] `json:"dependencies"` + DevDependencies Expected[map[string]string] `json:"devDependencies"` PeerDependencies Expected[map[string]string] `json:"peerDependencies"` OptionalDependencies Expected[map[string]string] `json:"optionalDependencies"` } diff --git a/internal/parser/references.go b/internal/parser/references.go index 22e1bbe5d7..e3c130eb10 100644 --- a/internal/parser/references.go +++ b/internal/parser/references.go @@ -35,10 +35,10 @@ func collectModuleReferences(file *ast.SourceFile, node *ast.Statement, inAmbien ast.SetImportsOfSourceFile(file, append(file.Imports(), moduleNameExpr)) // !!! removed `&& p.currentNodeModulesDepth == 0` if file.UsesUriStyleNodeCoreModules != core.TSTrue && !file.IsDeclarationFile { - if strings.HasPrefix(moduleName, "node:") && !exclusivelyPrefixedNodeCoreModules[moduleName] { + if strings.HasPrefix(moduleName, "node:") && !core.ExclusivelyPrefixedNodeCoreModules[moduleName] { // Presence of `node:` prefix takes precedence over unprefixed node core modules file.UsesUriStyleNodeCoreModules = core.TSTrue - } else if file.UsesUriStyleNodeCoreModules == core.TSUnknown && unprefixedNodeCoreModules[moduleName] { + } else if file.UsesUriStyleNodeCoreModules == core.TSUnknown && core.UnprefixedNodeCoreModules[moduleName] { // Avoid `unprefixedNodeCoreModules.has` for every import file.UsesUriStyleNodeCoreModules = core.TSFalse } @@ -75,68 +75,3 @@ func collectModuleReferences(file *ast.SourceFile, node *ast.Statement, inAmbien } } } - -var unprefixedNodeCoreModules = map[string]bool{ - "assert": true, - "assert/strict": true, - "async_hooks": true, - "buffer": true, - "child_process": true, - "cluster": true, - "console": true, - "constants": true, - "crypto": true, - "dgram": true, - "diagnostics_channel": true, - "dns": true, - "dns/promises": true, - "domain": true, - "events": true, - "fs": true, - "fs/promises": true, - "http": true, - "http2": true, - "https": true, - "inspector": true, - "inspector/promises": true, - "module": true, - "net": true, - "os": true, - "path": true, - "path/posix": true, - "path/win32": true, - "perf_hooks": true, - "process": true, - "punycode": true, - "querystring": true, - "readline": true, - "readline/promises": true, - "repl": true, - "stream": true, - "stream/consumers": true, - "stream/promises": true, - "stream/web": true, - "string_decoder": true, - "sys": true, - "test/mock_loader": true, - "timers": true, - "timers/promises": true, - "tls": true, - "trace_events": true, - "tty": true, - "url": true, - "util": true, - "util/types": true, - "v8": true, - "vm": true, - "wasi": true, - "worker_threads": true, - "zlib": true, -} - -var exclusivelyPrefixedNodeCoreModules = map[string]bool{ - "node:sea": true, - "node:sqlite": true, - "node:test": true, - "node:test/reporters": true, -} diff --git a/internal/project/discovertypings.go b/internal/project/discovertypings.go new file mode 100644 index 0000000000..b59b063317 --- /dev/null +++ b/internal/project/discovertypings.go @@ -0,0 +1,331 @@ +package project + +import ( + "encoding/json" + "fmt" + "maps" + "slices" + "unicode/utf8" + + "github.com/microsoft/typescript-go/internal/collections" + "github.com/microsoft/typescript-go/internal/core" + "github.com/microsoft/typescript-go/internal/packagejson" + "github.com/microsoft/typescript-go/internal/semver" + "github.com/microsoft/typescript-go/internal/tspath" + "github.com/microsoft/typescript-go/internal/vfs" +) + +type CachedTyping struct { + TypingsLocation string + Version semver.Version +} + +func IsTypingUpToDate(cachedTyping *CachedTyping, availableTypingVersions map[string]string) bool { + useVersion, ok := availableTypingVersions["ts"+core.VersionMajorMinor()] + if !ok { + useVersion = availableTypingVersions["latest"] + } + availableVersion := semver.MustParse(useVersion) + return availableVersion.Compare(&cachedTyping.Version) <= 0 +} + +func DiscoverTypings( + fs vfs.FS, + log func(s string), + typingsInfo *TypingsInfo, + fileNames []string, + projectRootPath string, + safeList map[string]string, + packageNameToTypingLocation *collections.SyncMap[string, *CachedTyping], + typesRegistry map[string]map[string]string, +) (cachedTypingPaths []string, newTypingNames []string, filesToWatch []string) { + // A typing name to typing file path mapping + inferredTypings := map[string]string{} + + // Only infer typings for .js and .jsx files + fileNames = core.Filter(fileNames, func(fileName string) bool { + return tspath.HasJSFileExtension(fileName) + }) + + if typingsInfo.TypeAcquisition.Include != nil { + addInferredTypings(fs, log, inferredTypings, typingsInfo.TypeAcquisition.Include, "Explicitly included types") + } + exclude := typingsInfo.TypeAcquisition.Exclude + + // Directories to search for package.json, bower.json and other typing information + if typingsInfo.CompilerOptions.Types == nil { + possibleSearchDirs := map[string]bool{} + for _, fileName := range fileNames { + possibleSearchDirs[tspath.GetDirectoryPath(fileName)] = true + } + possibleSearchDirs[projectRootPath] = true + for searchDir := range possibleSearchDirs { + filesToWatch = addTypingNamesAndGetFilesToWatch(fs, log, inferredTypings, filesToWatch, searchDir, "bower.json", "bower_components") + filesToWatch = addTypingNamesAndGetFilesToWatch(fs, log, inferredTypings, filesToWatch, searchDir, "package.json", "node_modules") + } + } + + if !typingsInfo.TypeAcquisition.DisableFilenameBasedTypeAcquisition.IsTrue() { + getTypingNamesFromSourceFileNames(fs, log, inferredTypings, safeList, fileNames) + } + + // add typings for unresolved imports + modules := slices.Compact(core.Map(typingsInfo.UnresolvedImports, core.NonRelativeModuleNameForTypingCache)) + addInferredTypings(fs, log, inferredTypings, modules, "Inferred typings from unresolved imports") + + // Remove typings that the user has added to the exclude list + for _, excludeTypingName := range exclude { + delete(inferredTypings, excludeTypingName) + log(fmt.Sprintf("TI:: Typing for %s is in exclude list, will be ignored.", excludeTypingName)) + } + + // Add the cached typing locations for inferred typings that are already installed + packageNameToTypingLocation.Range(func(name string, typing *CachedTyping) bool { + registryEntry := typesRegistry[name] + if inferredTypings[name] == "" && registryEntry != nil && IsTypingUpToDate(typing, registryEntry) { + inferredTypings[name] = typing.TypingsLocation + } + return true + }) + + for typing, inferred := range inferredTypings { + if inferred != "" { + cachedTypingPaths = append(cachedTypingPaths, inferred) + } else { + newTypingNames = append(newTypingNames, typing) + } + } + log(fmt.Sprintf("TI:: Finished typings discovery: cachedTypingsPaths: %v newTypingNames: %v, filesToWatch %v", cachedTypingPaths, newTypingNames, filesToWatch)) + return cachedTypingPaths, newTypingNames, filesToWatch +} + +func addInferredTyping(inferredTypings map[string]string, typingName string) { + if _, ok := inferredTypings[typingName]; !ok { + inferredTypings[typingName] = "" + } +} + +func addInferredTypings( + fs vfs.FS, + log func(s string), + inferredTypings map[string]string, + typingNames []string, message string, +) { + log(fmt.Sprintf("TI:: %s: %v", message, typingNames)) + for _, typingName := range typingNames { + addInferredTyping(inferredTypings, typingName) + } +} + +/** + * Infer typing names from given file names. For example, the file name "jquery-min.2.3.4.js" + * should be inferred to the 'jquery' typing name; and "angular-route.1.2.3.js" should be inferred + * to the 'angular-route' typing name. + * @param fileNames are the names for source files in the project + */ +func getTypingNamesFromSourceFileNames( + fs vfs.FS, + log func(s string), + inferredTypings map[string]string, + safeList map[string]string, + fileNames []string, +) { + hasJsxFile := false + var fromFileNames []string + for _, fileName := range fileNames { + hasJsxFile = hasJsxFile || tspath.FileExtensionIs(fileName, tspath.ExtensionJsx) + inferredTypingName := tspath.RemoveFileExtension(tspath.ToFileNameLowerCase(tspath.GetBaseFileName(fileName))) + cleanedTypingName := removeMinAndVersionNumbers(inferredTypingName) + if safeName, ok := safeList[cleanedTypingName]; ok { + fromFileNames = append(fromFileNames, safeName) + } + } + if len(fromFileNames) > 0 { + addInferredTypings(fs, log, inferredTypings, fromFileNames, "Inferred typings from file names") + } + if hasJsxFile { + log("TI:: Inferred 'react' typings due to presence of '.jsx' extension") + addInferredTyping(inferredTypings, "react") + } +} + +/** + * Adds inferred typings from manifest/module pairs (think package.json + node_modules) + * + * @param projectRootPath is the path to the directory where to look for package.json, bower.json and other typing information + * @param manifestName is the name of the manifest (package.json or bower.json) + * @param modulesDirName is the directory name for modules (node_modules or bower_components). Should be lowercase! + * @param filesToWatch are the files to watch for changes. We will push things into this array. + */ +func addTypingNamesAndGetFilesToWatch( + fs vfs.FS, + log func(s string), + inferredTypings map[string]string, + filesToWatch []string, + projectRootPath string, + manifestName string, + modulesDirName string, +) []string { + // First, we check the manifests themselves. They're not + // _required_, but they allow us to do some filtering when dealing + // with big flat dep directories. + manifestPath := tspath.CombinePaths(projectRootPath, manifestName) + var manifestTypingNames []string + manifestContents, ok := fs.ReadFile(manifestPath) + if ok { + var manifest packagejson.DependencyFields + filesToWatch = append(filesToWatch, manifestPath) + // var manifest map[string]any + err := json.Unmarshal([]byte(manifestContents), &manifest) + if err == nil { + manifestTypingNames = slices.AppendSeq(manifestTypingNames, maps.Keys(manifest.Dependencies.Value)) + manifestTypingNames = slices.AppendSeq(manifestTypingNames, maps.Keys(manifest.DevDependencies.Value)) + manifestTypingNames = slices.AppendSeq(manifestTypingNames, maps.Keys(manifest.OptionalDependencies.Value)) + manifestTypingNames = slices.AppendSeq(manifestTypingNames, maps.Keys(manifest.PeerDependencies.Value)) + addInferredTypings(fs, log, inferredTypings, manifestTypingNames, "Typing names in '"+manifestPath+"' dependencies") + } + } + + // Now we scan the directories for typing information in + // already-installed dependencies (if present). Note that this + // step happens regardless of whether a manifest was present, + // which is certainly a valid configuration, if an unusual one. + packagesFolderPath := tspath.CombinePaths(projectRootPath, modulesDirName) + filesToWatch = append(filesToWatch, packagesFolderPath) + if !fs.DirectoryExists(packagesFolderPath) { + return filesToWatch + } + + // There's two cases we have to take into account here: + // 1. If manifest is undefined, then we're not using a manifest. + // That means that we should scan _all_ dependencies at the top + // level of the modulesDir. + // 2. If manifest is defined, then we can do some special + // filtering to reduce the amount of scanning we need to do. + // + // Previous versions of this algorithm checked for a `_requiredBy` + // field in the package.json, but that field is only present in + // `npm@>=3 <7`. + + // Package names that do **not** provide their own typings, so + // we'll look them up. + var packageNames []string + + var dependencyManifestNames []string + if len(manifestTypingNames) > 0 { + // This is #1 described above. + for _, typingName := range manifestTypingNames { + dependencyManifestNames = append(dependencyManifestNames, tspath.CombinePaths(packagesFolderPath, typingName, manifestName)) + } + } else { + // And #2. Depth = 3 because scoped packages look like `node_modules/@foo/bar/package.json` + depth := 3 + for _, manifestPath := range vfs.ReadDirectory(fs, projectRootPath, packagesFolderPath, []string{tspath.ExtensionJson}, nil, nil, &depth) { + if tspath.GetBaseFileName(manifestPath) != manifestName { + continue + } + + // It's ok to treat + // `node_modules/@foo/bar/package.json` as a manifest, + // but not `node_modules/jquery/nested/package.json`. + // We only assume depth 3 is ok for formally scoped + // packages. So that needs this dance here. + + pathComponents := tspath.GetPathComponents(manifestPath, "") + lenPathComponents := len(pathComponents) + ch, _ := utf8.DecodeRuneInString(pathComponents[lenPathComponents-3]) + isScoped := ch == '@' + + if isScoped && tspath.ToFileNameLowerCase(pathComponents[lenPathComponents-4]) == modulesDirName || // `node_modules/@foo/bar` + !isScoped && tspath.ToFileNameLowerCase(pathComponents[lenPathComponents-3]) == modulesDirName { // `node_modules/foo` + dependencyManifestNames = append(dependencyManifestNames, manifestPath) + } + } + + } + + log(fmt.Sprintf("TI:: Searching for typing names in %s; all files: %v", packagesFolderPath, dependencyManifestNames)) + + // Once we have the names of things to look up, we iterate over + // and either collect their included typings, or add them to the + // list of typings we need to look up separately. + for _, manifestPath := range dependencyManifestNames { + manifestContents, ok := fs.ReadFile(manifestPath) + if !ok { + continue + } + manifest, err := packagejson.Parse([]byte(manifestContents)) + // If the package has its own d.ts typings, those will take precedence. Otherwise the package name will be used + // to download d.ts files from DefinitelyTyped + if err != nil || len(manifest.Name.Value) == 0 { + continue + } + ownTypes := manifest.Types.Value + if len(ownTypes) == 0 { + ownTypes = manifest.Typings.Value + } + if len(ownTypes) != 0 { + absolutePath := tspath.GetNormalizedAbsolutePath(ownTypes, tspath.GetDirectoryPath(manifestPath)) + if fs.FileExists(absolutePath) { + log(fmt.Sprintf("TI:: Package '%s' provides its own types.", manifest.Name.Value)) + inferredTypings[manifest.Name.Value] = absolutePath + } else { + log(fmt.Sprintf("TI:: Package '%s' provides its own types but they are missing.", manifest.Name.Value)) + } + } else { + packageNames = append(packageNames, manifest.Name.Value) + } + } + addInferredTypings(fs, log, inferredTypings, packageNames, " Found package names") + return filesToWatch +} + +/** + * Takes a string like "jquery-min.4.2.3" and returns "jquery" + * + * @internal + */ +func removeMinAndVersionNumbers(fileName string) string { + // We used to use the regex /[.-]((min)|(\d+(\.\d+)*))$/ and would just .replace it twice. + // Unfortunately, that regex has O(n^2) performance because v8 doesn't match from the end of the string. + // Instead, we now essentially scan the filename (backwards) ourselves. + end := len(fileName) + for pos := end; pos > 0; { + ch, size := utf8.DecodeLastRuneInString(fileName[:pos]) + if ch >= '0' && ch <= '9' { + // Match a \d+ segment + for { + pos -= size + ch, size = utf8.DecodeLastRuneInString(fileName[:pos]) + if pos <= 0 || ch < '0' || ch > '9' { + break + } + } + } else if pos > 4 && (ch == 'n' || ch == 'N') { + // Looking for "min" or "min" + // Already matched the 'n' + pos -= size + ch, size = utf8.DecodeLastRuneInString(fileName[:pos]) + if ch != 'i' && ch != 'I' { + break + } + pos -= size + ch, size = utf8.DecodeLastRuneInString(fileName[:pos]) + if ch != 'm' && ch != 'M' { + break + } + pos -= size + ch, size = utf8.DecodeLastRuneInString(fileName[:pos]) + } else { + // This character is not part of either suffix pattern + break + } + + if ch != '-' && ch != '.' { + break + } + pos -= size + end = pos + } + return fileName[0:end] +} diff --git a/internal/project/discovertypings_test.go b/internal/project/discovertypings_test.go new file mode 100644 index 0000000000..dd15959178 --- /dev/null +++ b/internal/project/discovertypings_test.go @@ -0,0 +1,380 @@ +package project_test + +import ( + "maps" + "testing" + + "github.com/microsoft/typescript-go/internal/collections" + "github.com/microsoft/typescript-go/internal/core" + "github.com/microsoft/typescript-go/internal/project" + "github.com/microsoft/typescript-go/internal/semver" + "github.com/microsoft/typescript-go/internal/testutil/projecttestutil" + "github.com/microsoft/typescript-go/internal/vfs/vfstest" + "gotest.tools/v3/assert" +) + +func TestDiscoverTypings(t *testing.T) { + t.Parallel() + t.Run("should use mappings from safe list", func(t *testing.T) { + t.Parallel() + var output []string + files := map[string]string{ + "/home/src/projects/project/app.js": "", + "/home/src/projects/project/jquery.js": "", + "/home/src/projects/project/chroma.min.js": "", + } + fs := vfstest.FromMap(files, false /*useCaseSensitiveFileNames*/) + cachedTypingPaths, newTypingNames, filesToWatch := project.DiscoverTypings( + fs, + func(s string) { + output = append(output, s) + }, + &project.TypingsInfo{ + CompilerOptions: &core.CompilerOptions{}, + TypeAcquisition: &core.TypeAcquisition{Enable: core.TSTrue}, + UnresolvedImports: []string{}, + }, + []string{"/home/src/projects/project/app.js", "/home/src/projects/project/jquery.js", "/home/src/projects/project/chroma.min.js"}, + "/home/src/projects/project", + map[string]string{ + "jquery": "jquery", + "chroma": "chroma-js", + }, + &collections.SyncMap[string, *project.CachedTyping]{}, + map[string]map[string]string{}, + ) + assert.Assert(t, cachedTypingPaths == nil) + assert.DeepEqual(t, core.NewSetFromItems(newTypingNames...), core.NewSetFromItems( + "jquery", + "chroma-js", + )) + assert.DeepEqual(t, filesToWatch, []string{ + "/home/src/projects/project/bower_components", + "/home/src/projects/project/node_modules", + }) + }) + + t.Run("should return node for core modules", func(t *testing.T) { + t.Parallel() + var output []string + files := map[string]string{ + "/home/src/projects/project/app.js": "", + } + fs := vfstest.FromMap(files, false /*useCaseSensitiveFileNames*/) + cachedTypingPaths, newTypingNames, filesToWatch := project.DiscoverTypings( + fs, + func(s string) { + output = append(output, s) + }, + &project.TypingsInfo{ + CompilerOptions: &core.CompilerOptions{}, + TypeAcquisition: &core.TypeAcquisition{Enable: core.TSTrue}, + UnresolvedImports: []string{"assert", "somename"}, + }, + []string{"/home/src/projects/project/app.js"}, + "/home/src/projects/project", + map[string]string{}, + &collections.SyncMap[string, *project.CachedTyping]{}, + map[string]map[string]string{}, + ) + assert.Assert(t, cachedTypingPaths == nil) + assert.DeepEqual(t, core.NewSetFromItems(newTypingNames...), core.NewSetFromItems( + "node", + "somename", + )) + assert.DeepEqual(t, filesToWatch, []string{ + "/home/src/projects/project/bower_components", + "/home/src/projects/project/node_modules", + }) + }) + + t.Run("should use cached locations", func(t *testing.T) { + t.Parallel() + var output []string + files := map[string]string{ + "/home/src/projects/project/app.js": "", + "/home/src/projects/project/node.d.ts": "", + } + fs := vfstest.FromMap(files, false /*useCaseSensitiveFileNames*/) + cache := collections.SyncMap[string, *project.CachedTyping]{} + cache.Store("node", &project.CachedTyping{ + TypingsLocation: "/home/src/projects/project/node.d.ts", + Version: semver.MustParse("1.3.0"), + }) + cachedTypingPaths, newTypingNames, filesToWatch := project.DiscoverTypings( + fs, + func(s string) { + output = append(output, s) + }, + &project.TypingsInfo{ + CompilerOptions: &core.CompilerOptions{}, + TypeAcquisition: &core.TypeAcquisition{Enable: core.TSTrue}, + UnresolvedImports: []string{"fs", "bar"}, + }, + []string{"/home/src/projects/project/app.js"}, + "/home/src/projects/project", + map[string]string{}, + &cache, + map[string]map[string]string{ + "node": projecttestutil.TypesRegistryConfig(), + }, + ) + assert.DeepEqual(t, cachedTypingPaths, []string{ + "/home/src/projects/project/node.d.ts", + }) + assert.DeepEqual(t, core.NewSetFromItems(newTypingNames...), core.NewSetFromItems( + "bar", + )) + assert.DeepEqual(t, filesToWatch, []string{ + "/home/src/projects/project/bower_components", + "/home/src/projects/project/node_modules", + }) + }) + + t.Run("should gracefully handle packages that have been removed from the types-registry", func(t *testing.T) { + t.Parallel() + var output []string + files := map[string]string{ + "/home/src/projects/project/app.js": "", + "/home/src/projects/project/node.d.ts": "", + } + fs := vfstest.FromMap(files, false /*useCaseSensitiveFileNames*/) + cache := collections.SyncMap[string, *project.CachedTyping]{} + cache.Store("node", &project.CachedTyping{ + TypingsLocation: "/home/src/projects/project/node.d.ts", + Version: semver.MustParse("1.3.0"), + }) + cachedTypingPaths, newTypingNames, filesToWatch := project.DiscoverTypings( + fs, + func(s string) { + output = append(output, s) + }, + &project.TypingsInfo{ + CompilerOptions: &core.CompilerOptions{}, + TypeAcquisition: &core.TypeAcquisition{Enable: core.TSTrue}, + UnresolvedImports: []string{"fs", "bar"}, + }, + []string{"/home/src/projects/project/app.js"}, + "/home/src/projects/project", + map[string]string{}, + &cache, + map[string]map[string]string{}, + ) + assert.Assert(t, cachedTypingPaths == nil) + assert.DeepEqual(t, core.NewSetFromItems(newTypingNames...), core.NewSetFromItems( + "node", + "bar", + )) + assert.DeepEqual(t, filesToWatch, []string{ + "/home/src/projects/project/bower_components", + "/home/src/projects/project/node_modules", + }) + }) + + t.Run("should search only 2 levels deep", func(t *testing.T) { + t.Parallel() + var output []string + files := map[string]string{ + "/home/src/projects/project/app.js": "", + "/home/src/projects/project/node_modules/a/package.json": `{ "name": "a" }`, + "/home/src/projects/project/node_modules/a/b/package.json": `{ "name": "b" }`, + } + fs := vfstest.FromMap(files, false /*useCaseSensitiveFileNames*/) + cachedTypingPaths, newTypingNames, filesToWatch := project.DiscoverTypings( + fs, + func(s string) { + output = append(output, s) + }, + &project.TypingsInfo{ + CompilerOptions: &core.CompilerOptions{}, + TypeAcquisition: &core.TypeAcquisition{Enable: core.TSTrue}, + UnresolvedImports: []string{}, + }, + []string{"/home/src/projects/project/app.js"}, + "/home/src/projects/project", + map[string]string{}, + &collections.SyncMap[string, *project.CachedTyping]{}, + map[string]map[string]string{}, + ) + assert.Assert(t, cachedTypingPaths == nil) + assert.DeepEqual(t, core.NewSetFromItems(newTypingNames...), core.NewSetFromItems( + "a", + )) + assert.DeepEqual(t, filesToWatch, []string{ + "/home/src/projects/project/bower_components", + "/home/src/projects/project/node_modules", + }) + }) + + t.Run("should support scoped packages", func(t *testing.T) { + t.Parallel() + var output []string + files := map[string]string{ + "/home/src/projects/project/app.js": "", + "/home/src/projects/project/node_modules/@a/b/package.json": `{ "name": "@a/b" }`, + } + fs := vfstest.FromMap(files, false /*useCaseSensitiveFileNames*/) + cachedTypingPaths, newTypingNames, filesToWatch := project.DiscoverTypings( + fs, + func(s string) { + output = append(output, s) + }, + &project.TypingsInfo{ + CompilerOptions: &core.CompilerOptions{}, + TypeAcquisition: &core.TypeAcquisition{Enable: core.TSTrue}, + UnresolvedImports: []string{}, + }, + []string{"/home/src/projects/project/app.js"}, + "/home/src/projects/project", + map[string]string{}, + &collections.SyncMap[string, *project.CachedTyping]{}, + map[string]map[string]string{}, + ) + assert.Assert(t, cachedTypingPaths == nil) + assert.DeepEqual(t, core.NewSetFromItems(newTypingNames...), core.NewSetFromItems( + "@a/b", + )) + assert.DeepEqual(t, filesToWatch, []string{ + "/home/src/projects/project/bower_components", + "/home/src/projects/project/node_modules", + }) + }) + + t.Run("should install expired typings", func(t *testing.T) { + t.Parallel() + var output []string + files := map[string]string{ + "/home/src/projects/project/app.js": "", + } + fs := vfstest.FromMap(files, false /*useCaseSensitiveFileNames*/) + cache := collections.SyncMap[string, *project.CachedTyping]{} + cache.Store("node", &project.CachedTyping{ + TypingsLocation: projecttestutil.TestTypingsLocation + "/node_modules/@types/node/index.d.ts", + Version: semver.MustParse("1.3.0"), + }) + cache.Store("commander", &project.CachedTyping{ + TypingsLocation: projecttestutil.TestTypingsLocation + "/node_modules/@types/commander/index.d.ts", + Version: semver.MustParse("1.0.0"), + }) + cachedTypingPaths, newTypingNames, filesToWatch := project.DiscoverTypings( + fs, + func(s string) { + output = append(output, s) + }, + &project.TypingsInfo{ + CompilerOptions: &core.CompilerOptions{}, + TypeAcquisition: &core.TypeAcquisition{Enable: core.TSTrue}, + UnresolvedImports: []string{"http", "commander"}, + }, + []string{"/home/src/projects/project/app.js"}, + "/home/src/projects/project", + map[string]string{}, + &cache, + map[string]map[string]string{ + "node": projecttestutil.TypesRegistryConfig(), + "commander": projecttestutil.TypesRegistryConfig(), + }, + ) + assert.DeepEqual(t, cachedTypingPaths, []string{ + "/home/src/Library/Caches/typescript/node_modules/@types/node/index.d.ts", + }) + assert.DeepEqual(t, core.NewSetFromItems(newTypingNames...), core.NewSetFromItems( + "commander", + )) + assert.DeepEqual(t, filesToWatch, []string{ + "/home/src/projects/project/bower_components", + "/home/src/projects/project/node_modules", + }) + }) + + t.Run("should install expired typings with prerelease version of tsserver", func(t *testing.T) { + t.Parallel() + var output []string + files := map[string]string{ + "/home/src/projects/project/app.js": "", + } + fs := vfstest.FromMap(files, false /*useCaseSensitiveFileNames*/) + cache := collections.SyncMap[string, *project.CachedTyping]{} + cache.Store("node", &project.CachedTyping{ + TypingsLocation: projecttestutil.TestTypingsLocation + "/node_modules/@types/node/index.d.ts", + Version: semver.MustParse("1.0.0"), + }) + config := maps.Clone(projecttestutil.TypesRegistryConfig()) + delete(config, "ts"+core.VersionMajorMinor()) + + cachedTypingPaths, newTypingNames, filesToWatch := project.DiscoverTypings( + fs, + func(s string) { + output = append(output, s) + }, + &project.TypingsInfo{ + CompilerOptions: &core.CompilerOptions{}, + TypeAcquisition: &core.TypeAcquisition{Enable: core.TSTrue}, + UnresolvedImports: []string{"http"}, + }, + []string{"/home/src/projects/project/app.js"}, + "/home/src/projects/project", + map[string]string{}, + &cache, + map[string]map[string]string{ + "node": config, + }, + ) + assert.Assert(t, cachedTypingPaths == nil) + assert.DeepEqual(t, core.NewSetFromItems(newTypingNames...), core.NewSetFromItems( + "node", + )) + assert.DeepEqual(t, filesToWatch, []string{ + "/home/src/projects/project/bower_components", + "/home/src/projects/project/node_modules", + }) + }) + + t.Run("prerelease typings are properly handled", func(t *testing.T) { + t.Parallel() + var output []string + files := map[string]string{ + "/home/src/projects/project/app.js": "", + } + fs := vfstest.FromMap(files, false /*useCaseSensitiveFileNames*/) + cache := collections.SyncMap[string, *project.CachedTyping]{} + cache.Store("node", &project.CachedTyping{ + TypingsLocation: projecttestutil.TestTypingsLocation + "/node_modules/@types/node/index.d.ts", + Version: semver.MustParse("1.3.0-next.0"), + }) + cache.Store("commander", &project.CachedTyping{ + TypingsLocation: projecttestutil.TestTypingsLocation + "/node_modules/@types/commander/index.d.ts", + Version: semver.MustParse("1.3.0-next.0"), + }) + config := maps.Clone(projecttestutil.TypesRegistryConfig()) + config["ts"+core.VersionMajorMinor()] = "1.3.0-next.1" + cachedTypingPaths, newTypingNames, filesToWatch := project.DiscoverTypings( + fs, + func(s string) { + output = append(output, s) + }, + &project.TypingsInfo{ + CompilerOptions: &core.CompilerOptions{}, + TypeAcquisition: &core.TypeAcquisition{Enable: core.TSTrue}, + UnresolvedImports: []string{"http", "commander"}, + }, + []string{"/home/src/projects/project/app.js"}, + "/home/src/projects/project", + map[string]string{}, + &cache, + map[string]map[string]string{ + "node": config, + "commander": projecttestutil.TypesRegistryConfig(), + }, + ) + assert.Assert(t, cachedTypingPaths == nil) + assert.DeepEqual(t, core.NewSetFromItems(newTypingNames...), core.NewSetFromItems( + "node", + "commander", + )) + assert.DeepEqual(t, filesToWatch, []string{ + "/home/src/projects/project/bower_components", + "/home/src/projects/project/node_modules", + }) + }) +} diff --git a/internal/project/host.go b/internal/project/host.go index 18bad0b808..94c4618e47 100644 --- a/internal/project/host.go +++ b/internal/project/host.go @@ -18,6 +18,7 @@ type Client interface { type ServiceHost interface { FS() vfs.FS DefaultLibraryPath() string + TypingsLocation() string GetCurrentDirectory() string NewLine() string diff --git a/internal/project/installnpmpackages_test.go b/internal/project/installnpmpackages_test.go new file mode 100644 index 0000000000..2f7801fae5 --- /dev/null +++ b/internal/project/installnpmpackages_test.go @@ -0,0 +1,520 @@ +package project_test + +import ( + "sync/atomic" + "testing" + + "github.com/microsoft/typescript-go/internal/project" + "gotest.tools/v3/assert" +) + +func TestInstallNpmPackages(t *testing.T) { + t.Parallel() + packageNames := []string{ + "@types/graphql@ts2.8", + "@types/highlight.js@ts2.8", + "@types/jest@ts2.8", + "@types/mini-css-extract-plugin@ts2.8", + "@types/mongoose@ts2.8", + "@types/pg@ts2.8", + "@types/webpack-bundle-analyzer@ts2.8", + "@types/enhanced-resolve@ts2.8", + "@types/eslint-plugin-prettier@ts2.8", + "@types/friendly-errors-webpack-plugin@ts2.8", + "@types/hammerjs@ts2.8", + "@types/history@ts2.8", + "@types/image-size@ts2.8", + "@types/js-cookie@ts2.8", + "@types/koa-compress@ts2.8", + "@types/less@ts2.8", + "@types/material-ui@ts2.8", + "@types/mysql@ts2.8", + "@types/nodemailer@ts2.8", + "@types/prettier@ts2.8", + "@types/query-string@ts2.8", + "@types/react-places-autocomplete@ts2.8", + "@types/react-router@ts2.8", + "@types/react-router-config@ts2.8", + "@types/react-select@ts2.8", + "@types/react-transition-group@ts2.8", + "@types/redux-form@ts2.8", + "@types/abbrev@ts2.8", + "@types/accepts@ts2.8", + "@types/acorn@ts2.8", + "@types/ansi-regex@ts2.8", + "@types/ansi-styles@ts2.8", + "@types/anymatch@ts2.8", + "@types/apollo-codegen@ts2.8", + "@types/are-we-there-yet@ts2.8", + "@types/argparse@ts2.8", + "@types/arr-union@ts2.8", + "@types/array-find-index@ts2.8", + "@types/array-uniq@ts2.8", + "@types/array-unique@ts2.8", + "@types/arrify@ts2.8", + "@types/assert-plus@ts2.8", + "@types/async@ts2.8", + "@types/autoprefixer@ts2.8", + "@types/aws4@ts2.8", + "@types/babel-code-frame@ts2.8", + "@types/babel-generator@ts2.8", + "@types/babel-plugin-syntax-jsx@ts2.8", + "@types/babel-template@ts2.8", + "@types/babel-traverse@ts2.8", + "@types/babel-types@ts2.8", + "@types/babylon@ts2.8", + "@types/base64-js@ts2.8", + "@types/basic-auth@ts2.8", + "@types/big.js@ts2.8", + "@types/bl@ts2.8", + "@types/bluebird@ts2.8", + "@types/body-parser@ts2.8", + "@types/bonjour@ts2.8", + "@types/boom@ts2.8", + "@types/brace-expansion@ts2.8", + "@types/braces@ts2.8", + "@types/brorand@ts2.8", + "@types/browser-resolve@ts2.8", + "@types/bson@ts2.8", + "@types/buffer-equal@ts2.8", + "@types/builtin-modules@ts2.8", + "@types/bytes@ts2.8", + "@types/callsites@ts2.8", + "@types/camelcase@ts2.8", + "@types/camelcase-keys@ts2.8", + "@types/caseless@ts2.8", + "@types/change-emitter@ts2.8", + "@types/check-types@ts2.8", + "@types/cheerio@ts2.8", + "@types/chokidar@ts2.8", + "@types/chownr@ts2.8", + "@types/circular-json@ts2.8", + "@types/classnames@ts2.8", + "@types/clean-css@ts2.8", + "@types/clone@ts2.8", + "@types/co-body@ts2.8", + "@types/color@ts2.8", + "@types/color-convert@ts2.8", + "@types/color-name@ts2.8", + "@types/color-string@ts2.8", + "@types/colors@ts2.8", + "@types/combined-stream@ts2.8", + "@types/common-tags@ts2.8", + "@types/component-emitter@ts2.8", + "@types/compressible@ts2.8", + "@types/compression@ts2.8", + "@types/concat-stream@ts2.8", + "@types/connect-history-api-fallback@ts2.8", + "@types/content-disposition@ts2.8", + "@types/content-type@ts2.8", + "@types/convert-source-map@ts2.8", + "@types/cookie@ts2.8", + "@types/cookie-signature@ts2.8", + "@types/cookies@ts2.8", + "@types/core-js@ts2.8", + "@types/cosmiconfig@ts2.8", + "@types/create-react-class@ts2.8", + "@types/cross-spawn@ts2.8", + "@types/cryptiles@ts2.8", + "@types/css-modules-require-hook@ts2.8", + "@types/dargs@ts2.8", + "@types/dateformat@ts2.8", + "@types/debug@ts2.8", + "@types/decamelize@ts2.8", + "@types/decompress@ts2.8", + "@types/decompress-response@ts2.8", + "@types/deep-equal@ts2.8", + "@types/deep-extend@ts2.8", + "@types/deepmerge@ts2.8", + "@types/defined@ts2.8", + "@types/del@ts2.8", + "@types/depd@ts2.8", + "@types/destroy@ts2.8", + "@types/detect-indent@ts2.8", + "@types/detect-newline@ts2.8", + "@types/diff@ts2.8", + "@types/doctrine@ts2.8", + "@types/download@ts2.8", + "@types/draft-js@ts2.8", + "@types/duplexer2@ts2.8", + "@types/duplexer3@ts2.8", + "@types/duplexify@ts2.8", + "@types/ejs@ts2.8", + "@types/end-of-stream@ts2.8", + "@types/entities@ts2.8", + "@types/escape-html@ts2.8", + "@types/escape-string-regexp@ts2.8", + "@types/escodegen@ts2.8", + "@types/eslint-scope@ts2.8", + "@types/eslint-visitor-keys@ts2.8", + "@types/esprima@ts2.8", + "@types/estraverse@ts2.8", + "@types/etag@ts2.8", + "@types/events@ts2.8", + "@types/execa@ts2.8", + "@types/exenv@ts2.8", + "@types/exit@ts2.8", + "@types/exit-hook@ts2.8", + "@types/expect@ts2.8", + "@types/express@ts2.8", + "@types/express-graphql@ts2.8", + "@types/extend@ts2.8", + "@types/extract-zip@ts2.8", + "@types/fancy-log@ts2.8", + "@types/fast-diff@ts2.8", + "@types/fast-levenshtein@ts2.8", + "@types/figures@ts2.8", + "@types/file-type@ts2.8", + "@types/filenamify@ts2.8", + "@types/filesize@ts2.8", + "@types/finalhandler@ts2.8", + "@types/find-root@ts2.8", + "@types/find-up@ts2.8", + "@types/findup-sync@ts2.8", + "@types/forever-agent@ts2.8", + "@types/form-data@ts2.8", + "@types/forwarded@ts2.8", + "@types/fresh@ts2.8", + "@types/from2@ts2.8", + "@types/fs-extra@ts2.8", + "@types/get-caller-file@ts2.8", + "@types/get-stdin@ts2.8", + "@types/get-stream@ts2.8", + "@types/get-value@ts2.8", + "@types/glob-base@ts2.8", + "@types/glob-parent@ts2.8", + "@types/glob-stream@ts2.8", + "@types/globby@ts2.8", + "@types/globule@ts2.8", + "@types/got@ts2.8", + "@types/graceful-fs@ts2.8", + "@types/gulp-rename@ts2.8", + "@types/gulp-sourcemaps@ts2.8", + "@types/gulp-util@ts2.8", + "@types/gzip-size@ts2.8", + "@types/handlebars@ts2.8", + "@types/has-ansi@ts2.8", + "@types/hasha@ts2.8", + "@types/he@ts2.8", + "@types/hoek@ts2.8", + "@types/html-entities@ts2.8", + "@types/html-minifier@ts2.8", + "@types/htmlparser2@ts2.8", + "@types/http-assert@ts2.8", + "@types/http-errors@ts2.8", + "@types/http-proxy@ts2.8", + "@types/http-proxy-middleware@ts2.8", + "@types/indent-string@ts2.8", + "@types/inflected@ts2.8", + "@types/inherits@ts2.8", + "@types/ini@ts2.8", + "@types/inline-style-prefixer@ts2.8", + "@types/inquirer@ts2.8", + "@types/internal-ip@ts2.8", + "@types/into-stream@ts2.8", + "@types/invariant@ts2.8", + "@types/ip@ts2.8", + "@types/ip-regex@ts2.8", + "@types/is-absolute-url@ts2.8", + "@types/is-binary-path@ts2.8", + "@types/is-finite@ts2.8", + "@types/is-glob@ts2.8", + "@types/is-my-json-valid@ts2.8", + "@types/is-number@ts2.8", + "@types/is-object@ts2.8", + "@types/is-path-cwd@ts2.8", + "@types/is-path-in-cwd@ts2.8", + "@types/is-promise@ts2.8", + "@types/is-scoped@ts2.8", + "@types/is-stream@ts2.8", + "@types/is-svg@ts2.8", + "@types/is-url@ts2.8", + "@types/is-windows@ts2.8", + "@types/istanbul-lib-coverage@ts2.8", + "@types/istanbul-lib-hook@ts2.8", + "@types/istanbul-lib-instrument@ts2.8", + "@types/istanbul-lib-report@ts2.8", + "@types/istanbul-lib-source-maps@ts2.8", + "@types/istanbul-reports@ts2.8", + "@types/jest-diff@ts2.8", + "@types/jest-docblock@ts2.8", + "@types/jest-get-type@ts2.8", + "@types/jest-matcher-utils@ts2.8", + "@types/jest-validate@ts2.8", + "@types/jpeg-js@ts2.8", + "@types/js-base64@ts2.8", + "@types/js-string-escape@ts2.8", + "@types/js-yaml@ts2.8", + "@types/jsbn@ts2.8", + "@types/jsdom@ts2.8", + "@types/jsesc@ts2.8", + "@types/json-parse-better-errors@ts2.8", + "@types/json-schema@ts2.8", + "@types/json-stable-stringify@ts2.8", + "@types/json-stringify-safe@ts2.8", + "@types/json5@ts2.8", + "@types/jsonfile@ts2.8", + "@types/jsontoxml@ts2.8", + "@types/jss@ts2.8", + "@types/keygrip@ts2.8", + "@types/keymirror@ts2.8", + "@types/keyv@ts2.8", + "@types/klaw@ts2.8", + "@types/koa-send@ts2.8", + "@types/leven@ts2.8", + "@types/listr@ts2.8", + "@types/load-json-file@ts2.8", + "@types/loader-runner@ts2.8", + "@types/loader-utils@ts2.8", + "@types/locate-path@ts2.8", + "@types/lodash-es@ts2.8", + "@types/lodash.assign@ts2.8", + "@types/lodash.camelcase@ts2.8", + "@types/lodash.clonedeep@ts2.8", + "@types/lodash.debounce@ts2.8", + "@types/lodash.escape@ts2.8", + "@types/lodash.flowright@ts2.8", + "@types/lodash.get@ts2.8", + "@types/lodash.isarguments@ts2.8", + "@types/lodash.isarray@ts2.8", + "@types/lodash.isequal@ts2.8", + "@types/lodash.isobject@ts2.8", + "@types/lodash.isstring@ts2.8", + "@types/lodash.keys@ts2.8", + "@types/lodash.memoize@ts2.8", + "@types/lodash.merge@ts2.8", + "@types/lodash.mergewith@ts2.8", + "@types/lodash.pick@ts2.8", + "@types/lodash.sortby@ts2.8", + "@types/lodash.tail@ts2.8", + "@types/lodash.template@ts2.8", + "@types/lodash.throttle@ts2.8", + "@types/lodash.unescape@ts2.8", + "@types/lodash.uniq@ts2.8", + "@types/log-symbols@ts2.8", + "@types/log-update@ts2.8", + "@types/loglevel@ts2.8", + "@types/loud-rejection@ts2.8", + "@types/lru-cache@ts2.8", + "@types/make-dir@ts2.8", + "@types/map-obj@ts2.8", + "@types/media-typer@ts2.8", + "@types/mem@ts2.8", + "@types/mem-fs@ts2.8", + "@types/memory-fs@ts2.8", + "@types/meow@ts2.8", + "@types/merge-descriptors@ts2.8", + "@types/merge-stream@ts2.8", + "@types/methods@ts2.8", + "@types/micromatch@ts2.8", + "@types/mime@ts2.8", + "@types/mime-db@ts2.8", + "@types/mime-types@ts2.8", + "@types/minimatch@ts2.8", + "@types/minimist@ts2.8", + "@types/minipass@ts2.8", + "@types/mkdirp@ts2.8", + "@types/mongodb@ts2.8", + "@types/morgan@ts2.8", + "@types/move-concurrently@ts2.8", + "@types/ms@ts2.8", + "@types/msgpack-lite@ts2.8", + "@types/multimatch@ts2.8", + "@types/mz@ts2.8", + "@types/negotiator@ts2.8", + "@types/node-dir@ts2.8", + "@types/node-fetch@ts2.8", + "@types/node-forge@ts2.8", + "@types/node-int64@ts2.8", + "@types/node-ipc@ts2.8", + "@types/node-notifier@ts2.8", + "@types/nomnom@ts2.8", + "@types/nopt@ts2.8", + "@types/normalize-package-data@ts2.8", + "@types/normalize-url@ts2.8", + "@types/number-is-nan@ts2.8", + "@types/object-assign@ts2.8", + "@types/on-finished@ts2.8", + "@types/on-headers@ts2.8", + "@types/once@ts2.8", + "@types/onetime@ts2.8", + "@types/opener@ts2.8", + "@types/opn@ts2.8", + "@types/optimist@ts2.8", + "@types/ora@ts2.8", + "@types/os-homedir@ts2.8", + "@types/os-locale@ts2.8", + "@types/os-tmpdir@ts2.8", + "@types/p-cancelable@ts2.8", + "@types/p-each-series@ts2.8", + "@types/p-event@ts2.8", + "@types/p-lazy@ts2.8", + "@types/p-limit@ts2.8", + "@types/p-locate@ts2.8", + "@types/p-map@ts2.8", + "@types/p-map-series@ts2.8", + "@types/p-reduce@ts2.8", + "@types/p-timeout@ts2.8", + "@types/p-try@ts2.8", + "@types/pako@ts2.8", + "@types/parse-glob@ts2.8", + "@types/parse-json@ts2.8", + "@types/parseurl@ts2.8", + "@types/path-exists@ts2.8", + "@types/path-is-absolute@ts2.8", + "@types/path-parse@ts2.8", + "@types/pg-pool@ts2.8", + "@types/pg-types@ts2.8", + "@types/pify@ts2.8", + "@types/pixelmatch@ts2.8", + "@types/pkg-dir@ts2.8", + "@types/pluralize@ts2.8", + "@types/pngjs@ts2.8", + "@types/prelude-ls@ts2.8", + "@types/pretty-bytes@ts2.8", + "@types/pretty-format@ts2.8", + "@types/progress@ts2.8", + "@types/promise-retry@ts2.8", + "@types/proxy-addr@ts2.8", + "@types/pump@ts2.8", + "@types/q@ts2.8", + "@types/qs@ts2.8", + "@types/range-parser@ts2.8", + "@types/rc@ts2.8", + "@types/rc-select@ts2.8", + "@types/rc-slider@ts2.8", + "@types/rc-tooltip@ts2.8", + "@types/rc-tree@ts2.8", + "@types/react-event-listener@ts2.8", + "@types/react-side-effect@ts2.8", + "@types/react-slick@ts2.8", + "@types/read-chunk@ts2.8", + "@types/read-pkg@ts2.8", + "@types/read-pkg-up@ts2.8", + "@types/recompose@ts2.8", + "@types/recursive-readdir@ts2.8", + "@types/relateurl@ts2.8", + "@types/replace-ext@ts2.8", + "@types/request@ts2.8", + "@types/request-promise-native@ts2.8", + "@types/require-directory@ts2.8", + "@types/require-from-string@ts2.8", + "@types/require-relative@ts2.8", + "@types/resolve@ts2.8", + "@types/resolve-from@ts2.8", + "@types/retry@ts2.8", + "@types/rx@ts2.8", + "@types/rx-lite@ts2.8", + "@types/rx-lite-aggregates@ts2.8", + "@types/safe-regex@ts2.8", + "@types/sane@ts2.8", + "@types/sass-graph@ts2.8", + "@types/sax@ts2.8", + "@types/scriptjs@ts2.8", + "@types/semver@ts2.8", + "@types/send@ts2.8", + "@types/serialize-javascript@ts2.8", + "@types/serve-index@ts2.8", + "@types/serve-static@ts2.8", + "@types/set-value@ts2.8", + "@types/shallowequal@ts2.8", + "@types/shelljs@ts2.8", + "@types/sockjs@ts2.8", + "@types/sockjs-client@ts2.8", + "@types/source-list-map@ts2.8", + "@types/source-map-support@ts2.8", + "@types/spdx-correct@ts2.8", + "@types/spdy@ts2.8", + "@types/split@ts2.8", + "@types/sprintf@ts2.8", + "@types/sprintf-js@ts2.8", + "@types/sqlstring@ts2.8", + "@types/sshpk@ts2.8", + "@types/stack-utils@ts2.8", + "@types/stat-mode@ts2.8", + "@types/statuses@ts2.8", + "@types/strict-uri-encode@ts2.8", + "@types/string-template@ts2.8", + "@types/strip-ansi@ts2.8", + "@types/strip-bom@ts2.8", + "@types/strip-json-comments@ts2.8", + "@types/supports-color@ts2.8", + "@types/svg2png@ts2.8", + "@types/svgo@ts2.8", + "@types/table@ts2.8", + "@types/tapable@ts2.8", + "@types/tar@ts2.8", + "@types/temp@ts2.8", + "@types/tempfile@ts2.8", + "@types/through@ts2.8", + "@types/through2@ts2.8", + "@types/tinycolor2@ts2.8", + "@types/tmp@ts2.8", + "@types/to-absolute-glob@ts2.8", + "@types/tough-cookie@ts2.8", + "@types/trim@ts2.8", + "@types/tryer@ts2.8", + "@types/type-check@ts2.8", + "@types/type-is@ts2.8", + "@types/ua-parser-js@ts2.8", + "@types/uglify-js@ts2.8", + "@types/uglifyjs-webpack-plugin@ts2.8", + "@types/underscore@ts2.8", + "@types/uniq@ts2.8", + "@types/uniqid@ts2.8", + "@types/untildify@ts2.8", + "@types/urijs@ts2.8", + "@types/url-join@ts2.8", + "@types/url-parse@ts2.8", + "@types/url-regex@ts2.8", + "@types/user-home@ts2.8", + "@types/util-deprecate@ts2.8", + "@types/util.promisify@ts2.8", + "@types/utils-merge@ts2.8", + "@types/uuid@ts2.8", + "@types/vali-date@ts2.8", + "@types/vary@ts2.8", + "@types/verror@ts2.8", + "@types/vinyl@ts2.8", + "@types/vinyl-fs@ts2.8", + "@types/warning@ts2.8", + "@types/watch@ts2.8", + "@types/watchpack@ts2.8", + "@types/webpack-dev-middleware@ts2.8", + "@types/webpack-sources@ts2.8", + "@types/which@ts2.8", + "@types/window-size@ts2.8", + "@types/wrap-ansi@ts2.8", + "@types/write-file-atomic@ts2.8", + "@types/ws@ts2.8", + "@types/xml2js@ts2.8", + "@types/xmlbuilder@ts2.8", + "@types/xtend@ts2.8", + "@types/yallist@ts2.8", + "@types/yargs@ts2.8", + "@types/yauzl@ts2.8", + "@types/yeoman-generator@ts2.8", + "@types/zen-observable@ts2.8", + "@types/react-content-loader@ts2.8", + } + t.Run("works when the command is too long to install all packages at once", func(t *testing.T) { + t.Parallel() + var calledCount atomic.Int32 + hasError := project.InstallNpmPackages(packageNames, func(packages []string, hasError *atomic.Bool) { + calledCount.Add(1) + }) + assert.Equal(t, hasError, false) + assert.Equal(t, int(calledCount.Load()), 2) + }) + + t.Run("installs remaining packages when one of the partial command fails", func(t *testing.T) { + t.Parallel() + var calledCount atomic.Int32 + hasError := project.InstallNpmPackages(packageNames, func(packages []string, hasError *atomic.Bool) { + calledCount.Add(1) + hasError.Store(true) + }) + assert.Equal(t, hasError, true) + assert.Equal(t, int(calledCount.Load()), 2) + }) +} diff --git a/internal/project/logger.go b/internal/project/logger.go index cc2db1c8bd..4298b4c3c4 100644 --- a/internal/project/logger.go +++ b/internal/project/logger.go @@ -6,6 +6,7 @@ import ( "io" "os" "strings" + "sync" "time" ) @@ -19,10 +20,10 @@ const ( ) type Logger struct { + mu sync.Mutex outputs []*bufio.Writer fileHandle *os.File level LogLevel - inGroup bool seq int } @@ -37,6 +38,8 @@ func NewLogger(outputs []io.Writer, file string, level LogLevel) *Logger { } func (l *Logger) SetFile(file string) { + l.mu.Lock() + defer l.mu.Unlock() if l.fileHandle != nil { oldWriter := l.outputs[len(l.outputs)-1] l.outputs = l.outputs[:len(l.outputs)-1] @@ -65,20 +68,6 @@ func (l *Logger) Error(s string) { l.msg(s, "Err") } -func (l *Logger) StartGroup() { - if l == nil { - return - } - l.inGroup = true -} - -func (l *Logger) EndGroup() { - if l == nil { - return - } - l.inGroup = false -} - func (l *Logger) LoggingEnabled() bool { return l != nil && len(l.outputs) > 0 } @@ -91,6 +80,8 @@ func (l *Logger) Close() { if l == nil { return } + l.mu.Lock() + defer l.mu.Unlock() for _, output := range l.outputs { _ = output.Flush() } @@ -103,6 +94,8 @@ func (l *Logger) msg(s string, messageType string) { if l == nil { return } + l.mu.Lock() + defer l.mu.Unlock() for _, output := range l.outputs { header := fmt.Sprintf("%s %d", messageType, l.seq) output.WriteString(header) //nolint: errcheck @@ -114,7 +107,5 @@ func (l *Logger) msg(s string, messageType string) { output.WriteRune('\n') //nolint: errcheck output.Flush() } - if !l.inGroup { - l.seq++ - } + l.seq++ } diff --git a/internal/project/project.go b/internal/project/project.go index 75be8ee20f..0b9827a0c7 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -6,6 +6,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" "github.com/microsoft/typescript-go/internal/ast" @@ -73,6 +74,7 @@ type ProjectHost interface { tsoptions.ParseConfigHost NewLine() string DefaultLibraryPath() string + TypingsInstaller() *TypingsInstaller DocumentRegistry() *DocumentRegistry GetScriptInfoByPath(path tspath.Path) *ScriptInfo GetOrCreateScriptInfoForFile(fileName string, path tspath.Path, scriptKind core.ScriptKind) *ScriptInfo @@ -84,6 +86,37 @@ type ProjectHost interface { Client() Client } +type TypingsInfo struct { + TypeAcquisition *core.TypeAcquisition + CompilerOptions *core.CompilerOptions + UnresolvedImports []string +} + +func setIsEqualTo(arr1 []string, arr2 []string) bool { + if len(arr1) == 0 { + return len(arr2) == 0 + } + if len(arr2) == 0 { + return len(arr1) == 0 + } + if slices.Equal(arr1, arr2) { + return true + } + compact1 := slices.Compact(arr1) + compact2 := slices.Compact(arr2) + slices.Sort(compact1) + slices.Sort(compact2) + return slices.Equal(compact1, compact2) +} + +func typeAcquisitionChanged(opt1 *core.TypeAcquisition, opt2 *core.TypeAcquisition) bool { + return opt1 != opt2 && + (opt1.Enable.IsTrue() != opt2.Enable.IsTrue() || + !setIsEqualTo(opt1.Include, opt2.Include) || + !setIsEqualTo(opt1.Exclude, opt2.Exclude) || + opt1.DisableFilenameBasedTypeAcquisition.IsTrue() != opt2.DisableFilenameBasedTypeAcquisition.IsTrue()) +} + var _ compiler.CompilerHost = (*Project)(nil) type Project struct { @@ -92,13 +125,14 @@ type Project struct { name string kind Kind - mu sync.Mutex - initialLoadPending bool - dirty bool - version int - deferredClose bool - pendingReload PendingReload - dirtyFilePath tspath.Path + mu sync.Mutex + initialLoadPending bool + dirty bool + version int + deferredClose bool + pendingReload PendingReload + dirtyFilePath tspath.Path + hasAddedorRemovedFiles atomic.Bool comparePathsOptions tspath.ComparePathsOptions currentDirectory string @@ -111,14 +145,24 @@ type Project struct { // But the ProjectService owns script infos, so it's not clear why there was an extra pointer. rootFileNames *collections.OrderedMap[tspath.Path, string] compilerOptions *core.CompilerOptions + typeAcquisition *core.TypeAcquisition parsedCommandLine *tsoptions.ParsedCommandLine program *compiler.Program checkerPool *checkerPool + typingsCacheMu sync.Mutex + unresolvedImportsPerFile map[*ast.SourceFile][]string + unresolvedImports []string + typingsInfo *TypingsInfo + typingFiles []string + // Watchers rootFilesWatch *watchedFiles[[]string] failedLookupsWatch *watchedFiles[map[tspath.Path]string] affectingLocationsWatch *watchedFiles[map[tspath.Path]string] + typingsFilesWatch *watchedFiles[map[tspath.Path]string] + typingsDirectoryWatch *watchedFiles[map[tspath.Path]string] + typingsWatchInvoked atomic.Bool } func NewConfiguredProject(configFileName string, configFilePath tspath.Path, host ProjectHost) *Project { @@ -128,7 +172,7 @@ func NewConfiguredProject(configFileName string, configFilePath tspath.Path, hos project.initialLoadPending = true client := host.Client() if host.IsWatchEnabled() && client != nil { - project.rootFilesWatch = newWatchedFiles(client, lsproto.WatchKindChange|lsproto.WatchKindCreate|lsproto.WatchKindDelete, core.Identity) + project.rootFilesWatch = newWatchedFiles(project, lsproto.WatchKindChange|lsproto.WatchKindCreate|lsproto.WatchKindDelete, core.Identity, "root files") } return project } @@ -155,9 +199,11 @@ func NewProject(name string, kind Kind, currentDirectory string, host ProjectHos } client := host.Client() if host.IsWatchEnabled() && client != nil { - globMapper := createGlobMapper(host) - project.failedLookupsWatch = newWatchedFiles(client, lsproto.WatchKindCreate, globMapper) - project.affectingLocationsWatch = newWatchedFiles(client, lsproto.WatchKindChange|lsproto.WatchKindCreate|lsproto.WatchKindDelete, globMapper) + globMapper := createResolutionLookupGlobMapper(host) + project.failedLookupsWatch = newWatchedFiles(project, lsproto.WatchKindCreate, globMapper, "failed lookup") + project.affectingLocationsWatch = newWatchedFiles(project, lsproto.WatchKindChange|lsproto.WatchKindCreate|lsproto.WatchKindDelete, globMapper, "affecting location") + project.typingsFilesWatch = newWatchedFiles(project, lsproto.WatchKindChange|lsproto.WatchKindCreate|lsproto.WatchKindDelete, globMapperForTypingsInstaller, "typings installer files") + project.typingsDirectoryWatch = newWatchedFiles(project, lsproto.WatchKindCreate|lsproto.WatchKindDelete, globMapperForTypingsInstaller, "typings installer directories") } project.markAsDirty() return project @@ -179,7 +225,7 @@ func (p *Project) GetCurrentDirectory() string { } func (p *Project) GetRootFileNames() []string { - return slices.Collect(p.rootFileNames.Values()) + return append(slices.Collect(p.rootFileNames.Values()), p.typingFiles...) } func (p *Project) GetCompilerOptions() *core.CompilerOptions { @@ -216,7 +262,7 @@ func (p *Project) NewLine() string { // Trace implements compiler.CompilerHost. func (p *Project) Trace(msg string) { - p.log(msg) + p.Log(msg) } // GetDefaultLibraryPath implements compiler.CompilerHost. @@ -308,24 +354,11 @@ func (p *Project) updateWatchers(ctx context.Context) { failedLookupGlobs, affectingLocationGlobs := p.getModuleResolutionWatchGlobs() if rootFileGlobs != nil { - if updated, err := p.rootFilesWatch.update(ctx, rootFileGlobs); err != nil { - p.log(fmt.Sprintf("Failed to update root file watch: %v", err)) - } else if updated { - p.log("Root file watches updated:\n" + formatFileList(rootFileGlobs, "\t", hr)) - } + p.rootFilesWatch.update(ctx, rootFileGlobs) } - if updated, err := p.failedLookupsWatch.update(ctx, failedLookupGlobs); err != nil { - p.log(fmt.Sprintf("Failed to update failed lookup watch: %v", err)) - } else if updated { - p.log("Failed lookup watches updated:\n" + formatFileList(p.failedLookupsWatch.globs, "\t", hr)) - } - - if updated, err := p.affectingLocationsWatch.update(ctx, affectingLocationGlobs); err != nil { - p.log(fmt.Sprintf("Failed to update affecting location watch: %v", err)) - } else if updated { - p.log("Affecting location watches updated:\n" + formatFileList(p.affectingLocationsWatch.globs, "\t", hr)) - } + p.failedLookupsWatch.update(ctx, failedLookupGlobs) + p.affectingLocationsWatch.update(ctx, affectingLocationGlobs) } // onWatchEventForNilScriptInfo is fired for watch events that are not the @@ -350,6 +383,24 @@ func (p *Project) onWatchEventForNilScriptInfo(fileName string) { } else if _, ok := p.affectingLocationsWatch.data[path]; ok { p.markAsDirty() } + + if !p.typingsWatchInvoked.Load() { + if _, ok := p.typingsFilesWatch.data[path]; ok { + p.typingsWatchInvoked.Store(true) + p.enqueueInstallTypingsForProject(nil, true) + } else if _, ok := p.typingsDirectoryWatch.data[path]; ok { + p.typingsWatchInvoked.Store(true) + p.enqueueInstallTypingsForProject(nil, true) + } else { + for dir := range p.typingsDirectoryWatch.data { + if tspath.ContainsPath(string(dir), string(path), p.comparePathsOptions) { + p.typingsWatchInvoked.Store(true) + p.enqueueInstallTypingsForProject(nil, true) + break + } + } + } + } } func (p *Project) getOrCreateScriptInfoAndAttachToProject(fileName string, scriptKind core.ScriptKind) *ScriptInfo { @@ -392,6 +443,11 @@ func (p *Project) markAsDirtyLocked() { } } +// Always called when p.mu lock was already acquired. +func (p *Project) onFileAddedOrRemoved() { + p.hasAddedorRemovedFiles.Store(true) +} + // updateGraph updates the set of files that contribute to the project. // Returns true if the set of files in has changed. NOTE: this is the // opposite of the return value in Strada, which was frequently inverted, @@ -405,7 +461,7 @@ func (p *Project) updateGraph() bool { } start := time.Now() - p.log("Starting updateGraph: Project: " + p.name) + p.Log("Starting updateGraph: Project: " + p.name) var writeFileNames bool oldProgram := p.program p.initialLoadPending = false @@ -424,12 +480,14 @@ func (p *Project) updateGraph() bool { } oldProgramReused := p.updateProgram() + hasAddedOrRemovedFiles := p.hasAddedorRemovedFiles.Load() + p.hasAddedorRemovedFiles.Store(false) p.dirty = false p.dirtyFilePath = "" if writeFileNames { - p.log(p.print(true /*writeFileNames*/, true /*writeFileExplanation*/, false /*writeFileVersionAndText*/)) + p.Log(p.print(true /*writeFileNames*/, true /*writeFileExplanation*/, false /*writeFileVersionAndText*/, &strings.Builder{})) } else if p.program != oldProgram { - p.log("Different program with same set of root files") + p.Log("Different program with same set of root files") } if !oldProgramReused { if oldProgram != nil { @@ -439,28 +497,34 @@ func (p *Project) updateGraph() bool { } } } + p.enqueueInstallTypingsForProject(oldProgram, hasAddedOrRemovedFiles) // TODO: this is currently always synchronously called by some kind of updating request, // but in Strada we throttle, so at least sometimes this should be considered top-level? p.updateWatchers(context.TODO()) } - p.log(fmt.Sprintf("Finishing updateGraph: Project: %s version: %d in %s", p.name, p.version, time.Since(start))) + p.Logf("Finishing updateGraph: Project: %s version: %d in %s", p.name, p.version, time.Since(start)) return true } func (p *Project) updateProgram() bool { if p.checkerPool != nil { - p.logf("Program %d used %d checker(s)", p.version, p.checkerPool.size()) + p.Logf("Program %d used %d checker(s)", p.version, p.checkerPool.size()) } var oldProgramReused bool if p.program == nil || p.dirtyFilePath == "" { rootFileNames := p.GetRootFileNames() compilerOptions := p.compilerOptions + var typingsLocation string + if typeAcquisition := p.getTypeAcquisition(); typeAcquisition != nil && typeAcquisition.Enable.IsTrue() { + typingsLocation = p.host.TypingsInstaller().TypingsLocation + } p.program = compiler.NewProgram(compiler.ProgramOptions{ - RootFiles: rootFileNames, - Host: p, - Options: compilerOptions, + RootFiles: rootFileNames, + Host: p, + Options: compilerOptions, + TypingsLocation: typingsLocation, CreateCheckerPool: func(program *compiler.Program) compiler.CheckerPool { - p.checkerPool = newCheckerPool(4, program, p.log) + p.checkerPool = newCheckerPool(4, program, p.Log) return p.checkerPool }, }) @@ -473,6 +537,227 @@ func (p *Project) updateProgram() bool { return oldProgramReused } +func (p *Project) allRootFilesAreJsOrDts() bool { + for _, fileName := range p.rootFileNames.Entries() { + switch p.getScriptKind(fileName) { + case core.ScriptKindTS: + if tspath.IsDeclarationFileName(fileName) { + break + } + fallthrough + case core.ScriptKindTSX: + return false + } + } + return true +} + +func (p *Project) getTypeAcquisition() *core.TypeAcquisition { + // !!! sheetal Remove local @types from include list which was done in Strada + if p.kind == KindInferred && p.typeAcquisition == nil { + var enable core.Tristate + if p.allRootFilesAreJsOrDts() { + enable = core.TSTrue + } + p.typeAcquisition = &core.TypeAcquisition{ + Enable: enable, + } + } + return p.typeAcquisition +} + +func (p *Project) enqueueInstallTypingsForProject(oldProgram *compiler.Program, forceRefresh bool) { + typingsInstaller := p.host.TypingsInstaller() + if typingsInstaller == nil { + return + } + + typeAcquisition := p.getTypeAcquisition() + if typeAcquisition == nil || !typeAcquisition.Enable.IsTrue() { + // !!! sheetal Should be probably done where we set typeAcquisition + p.unresolvedImports = nil + p.unresolvedImportsPerFile = nil + p.typingFiles = nil + return + } + + p.typingsCacheMu.Lock() + unresolvedImports := p.extractUnresolvedImports(oldProgram) + if forceRefresh || + p.typingsInfo == nil || + p.typingsInfo.CompilerOptions.GetAllowJS() != p.compilerOptions.GetAllowJS() || + typeAcquisitionChanged(typeAcquisition, p.typingsInfo.TypeAcquisition) || + !slices.Equal(p.typingsInfo.UnresolvedImports, unresolvedImports) { + // Note: entry is now poisoned since it does not really contain typings for a given combination of compiler options\typings options. + // instead it acts as a placeholder to prevent issuing multiple requests + typingsInfo := &TypingsInfo{ + TypeAcquisition: typeAcquisition, + CompilerOptions: p.compilerOptions, + UnresolvedImports: unresolvedImports, + } + p.typingsInfo = typingsInfo + p.typingsCacheMu.Unlock() + // something has been changed, issue a request to update typings + typingsInstaller.EnqueueInstallTypingsRequest(p, typingsInfo) + } else { + p.typingsCacheMu.Unlock() + } +} + +func (p *Project) extractUnresolvedImports(oldProgram *compiler.Program) []string { + // We dont want to this unless imports/resolutions have changed for any of the file - for later + + // tracing?.push(tracing.Phase.Session, "getUnresolvedImports", { count: sourceFiles.length }); + hasChanges := false + sourceFiles := p.program.GetSourceFiles() + sourceFilesSet := core.NewSetWithSizeHint[*ast.SourceFile](len(sourceFiles)) + + // !!! sheetal remove ambient module names from unresolved imports + // const ambientModules = program.getTypeChecker().getAmbientModules().map(mod => stripQuotes(mod.getName())); + for _, sourceFile := range sourceFiles { + if p.extractUnresolvedImportsFromSourceFile(sourceFile, oldProgram) { + hasChanges = true + } + sourceFilesSet.Add(sourceFile) + } + + if hasChanges || len(p.unresolvedImportsPerFile) != sourceFilesSet.Len() { + unResolvedImports := []string{} + for sourceFile, unResolvedInFile := range p.unresolvedImportsPerFile { + if sourceFilesSet.Has(sourceFile) { + unResolvedImports = append(unResolvedImports, unResolvedInFile...) + } else { + delete(p.unresolvedImportsPerFile, sourceFile) + } + } + + slices.Sort(unResolvedImports) + p.unresolvedImports = slices.Compact(unResolvedImports) + } + // tracing?.pop(); + return p.unresolvedImports +} + +func (p *Project) extractUnresolvedImportsFromSourceFile(file *ast.SourceFile, oldProgram *compiler.Program) bool { + _, ok := p.unresolvedImportsPerFile[file] + if ok { + return false + } + + unresolvedImports := []string{} + resolvedModules := p.program.GetResolvedModules()[file.Path()] + for cacheKey, resolution := range resolvedModules { + resolved := resolution.IsResolved() + if (!resolved || !tspath.ExtensionIsOneOf(resolution.Extension, tspath.SupportedTSExtensionsWithJsonFlat)) && + !tspath.IsExternalModuleNameRelative(cacheKey.Name) { + // !ambientModules.some(m => m === name) + unresolvedImports = append(unresolvedImports, cacheKey.Name) + } + } + + hasChanges := true + if oldProgram != nil { + oldFile := oldProgram.GetSourceFileByPath(file.Path()) + if oldFile != nil { + oldUnresolvedImports, ok := p.unresolvedImportsPerFile[oldFile] + if ok { + delete(p.unresolvedImportsPerFile, oldFile) + if slices.Equal(oldUnresolvedImports, unresolvedImports) { + unresolvedImports = oldUnresolvedImports + } else { + hasChanges = true + } + + } + } + } + if p.unresolvedImportsPerFile == nil { + p.unresolvedImportsPerFile = make(map[*ast.SourceFile][]string, len(p.program.GetSourceFiles())) + } + p.unresolvedImportsPerFile[file] = unresolvedImports + return hasChanges +} + +func (p *Project) UpdateTypingFiles(typingsInfo *TypingsInfo, typingFiles []string) { + p.mu.Lock() + defer p.mu.Unlock() + if p.typingsInfo != typingsInfo { + return + } + + typeAcquisition := p.getTypeAcquisition() + if typeAcquisition == nil || !typeAcquisition.Enable.IsTrue() { + typingFiles = nil + } else { + slices.Sort(typingFiles) + } + if !slices.Equal(typingFiles, p.typingFiles) { + // If typing files changed, then only schedule project update + p.typingFiles = typingFiles + + // // Invalidate files with unresolved imports + // this.resolutionCache.setFilesWithInvalidatedNonRelativeUnresolvedImports(this.cachedUnresolvedImportsPerFile); + + p.markAsDirtyLocked() + client := p.host.Client() + if client != nil { + err := client.RefreshDiagnostics(context.Background()) + if err != nil { + p.Logf("Error when refreshing diagnostics from updateTypingFiles %v", err) + } + } + } +} + +func (p *Project) WatchTypingLocations(files []string) { + client := p.host.Client() + if !p.host.IsWatchEnabled() || client == nil { + return + } + + p.typingsWatchInvoked.Store(false) + var typingsInstallerFileGlobs map[tspath.Path]string + var typingsInstallerDirectoryGlobs map[tspath.Path]string + // Create watches from list of files + for _, file := range files { + basename := tspath.GetBaseFileName(file) + if basename == "package.json" || basename == "bower.json" { + // package.json or bower.json exists, watch the file to detect changes and update typings + if typingsInstallerFileGlobs == nil { + typingsInstallerFileGlobs = map[tspath.Path]string{} + } + typingsInstallerFileGlobs[p.toPath(file)] = file + } else { + var globLocation string + // path in projectRoot, watch project root + if tspath.ContainsPath(p.currentDirectory, file, p.comparePathsOptions) { + currentDirectoryLen := len(p.currentDirectory) + 1 + subDirectory := strings.IndexRune(file[currentDirectoryLen:], tspath.DirectorySeparator) + if subDirectory != -1 { + // Watch subDirectory + globLocation = file[0 : currentDirectoryLen+subDirectory] + } else { + // Watch the directory itself + globLocation = file + } + } else { + // path in global cache, watch global cache + // else watch node_modules or bower_components + typingsLocation := p.host.TypingsInstaller().TypingsLocation + globLocation = core.IfElse(tspath.ContainsPath(typingsLocation, file, p.comparePathsOptions), typingsLocation, file) + } + // package.json or bower.json exists, watch the file to detect changes and update typings + if typingsInstallerDirectoryGlobs == nil { + typingsInstallerDirectoryGlobs = map[tspath.Path]string{} + } + typingsInstallerDirectoryGlobs[p.toPath(globLocation)] = fmt.Sprintf("%s/%s", globLocation, recursiveFileGlobPattern) + } + } + ctx := context.Background() + p.typingsFilesWatch.update(ctx, typingsInstallerFileGlobs) + p.typingsDirectoryWatch.update(ctx, typingsInstallerDirectoryGlobs) +} + func (p *Project) isOrphan() bool { switch p.kind { case KindInferred: @@ -504,10 +789,12 @@ func (p *Project) removeFile(info *ScriptInfo, fileExists bool, detachFromProjec switch p.kind { case KindInferred: p.rootFileNames.Delete(info.path) + p.typeAcquisition = nil case KindConfigured: p.pendingReload = PendingReloadFileNames } } + p.onFileAddedOrRemoved() // !!! // if (fileExists) { @@ -539,6 +826,9 @@ func (p *Project) addRoot(info *ScriptInfo) { panic("script info is already a root") } p.rootFileNames.Set(info.path, info.fileName) + if p.kind == KindInferred { + p.typeAcquisition = nil + } info.attachToProject(p) } @@ -569,7 +859,7 @@ func (p *Project) loadConfig() error { nil, /*extendedConfigCache*/ ) - p.logf("Config: %s : %s", + p.Logf("Config: %s : %s", p.configFileName, core.Must(core.StringifyJson(map[string]any{ "rootNames": parsedCommandLine.FileNames(), @@ -580,9 +870,11 @@ func (p *Project) loadConfig() error { p.parsedCommandLine = parsedCommandLine p.compilerOptions = parsedCommandLine.CompilerOptions() + p.typeAcquisition = parsedCommandLine.TypeAcquisition() p.setRootFiles(parsedCommandLine.FileNames()) } else { p.compilerOptions = &core.CompilerOptions{} + p.typeAcquisition = nil return fmt.Errorf("could not read file %q", p.configFileName) } return nil @@ -632,8 +924,46 @@ func (p *Project) clearSourceMapperCache() { // !!! } -func (p *Project) print(writeFileNames bool, writeFileExplanation bool, writeFileVersionAndText bool) string { - var builder strings.Builder +func (p *Project) GetFileNames(excludeFilesFromExternalLibraries bool, excludeConfigFiles bool) []string { + if p.program == nil { + return []string{} + } + + // !!! sheetal incomplete code + // if (!this.languageServiceEnabled) { + // // if language service is disabled assume that all files in program are root files + default library + // let rootFiles = this.getRootFiles(); + // if (this.compilerOptions) { + // const defaultLibrary = getDefaultLibFilePath(this.compilerOptions); + // if (defaultLibrary) { + // (rootFiles || (rootFiles = [])).push(asNormalizedPath(defaultLibrary)); + // } + // } + // return rootFiles; + // } + result := []string{} + sourceFiles := p.program.GetSourceFiles() + for _, sourceFile := range sourceFiles { + // if excludeFilesFromExternalLibraries && p.program.IsSourceFileFromExternalLibrary(sourceFile) { + // continue; + // } + result = append(result, sourceFile.FileName()) + } + // if (!excludeConfigFiles) { + // const configFile = p.program.GetCompilerOptions().configFile; + // if (configFile) { + // result = append(result, configFile.fileName); + // if (configFile.extendedSourceFiles) { + // for (const f of configFile.extendedSourceFiles) { + // result.push(asNormalizedPath(f)); + // } + // } + // } + // } + return result +} + +func (p *Project) print(writeFileNames bool, writeFileExplanation bool, writeFileVersionAndText bool, builder *strings.Builder) string { builder.WriteString(fmt.Sprintf("Project '%s' (%s)\n", p.name, p.kind.String())) if p.initialLoadPending { builder.WriteString("\tFiles (0) InitialLoadPending\n") @@ -658,12 +988,12 @@ func (p *Project) print(writeFileNames bool, writeFileExplanation bool, writeFil return builder.String() } -func (p *Project) log(s string) { +func (p *Project) Log(s string) { p.host.Log(s) } -func (p *Project) logf(format string, args ...interface{}) { - p.log(fmt.Sprintf(format, args...)) +func (p *Project) Logf(format string, args ...interface{}) { + p.Log(fmt.Sprintf(format, args...)) } func (p *Project) Close() { diff --git a/internal/project/scriptinfo.go b/internal/project/scriptinfo.go index 0e3691166c..2e6206c3db 100644 --- a/internal/project/scriptinfo.go +++ b/internal/project/scriptinfo.go @@ -122,6 +122,7 @@ func (s *ScriptInfo) attachToProject(project *Project) bool { if project.compilerOptions.PreserveSymlinks != core.TSTrue { s.ensureRealpath(project.FS()) } + project.onFileAddedOrRemoved() return true } return false diff --git a/internal/project/service.go b/internal/project/service.go index 23a18945c7..3ec6f02f94 100644 --- a/internal/project/service.go +++ b/internal/project/service.go @@ -30,6 +30,7 @@ type assignProjectResult struct { } type ServiceOptions struct { + TypingsInstallerOptions Logger *Logger PositionEncoding lsproto.PositionEncodingKind WatchEnabled bool @@ -60,11 +61,14 @@ type Service struct { filenameToScriptInfoVersion map[tspath.Path]int realpathToScriptInfosMu sync.Mutex realpathToScriptInfos map[tspath.Path]map[*ScriptInfo]struct{} + + typingsInstaller *TypingsInstaller } func NewService(host ServiceHost, options ServiceOptions) *Service { options.Logger.Info(fmt.Sprintf("currentDirectory:: %s useCaseSensitiveFileNames:: %t", host.GetCurrentDirectory(), host.FS().UseCaseSensitiveFileNames())) options.Logger.Info("libs Location:: " + host.DefaultLibraryPath()) + options.Logger.Info("globalTypingsCacheLocation:: " + host.TypingsLocation()) service := &Service{ host: host, options: options, @@ -104,6 +108,10 @@ func (s *Service) Log(msg string) { s.options.Logger.Info(msg) } +func (s *Service) HasLevel(level LogLevel) bool { + return s.options.Logger.HasLevel(level) +} + // NewLine implements ProjectHost. func (s *Service) NewLine() string { return s.host.NewLine() @@ -114,6 +122,21 @@ func (s *Service) DefaultLibraryPath() string { return s.host.DefaultLibraryPath() } +// TypingsInstaller implements ProjectHost. +func (s *Service) TypingsInstaller() *TypingsInstaller { + if s.typingsInstaller != nil { + return s.typingsInstaller + } + + if typingsLocation := s.host.TypingsLocation(); typingsLocation != "" { + s.typingsInstaller = &TypingsInstaller{ + TypingsLocation: typingsLocation, + options: &s.options.TypingsInstallerOptions, + } + } + return s.typingsInstaller +} + // DocumentRegistry implements ProjectHost. func (s *Service) DocumentRegistry() *DocumentRegistry { return s.documentRegistry @@ -283,6 +306,9 @@ func (s *Service) OnWatchedFilesChanged(ctx context.Context, changes []*lsproto. for _, project := range s.configuredProjects { project.onWatchEventForNilScriptInfo(fileName) } + for _, project := range s.inferredProjects { + project.onWatchEventForNilScriptInfo(fileName) + } } } @@ -309,7 +335,6 @@ func (s *Service) onConfigFileChanged(project *Project, changeKind lsproto.FileC project.pendingReload = PendingReloadFull project.markAsDirty() } - project.updateGraph() return nil } @@ -407,7 +432,7 @@ func (s *Service) updateProjectGraphs(projects []*Project, clearSourceMapperCach if clearSourceMapperCache { project.clearSourceMapperCache() } - project.updateGraph() + project.markAsDirty() } } @@ -494,6 +519,10 @@ func (s *Service) getConfigFileNameForFile(info *ScriptInfo, findFromCacheOnly b if s.configFileExists(tsconfigPath) { return tsconfigPath, true } + jsconfigPath := tspath.CombinePaths(directory, "jsconfig.json") + if s.configFileExists(jsconfigPath) { + return jsconfigPath, true + } if strings.HasSuffix(directory, "/node_modules") { return "", true } @@ -738,21 +767,21 @@ func (s *Service) printProjects() { return } - s.options.Logger.StartGroup() + var builder strings.Builder for _, project := range s.configuredProjects { - s.Log(project.print(false /*writeFileNames*/, false /*writeFileExpanation*/, false /*writeFileVersionAndText*/)) + project.print(false /*writeFileNames*/, false /*writeFileExpanation*/, false /*writeFileVersionAndText*/, &builder) } for _, project := range s.inferredProjects { - s.Log(project.print(false /*writeFileNames*/, false /*writeFileExpanation*/, false /*writeFileVersionAndText*/)) + project.print(false /*writeFileNames*/, false /*writeFileExpanation*/, false /*writeFileVersionAndText*/, &builder) } - s.Log("Open files: ") + builder.WriteString("Open files: ") for path, projectRootPath := range s.openFiles { info := s.GetScriptInfoByPath(path) - s.logf("\tFileName: %s ProjectRootPath: %s", info.fileName, projectRootPath) - s.Log("\t\tProjects: " + strings.Join(core.Map(info.containingProjects, func(project *Project) string { return project.name }), ", ")) + builder.WriteString(fmt.Sprintf("\tFileName: %s ProjectRootPath: %s", info.fileName, projectRootPath)) + builder.WriteString("\t\tProjects: " + strings.Join(core.Map(info.containingProjects, func(project *Project) string { return project.name }), ", ")) } - s.options.Logger.EndGroup() + s.Log(builder.String()) } func (s *Service) logf(format string, args ...any) { diff --git a/internal/project/service_test.go b/internal/project/service_test.go index c4533383cc..0364598d65 100644 --- a/internal/project/service_test.go +++ b/internal/project/service_test.go @@ -37,7 +37,7 @@ func TestService(t *testing.T) { t.Parallel() t.Run("create configured project", func(t *testing.T) { t.Parallel() - service, _ := projecttestutil.Setup(defaultFiles) + service, _ := projecttestutil.Setup(defaultFiles, nil) assert.Equal(t, len(service.Projects()), 0) service.OpenFile("/home/projects/TS/p1/src/index.ts", defaultFiles["/home/projects/TS/p1/src/index.ts"].(string), core.ScriptKindTS, "") assert.Equal(t, len(service.Projects()), 1) @@ -50,7 +50,7 @@ func TestService(t *testing.T) { t.Run("create inferred project", func(t *testing.T) { t.Parallel() - service, _ := projecttestutil.Setup(defaultFiles) + service, _ := projecttestutil.Setup(defaultFiles, nil) service.OpenFile("/home/projects/TS/p1/config.ts", defaultFiles["/home/projects/TS/p1/config.ts"].(string), core.ScriptKindTS, "") // Find tsconfig, load, notice config.ts is not included, create inferred project assert.Equal(t, len(service.Projects()), 2) @@ -60,7 +60,7 @@ func TestService(t *testing.T) { t.Run("inferred project for in-memory files", func(t *testing.T) { t.Parallel() - service, _ := projecttestutil.Setup(defaultFiles) + service, _ := projecttestutil.Setup(defaultFiles, nil) service.OpenFile("/home/projects/TS/p1/config.ts", defaultFiles["/home/projects/TS/p1/config.ts"].(string), core.ScriptKindTS, "") service.OpenFile("^/untitled/ts-nul-authority/Untitled-1", "x", core.ScriptKindTS, "") service.OpenFile("^/untitled/ts-nul-authority/Untitled-2", "y", core.ScriptKindTS, "") @@ -77,7 +77,7 @@ func TestService(t *testing.T) { jsFiles := map[string]any{ "/home/projects/TS/p1/index.js": `import { x } from "./x";`, } - service, _ := projecttestutil.Setup(jsFiles) + service, _ := projecttestutil.Setup(jsFiles, nil) service.OpenFile("/home/projects/TS/p1/index.js", jsFiles["/home/projects/TS/p1/index.js"].(string), core.ScriptKindJS, "") assert.Equal(t, len(service.Projects()), 1) project := service.Projects()[0] @@ -89,7 +89,7 @@ func TestService(t *testing.T) { t.Parallel() t.Run("update script info eagerly and program lazily", func(t *testing.T) { t.Parallel() - service, _ := projecttestutil.Setup(defaultFiles) + service, _ := projecttestutil.Setup(defaultFiles, nil) service.OpenFile("/home/projects/TS/p1/src/x.ts", defaultFiles["/home/projects/TS/p1/src/x.ts"].(string), core.ScriptKindTS, "") info, proj := service.EnsureDefaultProjectForFile("/home/projects/TS/p1/src/x.ts") programBefore := proj.GetProgram() @@ -127,7 +127,7 @@ func TestService(t *testing.T) { t.Run("unchanged source files are reused", func(t *testing.T) { t.Parallel() - service, _ := projecttestutil.Setup(defaultFiles) + service, _ := projecttestutil.Setup(defaultFiles, nil) service.OpenFile("/home/projects/TS/p1/src/x.ts", defaultFiles["/home/projects/TS/p1/src/x.ts"].(string), core.ScriptKindTS, "") _, proj := service.EnsureDefaultProjectForFile("/home/projects/TS/p1/src/x.ts") programBefore := proj.GetProgram() @@ -165,7 +165,7 @@ func TestService(t *testing.T) { t.Parallel() files := maps.Clone(defaultFiles) files["/home/projects/TS/p1/y.ts"] = `export const y = 2;` - service, _ := projecttestutil.Setup(files) + service, _ := projecttestutil.Setup(files, nil) service.OpenFile("/home/projects/TS/p1/src/index.ts", files["/home/projects/TS/p1/src/index.ts"].(string), core.ScriptKindTS, "") assert.Check(t, service.GetScriptInfo("/home/projects/TS/p1/y.ts") == nil) @@ -209,7 +209,7 @@ func TestService(t *testing.T) { }, "include": ["src/index.ts"] }` - service, host := projecttestutil.Setup(files) + service, host := projecttestutil.Setup(files, nil) service.OpenFile("/home/projects/TS/p1/src/index.ts", files["/home/projects/TS/p1/src/index.ts"].(string), core.ScriptKindTS, "") _, project := service.EnsureDefaultProjectForFile("/home/projects/TS/p1/src/index.ts") programBefore := project.GetProgram() @@ -270,7 +270,7 @@ func TestService(t *testing.T) { t.Parallel() t.Run("delete a file, close it, recreate it", func(t *testing.T) { t.Parallel() - service, host := projecttestutil.Setup(defaultFiles) + service, host := projecttestutil.Setup(defaultFiles, nil) service.OpenFile("/home/projects/TS/p1/src/x.ts", defaultFiles["/home/projects/TS/p1/src/x.ts"].(string), core.ScriptKindTS, "") service.OpenFile("/home/projects/TS/p1/src/index.ts", defaultFiles["/home/projects/TS/p1/src/index.ts"].(string), core.ScriptKindTS, "") assert.Equal(t, service.SourceFileCount(), 2) @@ -299,7 +299,7 @@ func TestService(t *testing.T) { t.Parallel() files := maps.Clone(defaultFiles) delete(files, "/home/projects/TS/p1/tsconfig.json") - service, host := projecttestutil.Setup(files) + service, host := projecttestutil.Setup(files, nil) service.OpenFile("/home/projects/TS/p1/src/x.ts", defaultFiles["/home/projects/TS/p1/src/x.ts"].(string), core.ScriptKindTS, "") service.OpenFile("/home/projects/TS/p1/src/index.ts", defaultFiles["/home/projects/TS/p1/src/index.ts"].(string), core.ScriptKindTS, "") @@ -334,7 +334,7 @@ func TestService(t *testing.T) { }, }` files["/home/projects/TS/p2/src/index.ts"] = `import { x } from "../../p1/src/x";` - service, _ := projecttestutil.Setup(files) + service, _ := projecttestutil.Setup(files, nil) service.OpenFile("/home/projects/TS/p1/src/index.ts", files["/home/projects/TS/p1/src/index.ts"].(string), core.ScriptKindTS, "") service.OpenFile("/home/projects/TS/p2/src/index.ts", files["/home/projects/TS/p2/src/index.ts"].(string), core.ScriptKindTS, "") assert.Equal(t, len(service.Projects()), 2) @@ -357,7 +357,7 @@ func TestService(t *testing.T) { } }` files["/home/projects/TS/p2/src/index.ts"] = `import { x } from "../../p1/src/x";` - service, _ := projecttestutil.Setup(files) + service, _ := projecttestutil.Setup(files, nil) service.OpenFile("/home/projects/TS/p1/src/index.ts", files["/home/projects/TS/p1/src/index.ts"].(string), core.ScriptKindTS, "") service.OpenFile("/home/projects/TS/p2/src/index.ts", files["/home/projects/TS/p2/src/index.ts"].(string), core.ScriptKindTS, "") assert.Equal(t, len(service.Projects()), 2) @@ -375,7 +375,7 @@ func TestService(t *testing.T) { t.Run("change open file", func(t *testing.T) { t.Parallel() - service, host := projecttestutil.Setup(defaultFiles) + service, host := projecttestutil.Setup(defaultFiles, nil) service.OpenFile("/home/projects/TS/p1/src/x.ts", defaultFiles["/home/projects/TS/p1/src/x.ts"].(string), core.ScriptKindTS, "") service.OpenFile("/home/projects/TS/p1/src/index.ts", defaultFiles["/home/projects/TS/p1/src/index.ts"].(string), core.ScriptKindTS, "") _, project := service.EnsureDefaultProjectForFile("/home/projects/TS/p1/src/index.ts") @@ -396,7 +396,7 @@ func TestService(t *testing.T) { t.Run("change closed program file", func(t *testing.T) { t.Parallel() - service, host := projecttestutil.Setup(defaultFiles) + service, host := projecttestutil.Setup(defaultFiles, nil) service.OpenFile("/home/projects/TS/p1/src/index.ts", defaultFiles["/home/projects/TS/p1/src/index.ts"].(string), core.ScriptKindTS, "") _, project := service.EnsureDefaultProjectForFile("/home/projects/TS/p1/src/index.ts") programBefore := project.GetProgram() @@ -429,7 +429,7 @@ func TestService(t *testing.T) { let y: number = x;`, } - service, host := projecttestutil.Setup(files) + service, host := projecttestutil.Setup(files, nil) service.OpenFile("/home/projects/TS/p1/src/index.ts", files["/home/projects/TS/p1/src/index.ts"].(string), core.ScriptKindTS, "") _, project := service.EnsureDefaultProjectForFile("/home/projects/TS/p1/src/index.ts") program := project.GetProgram() @@ -466,7 +466,7 @@ func TestService(t *testing.T) { "/home/projects/TS/p1/src/x.ts": `export declare const x: number | undefined;`, "/home/projects/TS/p1/src/index.ts": `import { x } from "./x";`, } - service, host := projecttestutil.Setup(files) + service, host := projecttestutil.Setup(files, nil) service.OpenFile("/home/projects/TS/p1/src/index.ts", files["/home/projects/TS/p1/src/index.ts"].(string), core.ScriptKindTS, "") _, project := service.EnsureDefaultProjectForFile("/home/projects/TS/p1/src/index.ts") program := project.GetProgram() @@ -499,7 +499,7 @@ func TestService(t *testing.T) { "/home/projects/TS/p1/src/index.ts": `let x = 2;`, "/home/projects/TS/p1/src/x.ts": `let y = x;`, } - service, host := projecttestutil.Setup(files) + service, host := projecttestutil.Setup(files, nil) service.OpenFile("/home/projects/TS/p1/src/x.ts", files["/home/projects/TS/p1/src/x.ts"].(string), core.ScriptKindTS, "") _, project := service.EnsureDefaultProjectForFile("/home/projects/TS/p1/src/x.ts") program := project.GetProgram() @@ -530,7 +530,7 @@ func TestService(t *testing.T) { }`, "/home/projects/TS/p1/src/index.ts": `import { y } from "./y";`, } - service, host := projecttestutil.Setup(files) + service, host := projecttestutil.Setup(files, nil) service.OpenFile("/home/projects/TS/p1/src/index.ts", files["/home/projects/TS/p1/src/index.ts"].(string), core.ScriptKindTS, "") _, project := service.EnsureDefaultProjectForFile("/home/projects/TS/p1/src/index.ts") program := project.GetProgram() @@ -588,7 +588,7 @@ func TestService(t *testing.T) { }`, "/home/projects/TS/p1/src/index.ts": `import { z } from "./z";`, } - service, host := projecttestutil.Setup(files) + service, host := projecttestutil.Setup(files, nil) service.OpenFile("/home/projects/TS/p1/src/index.ts", files["/home/projects/TS/p1/src/index.ts"].(string), core.ScriptKindTS, "") _, project := service.EnsureDefaultProjectForFile("/home/projects/TS/p1/src/index.ts") program := project.GetProgram() @@ -629,7 +629,7 @@ func TestService(t *testing.T) { }`, "/home/projects/TS/p1/src/index.ts": `a;`, } - service, host := projecttestutil.Setup(files) + service, host := projecttestutil.Setup(files, nil) service.OpenFile("/home/projects/TS/p1/src/index.ts", files["/home/projects/TS/p1/src/index.ts"].(string), core.ScriptKindTS, "") _, project := service.EnsureDefaultProjectForFile("/home/projects/TS/p1/src/index.ts") program := project.GetProgram() diff --git a/internal/project/ti.go b/internal/project/ti.go new file mode 100644 index 0000000000..a300f0898b --- /dev/null +++ b/internal/project/ti.go @@ -0,0 +1,651 @@ +package project + +import ( + "encoding/json" + "fmt" + "os/exec" + "sync" + "sync/atomic" + + "github.com/microsoft/typescript-go/internal/collections" + "github.com/microsoft/typescript-go/internal/core" + "github.com/microsoft/typescript-go/internal/module" + "github.com/microsoft/typescript-go/internal/semver" + "github.com/microsoft/typescript-go/internal/tspath" +) + +type TypesMapFile struct { + TypesMap map[string]SafeListEntry `json:"typesMap"` + SimpleMap map[string]string `json:"simpleMap,omitzero"` +} + +type SafeListEntry struct { + Match string `json:"match"` + Exclude [][]any `json:"exclude"` + Types []string `json:"types"` +} + +type PendingRequest struct { + requestId int32 + packageNames []string + filteredTypings []string + currentlyCachedTypings []string + p *Project + typingsInfo *TypingsInfo +} + +type NpmInstallOperation func(string, []string) ([]byte, error) + +type TypingsInstallerStatus struct { + RequestId int32 + Project *Project + Status string +} + +type TypingsInstallerOptions struct { + // !!! sheetal strada params to keep or not + // const typingSafeListLocation = ts.server.findArgument(ts.server.Arguments.TypingSafeListLocation); + // const typesMapLocation = ts.server.findArgument(ts.server.Arguments.TypesMapLocation); + // const npmLocation = ts.server.findArgument(ts.server.Arguments.NpmLocation); + // const validateDefaultNpmLocation = ts.server.hasArgument(ts.server.Arguments.ValidateDefaultNpmLocation); + ThrottleLimit int + + // For testing + NpmInstall NpmInstallOperation + InstallStatus chan TypingsInstallerStatus +} + +type TypingsInstaller struct { + TypingsLocation string + options *TypingsInstallerOptions + + initOnce sync.Once + + packageNameToTypingLocation collections.SyncMap[string, *CachedTyping] + missingTypingsSet collections.SyncMap[string, bool] + + typesRegistry map[string]map[string]string + typesMap *TypesMapFile + safeList map[string]string + + installRunCount atomic.Int32 + inFlightRequestCount int + pendingRunRequests []*PendingRequest + pendingRunRequestsMu sync.Mutex +} + +func (ti *TypingsInstaller) IsKnownTypesPackageName(p *Project, name string) bool { + // We want to avoid looking this up in the registry as that is expensive. So first check that it's actually an NPM package. + validationResult, _, _ := ValidatePackageName(name) + if validationResult != NameOk { + return false + } + // Strada did this lazily - is that needed here to not waiting on and returning false on first request + ti.init(p) + _, ok := ti.typesRegistry[name] + return ok +} + +// !!! sheetal currently we use latest instead of core.VersionMajorMinor() +const TsVersionToUse = "latest" + +func (ti *TypingsInstaller) InstallPackage(p *Project, fileName string, packageName string) { + cwd, ok := tspath.ForEachAncestorDirectory(tspath.GetDirectoryPath(fileName), func(directory string) (string, bool) { + if p.FS().FileExists(tspath.CombinePaths(directory, "package.json")) { + return directory, true + } + return "", false + }) + if !ok { + cwd = p.GetCurrentDirectory() + } + if cwd != "" { + go ti.installWorker(p, -1, []string{packageName}, cwd, func( + p *Project, + requestId int32, + packageNames []string, + success bool, + ) { + // !!! sheetal events to send + // const message = success ? + // + // `Package ${packageName} installed.` : + // `There was an error installing ${packageName}.`; + // + // const response: PackageInstalledResponse = { + // kind: ActionPackageInstalled, + // projectName, + // id, + // success, + // message, + // }; + // + + // this.sendResponse(response); + // // The behavior is the same as for setTypings, so send the same event. + // this.event(response, "setTypings"); -- Used same event name - do we need it ? + }) + } else { + // !!! sheetal events to send + // const response: PackageInstalledResponse = { + // kind: ActionPackageInstalled, + // projectName, + // id, + // success: false, + // message: "Could not determine a project root path.", + // }; + // this.sendResponse(response); + // // The behavior is the same as for setTypings, so send the same event. + // this.event(response, "setTypings"); -- Used same event name - do we need it ? + } +} + +func (ti *TypingsInstaller) EnqueueInstallTypingsRequest(p *Project, typingsInfo *TypingsInfo) { + // because we arent using buffers, no need to throttle for requests here + p.Log("TI:: Got install request for: " + p.Name()) + go ti.discoverAndInstallTypings( + p, + typingsInfo, + p.GetFileNames( /*excludeFilesFromExternalLibraries*/ true /*excludeConfigFiles*/, true), + p.GetCurrentDirectory(), + ) //.concat(project.getExcludedFiles()) // !!! sheetal we dont have excluded files in project yet +} + +func (ti *TypingsInstaller) discoverAndInstallTypings(p *Project, typingsInfo *TypingsInfo, fileNames []string, projectRootPath string) { + ti.init((p)) + + ti.initializeSafeList(p) + + cachedTypingPaths, newTypingNames, filesToWatch := DiscoverTypings( + p.FS(), + p.Log, + typingsInfo, + fileNames, + projectRootPath, + ti.safeList, + &ti.packageNameToTypingLocation, + ti.typesRegistry, + ) + + // start watching files + p.WatchTypingLocations(filesToWatch) + + requestId := ti.installRunCount.Add(1) + // install typings + if len(newTypingNames) > 0 { + filteredTypings := ti.filterTypings(p, newTypingNames) + if len(filteredTypings) != 0 { + ti.installTypings(p, typingsInfo, requestId, cachedTypingPaths, filteredTypings) + return + } + p.Log("TI:: All typings are known to be missing or invalid - no need to install more typings") + } else { + p.Log("TI:: No new typings were requested as a result of typings discovery") + } + p.UpdateTypingFiles(typingsInfo, cachedTypingPaths) + // !!! sheetal events to send + // this.event(response, "setTypings"); + + if ti.options.InstallStatus != nil { + ti.options.InstallStatus <- TypingsInstallerStatus{ + RequestId: requestId, + Project: p, + Status: fmt.Sprintf("Skipped %d typings", len(newTypingNames)), + } + } +} + +func (ti *TypingsInstaller) installTypings( + p *Project, + typingsInfo *TypingsInfo, + requestId int32, + currentlyCachedTypings []string, + filteredTypings []string, +) { + // !!! sheetal events to send + // send progress event + // this.sendResponse({ + // kind: EventBeginInstallTypes, + // eventId: requestId, + // typingsInstallerVersion: version, + // projectName: req.projectName, + // } as BeginInstallTypes); + + // const body: protocol.BeginInstallTypesEventBody = { + // eventId: response.eventId, + // packages: response.packagesToInstall, + // }; + // const eventName: protocol.BeginInstallTypesEventName = "beginInstallTypes"; + // this.event(body, eventName); + + scopedTypings := make([]string, len(filteredTypings)) + for i, packageName := range filteredTypings { + scopedTypings[i] = fmt.Sprintf("@types/%s@%s", packageName, TsVersionToUse) // @tscore.VersionMajorMinor) // This is normally @tsVersionMajorMinor but for now lets use latest + } + + request := &PendingRequest{ + requestId: requestId, + packageNames: scopedTypings, + filteredTypings: filteredTypings, + currentlyCachedTypings: currentlyCachedTypings, + p: p, + typingsInfo: typingsInfo, + } + ti.pendingRunRequestsMu.Lock() + if ti.inFlightRequestCount < ti.options.ThrottleLimit { + ti.inFlightRequestCount++ + ti.pendingRunRequestsMu.Unlock() + ti.invokeRoutineToInstallTypings(request) + } else { + ti.pendingRunRequests = append(ti.pendingRunRequests, request) + ti.pendingRunRequestsMu.Unlock() + } +} + +func (ti *TypingsInstaller) invokeRoutineToInstallTypings( + request *PendingRequest, +) { + go ti.installWorker( + request.p, + request.requestId, + request.packageNames, + ti.TypingsLocation, + func( + p *Project, + requestId int32, + packageNames []string, + success bool, + ) { + ti.pendingRunRequestsMu.Lock() + pendingRequestsCount := len(ti.pendingRunRequests) + var nextRequest *PendingRequest + if pendingRequestsCount == 0 { + ti.inFlightRequestCount-- + } else { + nextRequest = ti.pendingRunRequests[0] + if pendingRequestsCount == 1 { + ti.pendingRunRequests = nil + } else { + ti.pendingRunRequests = ti.pendingRunRequests[1:] + } + } + ti.pendingRunRequestsMu.Unlock() + if nextRequest != nil { + ti.invokeRoutineToInstallTypings(nextRequest) + } + + if success { + p.Logf("TI:: Installed typings %v", packageNames) + var installedTypingFiles []string + resolver := module.NewResolver(p, &core.CompilerOptions{ModuleResolution: core.ModuleResolutionKindNodeNext}, "", "") + for _, packageName := range request.filteredTypings { + typingFile := ti.typingToFileName(resolver, packageName) + if typingFile == "" { + ti.missingTypingsSet.Store(packageName, true) + continue + } + + // packageName is guaranteed to exist in typesRegistry by filterTypings + distTags := ti.typesRegistry[packageName] + useVersion, ok := distTags["ts"+core.VersionMajorMinor()] + if !ok { + useVersion = distTags["latest"] + } + newVersion := semver.MustParse(useVersion) + newTyping := &CachedTyping{TypingsLocation: typingFile, Version: newVersion} + ti.packageNameToTypingLocation.Store(packageName, newTyping) + installedTypingFiles = append(installedTypingFiles, typingFile) + } + p.Logf("TI:: Installed typing files %v", installedTypingFiles) + p.UpdateTypingFiles(request.typingsInfo, append(request.currentlyCachedTypings, installedTypingFiles...)) + // DO we really need these events + // this.event(response, "setTypings"); + } else { + p.Logf("TI:: install request failed, marking packages as missing to prevent repeated requests: %v", request.filteredTypings) + for _, typing := range request.filteredTypings { + ti.missingTypingsSet.Store(typing, true) + } + } + + // !!! sheetal events to send + // const response: EndInstallTypes = { + // kind: EventEndInstallTypes, + // eventId: requestId, + // projectName: req.projectName, + // packagesToInstall: scopedTypings, + // installSuccess: ok, + // typingsInstallerVersion: version, + // }; + // this.sendResponse(response); + + // if (this.telemetryEnabled) { + // const body: protocol.TypingsInstalledTelemetryEventBody = { + // telemetryEventName: "typingsInstalled", + // payload: { + // installedPackages: response.packagesToInstall.join(","), + // installSuccess: response.installSuccess, + // typingsInstallerVersion: response.typingsInstallerVersion, + // }, + // }; + // const eventName: protocol.TelemetryEventName = "telemetry"; + // this.event(body, eventName); + // } + + // const body: protocol.EndInstallTypesEventBody = { + // eventId: response.eventId, + // packages: response.packagesToInstall, + // success: response.installSuccess, + // }; + // const eventName: protocol.EndInstallTypesEventName = "endInstallTypes"; + // this.event(body, eventName); + + if ti.options.InstallStatus != nil { + ti.options.InstallStatus <- TypingsInstallerStatus{ + RequestId: requestId, + Project: p, + Status: core.IfElse(success, "Success", "Fail"), + } + } + }, + ) +} + +func (ti *TypingsInstaller) installWorker( + p *Project, + requestId int32, + packageNames []string, + cwd string, + onRequestComplete func( + p *Project, + requestId int32, + packageNames []string, + success bool, + ), +) { + p.Logf("TI:: #%d with cwd: %s arguments: %v", requestId, cwd, packageNames) + hasError := InstallNpmPackages(packageNames, func(packageNames []string, hasError *atomic.Bool) { + var npmArgs []string + npmArgs = append(npmArgs, "install", "--ignore-scripts") + npmArgs = append(npmArgs, packageNames...) + npmArgs = append(npmArgs, "--save-dev", "--user-agent=\"typesInstaller/"+core.Version()+"\"") + output, err := ti.options.NpmInstall(cwd, npmArgs) + if err != nil { + p.Logf("TI:: Output is: %s", output) + hasError.Store(true) + } + }) + p.Logf("TI:: npm install #%d completed", requestId) + onRequestComplete(p, requestId, packageNames, !hasError) +} + +func InstallNpmPackages( + packageNames []string, + installPackages func(packages []string, hasError *atomic.Bool), +) bool { + var hasError atomic.Bool + hasError.Store(false) + + wg := core.NewWorkGroup(false) + currentCommandStart := 0 + currentCommandEnd := 0 + currentCommandSize := 100 + for _, packageName := range packageNames { + currentCommandSize = currentCommandSize + len(packageName) + 1 + if currentCommandSize < 8000 { + currentCommandEnd++ + } else { + packages := packageNames[currentCommandStart:currentCommandEnd] + wg.Queue(func() { + installPackages(packages, &hasError) + }) + currentCommandStart = currentCommandEnd + currentCommandSize = 100 + len(packageName) + 1 + currentCommandEnd++ + } + } + wg.Queue(func() { + installPackages(packageNames[currentCommandStart:currentCommandEnd], &hasError) + }) + wg.RunAndWait() + return hasError.Load() +} + +func (ti *TypingsInstaller) filterTypings( + p *Project, + typingsToInstall []string, +) []string { + var result []string + for _, typing := range typingsToInstall { + typingKey := module.MangleScopedPackageName(typing) + if _, ok := ti.missingTypingsSet.Load(typingKey); ok { + p.Logf("TI:: '%s':: '%s' is in missingTypingsSet - skipping...", typing, typingKey) + continue + } + validationResult, name, isScopeName := ValidatePackageName(typing) + if validationResult != NameOk { + // add typing name to missing set so we won't process it again + ti.missingTypingsSet.Store(typingKey, true) + p.Log("TI:: " + RenderPackageNameValidationFailure(typing, validationResult, name, isScopeName)) + continue + } + typesRegistryEntry, ok := ti.typesRegistry[typingKey] + if !ok { + p.Logf("TI:: '%s':: Entry for package '%s' does not exist in local types registry - skipping...", typing, typingKey) + continue + } + if typingLocation, ok := ti.packageNameToTypingLocation.Load(typingKey); ok && IsTypingUpToDate(typingLocation, typesRegistryEntry) { + p.Logf("TI:: '%s':: '%s' already has an up-to-date typing - skipping...", typing, typingKey) + continue + } + result = append(result, typingKey) + } + return result +} + +func (ti *TypingsInstaller) init(p *Project) { + ti.initOnce.Do(func() { + p.Log("TI:: Global cache location '" + ti.TypingsLocation + "'") //, safe file path '" + safeListPath + "', types map path '" + typesMapLocation + "`") + ti.processCacheLocation(p) + + // !!! sheetal handle npm path here if we would support it + // // If the NPM path contains spaces and isn't wrapped in quotes, do so. + // if (this.npmPath.includes(" ") && this.npmPath[0] !== `"`) { + // this.npmPath = `"${this.npmPath}"`; + // } + // if (this.log.isEnabled()) { + // this.log.writeLine(`Process id: ${process.pid}`); + // this.log.writeLine(`NPM location: ${this.npmPath} (explicit '${ts.server.Arguments.NpmLocation}' ${npmLocation === undefined ? "not " : ""} provided)`); + // this.log.writeLine(`validateDefaultNpmLocation: ${validateDefaultNpmLocation}`); + // } + + ti.ensureTypingsLocationExists(p) + p.Log("TI:: Updating types-registry@latest npm package...") + if _, err := ti.options.NpmInstall(ti.TypingsLocation, []string{"install", "--ignore-scripts", "types-registry@latest"}); err == nil { + p.Log("TI:: Updated types-registry npm package") + } else { + p.Logf("TI:: Error updating types-registry package: %v", err) + // !!! sheetal events to send + // // store error info to report it later when it is known that server is already listening to events from typings installer + // this.delayedInitializationError = { + // kind: "event::initializationFailed", + // message: (e as Error).message, + // stack: (e as Error).stack, + // }; + + // const body: protocol.TypesInstallerInitializationFailedEventBody = { + // message: response.message, + // }; + // const eventName: protocol.TypesInstallerInitializationFailedEventName = "typesInstallerInitializationFailed"; + // this.event(body, eventName); + } + + ti.typesRegistry = ti.loadTypesRegistryFile(p) + }) +} + +type NpmConfig struct { + DevDependencies map[string]any `json:"devDependencies"` +} + +type NpmDependecyEntry struct { + Version string `json:"version"` +} +type NpmLock struct { + Dependencies map[string]NpmDependecyEntry `json:"dependencies"` + Packages map[string]NpmDependecyEntry `json:"packages"` +} + +func (ti *TypingsInstaller) processCacheLocation(p *Project) { + p.Log("TI:: Processing cache location " + ti.TypingsLocation) + packageJson := tspath.CombinePaths(ti.TypingsLocation, "package.json") + packageLockJson := tspath.CombinePaths(ti.TypingsLocation, "package-lock.json") + p.Log("TI:: Trying to find '" + packageJson + "'...") + if p.FS().FileExists(packageJson) && p.FS().FileExists((packageLockJson)) { + var npmConfig NpmConfig + npmConfigContents := parseNpmConfigOrLock(p, packageJson, &npmConfig) + var npmLock NpmLock + npmLockContents := parseNpmConfigOrLock(p, packageLockJson, &npmLock) + + p.Log("TI:: Loaded content of " + packageJson + ": " + npmConfigContents) + p.Log("TI:: Loaded content of " + packageLockJson + ": " + npmLockContents) + + // !!! sheetal strada uses Node10 + resolver := module.NewResolver(p, &core.CompilerOptions{ModuleResolution: core.ModuleResolutionKindNodeNext}, "", "") + if npmConfig.DevDependencies != nil && (npmLock.Packages != nil || npmLock.Dependencies != nil) { + for key := range npmConfig.DevDependencies { + npmLockValue, npmLockValueExists := npmLock.Packages["node_modules/"+key] + if !npmLockValueExists { + npmLockValue, npmLockValueExists = npmLock.Dependencies[key] + if !npmLockValueExists { + // if package in package.json but not package-lock.json, skip adding to cache so it is reinstalled on next use + continue + } + } + // key is @types/ + packageName := tspath.GetBaseFileName(key) + if packageName == "" { + continue + } + typingFile := ti.typingToFileName(resolver, packageName) + if typingFile == "" { + ti.missingTypingsSet.Store(packageName, true) + continue + } + if existingTypingFile, existingTypingsFilePresent := ti.packageNameToTypingLocation.Load(packageName); existingTypingsFilePresent { + if existingTypingFile.TypingsLocation == typingFile { + continue + } + p.Log("TI:: New typing for package " + packageName + " from " + typingFile + " conflicts with existing typing file " + existingTypingFile.TypingsLocation) + } + p.Log("TI:: Adding entry into typings cache: " + packageName + " => " + typingFile) + version := npmLockValue.Version + if version == "" { + continue + } + + newTyping := &CachedTyping{ + TypingsLocation: typingFile, + Version: semver.MustParse(version), + } + ti.packageNameToTypingLocation.Store(packageName, newTyping) + } + } + } + p.Log("TI:: Finished processing cache location " + ti.TypingsLocation) +} + +func parseNpmConfigOrLock[T NpmConfig | NpmLock](p *Project, location string, config *T) string { + contents, _ := p.FS().ReadFile(location) + _ = json.Unmarshal([]byte(contents), config) + return contents +} + +func (ti *TypingsInstaller) ensureTypingsLocationExists(p *Project) { + npmConfigPath := tspath.CombinePaths(ti.TypingsLocation, "package.json") + p.Log("TI:: Npm config file: " + npmConfigPath) + + if !p.FS().FileExists(npmConfigPath) { + p.Logf("TI:: Npm config file: '%s' is missing, creating new one...", npmConfigPath) + err := p.FS().WriteFile(npmConfigPath, "{ \"private\": true }", false) + if err != nil { + p.Logf("TI:: Npm config file write failed: %v", err) + } + } +} + +func (ti *TypingsInstaller) typingToFileName(resolver *module.Resolver, packageName string) string { + result := resolver.ResolveModuleName(packageName, tspath.CombinePaths(ti.TypingsLocation, "index.d.ts"), core.ModuleKindNone, nil) + return result.ResolvedFileName +} + +func (ti *TypingsInstaller) loadTypesRegistryFile(p *Project) map[string]map[string]string { + typesRegistryFile := tspath.CombinePaths(ti.TypingsLocation, "node_modules/types-registry/index.json") + typesRegistryFileContents, ok := p.FS().ReadFile(typesRegistryFile) + if ok { + var entries map[string]map[string]map[string]string + err := json.Unmarshal([]byte(typesRegistryFileContents), &entries) + if err == nil { + if typesRegistry, ok := entries["entries"]; ok { + return typesRegistry + } + } + p.Logf("TI:: Error when loading types registry file '%s': %v", typesRegistryFile, err) + } else { + p.Logf("TI:: Error reading types registry file '%s'", typesRegistryFile) + } + return map[string]map[string]string{} +} + +func (ti *TypingsInstaller) initializeSafeList(p *Project) { + if ti.safeList != nil { + return + } + ti.loadTypesMap(p) + if ti.typesMap.SimpleMap != nil { + p.Logf("TI:: Loaded safelist from types map file '%s'", tspath.CombinePaths(p.DefaultLibraryPath(), "typesMap.json")) + ti.safeList = ti.typesMap.SimpleMap + return + } + + p.Logf("TI:: Failed to load safelist from types map file '$%s'", tspath.CombinePaths(p.DefaultLibraryPath(), "typesMap.json")) + ti.loadSafeList(p) +} + +func (ti *TypingsInstaller) loadTypesMap(p *Project) { + if ti.typesMap != nil { + return + } + typesMapLocation := tspath.CombinePaths(p.DefaultLibraryPath(), "typesMap.json") + typesMapContents, ok := p.FS().ReadFile(typesMapLocation) + if ok { + err := json.Unmarshal([]byte(typesMapContents), &ti.typesMap) + if err != nil { + return + } + p.Logf("TI:: Error when parsing typesMapLocation '%s': %v", typesMapLocation, err) + } else { + p.Logf("TI:: Error reading typesMapLocation '%s'", typesMapLocation) + } + ti.typesMap = &TypesMapFile{} +} + +func (ti *TypingsInstaller) loadSafeList(p *Project) { + safeListLocation := tspath.CombinePaths(p.DefaultLibraryPath(), "typingSafeList.json") + safeListContents, ok := p.FS().ReadFile(safeListLocation) + if ok { + err := json.Unmarshal([]byte(safeListContents), &ti.safeList) + if err != nil { + return + } + p.Logf("TI:: Error when parsing safeListLocation '%s': %v", safeListLocation, err) + } else { + p.Logf("TI:: Error reading safeListLocation '%s'", safeListLocation) + } + ti.safeList = map[string]string{} +} + +func NpmInstall(cwd string, npmInstallArgs []string) ([]byte, error) { + cmd := exec.Command("npm", npmInstallArgs...) + cmd.Dir = cwd + return cmd.Output() +} diff --git a/internal/project/ti_test.go b/internal/project/ti_test.go new file mode 100644 index 0000000000..4c82773f99 --- /dev/null +++ b/internal/project/ti_test.go @@ -0,0 +1,799 @@ +package project_test + +import ( + "slices" + "testing" + + "github.com/microsoft/typescript-go/internal/bundled" + "github.com/microsoft/typescript-go/internal/core" + "github.com/microsoft/typescript-go/internal/lsp/lsproto" + "github.com/microsoft/typescript-go/internal/project" + "github.com/microsoft/typescript-go/internal/testutil/projecttestutil" + "gotest.tools/v3/assert" +) + +func TestTi(t *testing.T) { + t.Parallel() + if !bundled.Embedded { + t.Skip("bundled files are not embedded") + } + + t.Run("local module should not be picked up", func(t *testing.T) { + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": `const c = require('./config');`, + "/user/username/projects/project/config.js": `export let x = 1`, + "/user/username/projects/project/jsconfig.json": `{ + "compilerOptions": { "moduleResolution": "commonjs" }, + "typeAcquisition": { "enable": true }, + }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + TypesRegistry: []string{"config"}, + }, + }) + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + assert.Equal(t, len(service.Projects()), 1) + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + assert.Equal(t, p.Kind(), project.KindConfigured) + program := p.CurrentProgram() + assert.Assert(t, program.GetSourceFile("/user/username/projects/project/config.js") != nil) + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Skipped 0 typings", + }) + }) + + t.Run("configured projects", func(t *testing.T) { + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": ``, + "/user/username/projects/project/tsconfig.json": `{ + "compilerOptions": { "allowJs": true }, + "typeAcquisition": { "enable": true }, + }`, + "/user/username/projects/project/package.json": `{ + "name": "test", + "dependencies": { + "jquery": "^3.1.0" + } + }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + PackageToFile: map[string]string{ + "jquery": `declare const $: { x: number }`, + }, + }, + }) + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + assert.Equal(t, len(service.Projects()), 1) + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + assert.Equal(t, p.Kind(), project.KindConfigured) + success := <-host.ServiceOptions.InstallStatus + assert.Equal(t, success, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Success", + }) + program := p.GetProgram() + assert.Assert(t, program.GetSourceFile(projecttestutil.TestTypingsLocation+"/node_modules/@types/jquery/index.d.ts") != nil) + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 2, + Project: p, + Status: "Skipped 0 typings", + }) + }) + + t.Run("inferred projects", func(t *testing.T) { + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": ``, + "/user/username/projects/project/package.json": `{ + "name": "test", + "dependencies": { + "jquery": "^3.1.0" + } + }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + PackageToFile: map[string]string{ + "jquery": `declare const $: { x: number }`, + }, + }, + }) + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + assert.Equal(t, len(service.Projects()), 1) + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + assert.Equal(t, p.Kind(), project.KindInferred) + success := <-host.ServiceOptions.InstallStatus + assert.Equal(t, success, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Success", + }) + program := p.GetProgram() + assert.Assert(t, program.GetSourceFile(projecttestutil.TestTypingsLocation+"/node_modules/@types/jquery/index.d.ts") != nil) + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 2, + Project: p, + Status: "Skipped 1 typings", + }) + }) + + t.Run("type acquisition with disableFilenameBasedTypeAcquisition:true", func(t *testing.T) { + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/jquery.js": ``, + "/user/username/projects/project/tsconfig.json": `{ + "compilerOptions": { "allowJs": true }, + "typeAcquisition": { "enable": true, "disableFilenameBasedTypeAcquisition": true }, + }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + TypesRegistry: []string{"jquery"}, + }, + }) + service.OpenFile("/user/username/projects/project/jquery.js", files["/user/username/projects/project/jquery.js"].(string), core.ScriptKindJS, "") + assert.Equal(t, len(service.Projects()), 1) + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/jquery.js") + assert.Equal(t, p.Kind(), project.KindConfigured) + + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Skipped 0 typings", + }) + }) + + t.Run("deduplicate from local @types packages", func(t *testing.T) { + t.Skip("Todo - implement removing local @types from include list") + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": "", + "/user/username/projects/project/node_modules/@types/node/index.d.ts": "declare var node;", + "/user/username/projects/project/jsconfig.json": `{ + "typeAcquisition": { "include": ["node"] }, + }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + TypesRegistry: []string{"node"}, + }, + }) + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + assert.Equal(t, len(service.Projects()), 1) + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + assert.Equal(t, p.Kind(), project.KindConfigured) + + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Skipped 0 typings", + }) + }) + + t.Run("Throttle - scheduled run install requests without reaching limit", func(t *testing.T) { + t.Parallel() + files := map[string]any{ + "/user/username/projects/project1/app.js": "", + "/user/username/projects/project1/file3.d.ts": "", + "/user/username/projects/project1/jsconfig.json": `{ + "typeAcquisition": { "include": ["jquery", "cordova", "lodash"] }, + }`, + "/user/username/projects/project2/app.js": "", + "/user/username/projects/project2/file3.d.ts": "", + "/user/username/projects/project2/jsconfig.json": `{ + "typeAcquisition": { "include": ["grunt", "gulp", "commander"] }, + }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + PackageToFile: map[string]string{ + "commander": "declare const commander: { x: number }", + "jquery": "declare const jquery: { x: number }", + "lodash": "declare const lodash: { x: number }", + "cordova": "declare const cordova: { x: number }", + "grunt": "declare const grunt: { x: number }", + "gulp": "declare const grunt: { x: number }", + }, + }, + }) + + service.OpenFile("/user/username/projects/project1/app.js", files["/user/username/projects/project1/app.js"].(string), core.ScriptKindJS, "") + service.OpenFile("/user/username/projects/project2/app.js", files["/user/username/projects/project2/app.js"].(string), core.ScriptKindJS, "") + _, p1 := service.EnsureDefaultProjectForFile("/user/username/projects/project1/app.js") + _, p2 := service.EnsureDefaultProjectForFile("/user/username/projects/project2/app.js") + var installStatuses []project.TypingsInstallerStatus + installStatuses = append(installStatuses, <-host.ServiceOptions.InstallStatus, <-host.ServiceOptions.InstallStatus) + // Order can be non deterministic since they both will run in parallel + assert.Assert(t, slices.Contains(installStatuses, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p1, + Status: "Success", + })) + assert.Assert(t, slices.Contains(installStatuses, project.TypingsInstallerStatus{ + RequestId: 2, + Project: p2, + Status: "Success", + })) + }) + + t.Run("Throttle - scheduled run install requests reaching limit", func(t *testing.T) { + t.Parallel() + files := map[string]any{ + "/user/username/projects/project1/app.js": "", + "/user/username/projects/project1/file3.d.ts": "", + "/user/username/projects/project1/jsconfig.json": `{ + "typeAcquisition": { "include": ["jquery", "cordova", "lodash"] }, + }`, + "/user/username/projects/project2/app.js": "", + "/user/username/projects/project2/file3.d.ts": "", + "/user/username/projects/project2/jsconfig.json": `{ + "typeAcquisition": { "include": ["grunt", "gulp", "commander"] }, + }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + PackageToFile: map[string]string{ + "commander": "declare const commander: { x: number }", + "jquery": "declare const jquery: { x: number }", + "lodash": "declare const lodash: { x: number }", + "cordova": "declare const cordova: { x: number }", + "grunt": "declare const grunt: { x: number }", + "gulp": "declare const grunt: { x: number }", + }, + }, + TypingsInstallerOptions: project.TypingsInstallerOptions{ + ThrottleLimit: 1, + }, + }) + + service.OpenFile("/user/username/projects/project1/app.js", files["/user/username/projects/project1/app.js"].(string), core.ScriptKindJS, "") + service.OpenFile("/user/username/projects/project2/app.js", files["/user/username/projects/project2/app.js"].(string), core.ScriptKindJS, "") + _, p1 := service.EnsureDefaultProjectForFile("/user/username/projects/project1/app.js") + _, p2 := service.EnsureDefaultProjectForFile("/user/username/projects/project2/app.js") + // Order is determinate since second install will run only after completing first one + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p1, + Status: "Success", + }) + status = <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 2, + Project: p2, + Status: "Success", + }) + }) + + t.Run("discover from node_modules", func(t *testing.T) { + t.Skip("Skip for now - to add back when we skip external library files to lookup typings for") + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": "", + "/user/username/projects/project/package.json": `{ + "dependencies": { + "jquery": "1.0.0", + }, + }`, + "/user/username/projects/project/jsconfig.json": `{}`, + "/user/username/projects/project/node_modules/commander/index.js": "", + "/user/username/projects/project/node_modules/commander/package.json": `{ "name": "commander" }`, + "/user/username/projects/project/node_modules/jquery/index.js": "", + "/user/username/projects/project/node_modules/jquery/package.json": `{ "name": "jquery" }`, + "/user/username/projects/project/node_modules/jquery/nested/package.json": `{ "name": "nested" }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + TypesRegistry: []string{"nested", "commander"}, + PackageToFile: map[string]string{ + "jquery": "declare const jquery: { x: number }", + }, + }, + }) + + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + // Order is determinate since second install will run only after completing first one + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Success", + }) + }) + + // Explicit types prevent automatic inclusion from package.json listing + t.Run("discover from node_modules empty types", func(t *testing.T) { + t.Skip("Skip for now - to add back when we skip external library files to lookup typings for") + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": "", + "/user/username/projects/project/package.json": `{ + "dependencies": { + "jquery": "1.0.0", + }, + }`, + "/user/username/projects/project/jsconfig.json": `{ + "compilerOptions": { + "types": [] + } + }`, + "/user/username/projects/project/node_modules/commander/index.js": "", + "/user/username/projects/project/node_modules/commander/package.json": `{ "name": "commander" }`, + "/user/username/projects/project/node_modules/jquery/index.js": "", + "/user/username/projects/project/node_modules/jquery/package.json": `{ "name": "jquery" }`, + "/user/username/projects/project/node_modules/jquery/nested/package.json": `{ "name": "nested" }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + TypesRegistry: []string{"nested", "commander"}, + PackageToFile: map[string]string{ + "jquery": "declare const jquery: { x: number }", + }, + }, + }) + + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + // Order is determinate since second install will run only after completing first one + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Success", + }) + }) + + // A type reference directive will not resolve to the global typings cache + t.Run("discover from node_modules explicit types", func(t *testing.T) { + t.Skip("Skip for now - to add back when we skip external library files to lookup typings for") + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": "", + "/user/username/projects/project/package.json": `{ + "dependencies": { + "jquery": "1.0.0", + }, + }`, + "/user/username/projects/project/jsconfig.json": `{ + "compilerOptions": { + "types": ["jquery"] + } + }`, + "/user/username/projects/project/node_modules/commander/index.js": "", + "/user/username/projects/project/node_modules/commander/package.json": `{ "name": "commander" }`, + "/user/username/projects/project/node_modules/jquery/index.js": "", + "/user/username/projects/project/node_modules/jquery/package.json": `{ "name": "jquery" }`, + "/user/username/projects/project/node_modules/jquery/nested/package.json": `{ "name": "nested" }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + TypesRegistry: []string{"nested", "commander"}, + PackageToFile: map[string]string{ + "jquery": "declare const jquery: { x: number }", + }, + }, + }) + + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + // Order is determinate since second install will run only after completing first one + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Success", + }) + }) + + // However, explicit types will not prevent unresolved imports from pulling in typings + t.Run("discover from node_modules empty types has import", func(t *testing.T) { + t.Skip("Skip for now - to add back when we skip external library files to lookup typings for") + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": `import "jquery";`, + "/user/username/projects/project/package.json": `{ + "dependencies": { + "jquery": "1.0.0", + }, + }`, + "/user/username/projects/project/jsconfig.json": `{ + "compilerOptions": { + "types": [] + } + }`, + "/user/username/projects/project/node_modules/commander/index.js": "", + "/user/username/projects/project/node_modules/commander/package.json": `{ "name": "commander" }`, + "/user/username/projects/project/node_modules/jquery/index.js": "", + "/user/username/projects/project/node_modules/jquery/package.json": `{ "name": "jquery" }`, + "/user/username/projects/project/node_modules/jquery/nested/package.json": `{ "name": "nested" }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + TypesRegistry: []string{"nested", "commander"}, + PackageToFile: map[string]string{ + "jquery": "declare const jquery: { x: number }", + }, + }, + }) + + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + // Order is determinate since second install will run only after completing first one + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Success", + }) + }) + + t.Run("discover from bower_components", func(t *testing.T) { + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": ``, + "/user/username/projects/project/jsconfig.json": `{}`, + "/user/username/projects/project/bower_components/jquery/index.js": "", + "/user/username/projects/project/bower_components/jquery/bower.json": `{ "name": "jquery" }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + PackageToFile: map[string]string{ + "jquery": "declare const jquery: { x: number }", + }, + }, + }) + + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + // Order is determinate since second install will run only after completing first one + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Success", + }) + program := p.GetProgram() + assert.Assert(t, program.GetSourceFile(projecttestutil.TestTypingsLocation+"/node_modules/@types/jquery/index.d.ts") != nil) + }) + + t.Run("discover from bower.json", func(t *testing.T) { + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": ``, + "/user/username/projects/project/jsconfig.json": `{}`, + "/user/username/projects/project/bower.json": `{ + "dependencies": { + "jquery": "^3.1.0" + } + }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + PackageToFile: map[string]string{ + "jquery": "declare const jquery: { x: number }", + }, + }, + }) + + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + // Order is determinate since second install will run only after completing first one + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Success", + }) + program := p.GetProgram() + assert.Assert(t, program.GetSourceFile(projecttestutil.TestTypingsLocation+"/node_modules/@types/jquery/index.d.ts") != nil) + }) + + t.Run("Malformed package.json should be watched", func(t *testing.T) { + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": ``, + "/user/username/projects/project/package.json": `{ "dependencies": { "co } }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + PackageToFile: map[string]string{ + "commander": "export let x: number", + }, + }, + }) + + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + // Order is determinate since second install will run only after completing first one + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Skipped 1 typings", + }) + assert.NilError(t, host.FS().WriteFile( + "/user/username/projects/project/package.json", + `{ "dependencies": { "commander": "0.0.2" } }`, + false, + )) + assert.NilError(t, service.OnWatchedFilesChanged(t.Context(), []*lsproto.FileEvent{ + { + Type: lsproto.FileChangeTypeChanged, + Uri: "file:///user/username/projects/project/package.json", + }, + })) + status = <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 2, + Project: p, + Status: "Success", + }) + program := p.GetProgram() + assert.Assert(t, program.GetSourceFile(projecttestutil.TestTypingsLocation+"/node_modules/@types/commander/index.d.ts") != nil) + }) + + t.Run("should install typings for unresolved imports", func(t *testing.T) { + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": ` + import * as fs from "fs"; + import * as commander from "commander"; + import * as component from "@ember/component"; + `, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + PackageToFile: map[string]string{ + "node": "export let node: number", + "commander": "export let commander: number", + "ember__component": "export let ember__component: number", + }, + }, + }) + + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + // Order is determinate since second install will run only after completing first one + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Success", + }) + program := p.GetProgram() + assert.Assert(t, program.GetSourceFile(projecttestutil.TestTypingsLocation+"/node_modules/@types/node/index.d.ts") != nil) + assert.Assert(t, program.GetSourceFile(projecttestutil.TestTypingsLocation+"/node_modules/@types/commander/index.d.ts") != nil) + assert.Assert(t, program.GetSourceFile(projecttestutil.TestTypingsLocation+"/node_modules/@types/ember__component/index.d.ts") != nil) + }) + + t.Run("should redo resolution that resolved to '.js' file after typings are installed", func(t *testing.T) { + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": ` + import * as commander from "commander"; + `, + "/user/username/projects/node_modules/commander/index.js": "module.exports = 0", + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + PackageToFile: map[string]string{ + "commander": "export let commander: number", + }, + }, + }) + + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + // Order is determinate since second install will run only after completing first one + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Success", + }) + program := p.GetProgram() + assert.Assert(t, program.GetSourceFile(projecttestutil.TestTypingsLocation+"/node_modules/@types/commander/index.d.ts") != nil) + assert.Assert(t, program.GetSourceFile("/user/username/projects/node_modules/commander/index.js") == nil) + }) + + t.Run("expired cache entry (inferred project, should install typings)", func(t *testing.T) { + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": "", + "/user/username/projects/project/package.json": `{ + "name": "test", + "dependencies": { + "jquery": "^3.1.0" + } + }`, + projecttestutil.TestTypingsLocation + "/node_modules/@types/jquery/index.d.ts": "export const x = 10;", + projecttestutil.TestTypingsLocation + "/package.json": `{ + "dependencies": { + "types-registry": "^0.1.317" + }, + "devDependencies": { + "@types/jquery": "^1.0.0" + } + }`, + projecttestutil.TestTypingsLocation + "/package-lock.json": `{ + "dependencies": { + "@types/jquery": { + "version": "1.0.0" + } + } + }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + PackageToFile: map[string]string{ + "jquery": "export const y = 10", + }, + }, + }) + + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + // Order is determinate since second install will run only after completing first one + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Success", + }) + program := p.GetProgram() + assert.Equal(t, program.GetSourceFile(projecttestutil.TestTypingsLocation+"/node_modules/@types/jquery/index.d.ts").Text(), "export const y = 10") + }) + + t.Run("non-expired cache entry (inferred project, should not install typings)", func(t *testing.T) { + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": "", + "/user/username/projects/project/package.json": `{ + "name": "test", + "dependencies": { + "jquery": "^3.1.0" + } + }`, + projecttestutil.TestTypingsLocation + "/node_modules/@types/jquery/index.d.ts": "export const x = 10;", + projecttestutil.TestTypingsLocation + "/package.json": `{ + "dependencies": { + "types-registry": "^0.1.317" + }, + "devDependencies": { + "@types/jquery": "^1.3.0" + } + }`, + projecttestutil.TestTypingsLocation + "/package-lock.json": `{ + "dependencies": { + "@types/jquery": { + "version": "1.3.0" + } + } + }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + TypesRegistry: []string{"jquery"}, + }, + }) + + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + // Order is determinate since second install will run only after completing first one + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Skipped 1 typings", + }) + program := p.GetProgram() + assert.Equal(t, program.GetSourceFile(projecttestutil.TestTypingsLocation+"/node_modules/@types/jquery/index.d.ts").Text(), "export const x = 10;") + }) + + t.Run("expired cache entry (inferred project, should install typings) lockfile3", func(t *testing.T) { + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": "", + "/user/username/projects/project/package.json": `{ + "name": "test", + "dependencies": { + "jquery": "^3.1.0" + } + }`, + projecttestutil.TestTypingsLocation + "/node_modules/@types/jquery/index.d.ts": "export const x = 10;", + projecttestutil.TestTypingsLocation + "/package.json": `{ + "dependencies": { + "types-registry": "^0.1.317" + }, + "devDependencies": { + "@types/jquery": "^1.0.0" + } + }`, + projecttestutil.TestTypingsLocation + "/package-lock.json": `{ + "packages": { + "node_modules/@types/jquery": { + "version": "1.0.0" + } + } + }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + PackageToFile: map[string]string{ + "jquery": "export const y = 10", + }, + }, + }) + + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + // Order is determinate since second install will run only after completing first one + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Success", + }) + program := p.GetProgram() + assert.Equal(t, program.GetSourceFile(projecttestutil.TestTypingsLocation+"/node_modules/@types/jquery/index.d.ts").Text(), "export const y = 10") + }) + + t.Run("non-expired cache entry (inferred project, should not install typings) lockfile3", func(t *testing.T) { + t.Parallel() + files := map[string]any{ + "/user/username/projects/project/app.js": "", + "/user/username/projects/project/package.json": `{ + "name": "test", + "dependencies": { + "jquery": "^3.1.0" + } + }`, + projecttestutil.TestTypingsLocation + "/node_modules/@types/jquery/index.d.ts": "export const x = 10;", + projecttestutil.TestTypingsLocation + "/package.json": `{ + "dependencies": { + "types-registry": "^0.1.317" + }, + "devDependencies": { + "@types/jquery": "^1.3.0" + } + }`, + projecttestutil.TestTypingsLocation + "/package-lock.json": `{ + "packages": { + "node_modules/@types/jquery": { + "version": "1.3.0" + } + } + }`, + } + service, host := projecttestutil.Setup(files, &projecttestutil.TestTypingsInstaller{ + TestTypingsInstallerOptions: projecttestutil.TestTypingsInstallerOptions{ + TypesRegistry: []string{"jquery"}, + }, + }) + + service.OpenFile("/user/username/projects/project/app.js", files["/user/username/projects/project/app.js"].(string), core.ScriptKindJS, "") + _, p := service.EnsureDefaultProjectForFile("/user/username/projects/project/app.js") + // Order is determinate since second install will run only after completing first one + status := <-host.ServiceOptions.InstallStatus + assert.Equal(t, status, project.TypingsInstallerStatus{ + RequestId: 1, + Project: p, + Status: "Skipped 1 typings", + }) + program := p.GetProgram() + assert.Equal(t, program.GetSourceFile(projecttestutil.TestTypingsLocation+"/node_modules/@types/jquery/index.d.ts").Text(), "export const x = 10;") + }) +} diff --git a/internal/project/validatepackagename.go b/internal/project/validatepackagename.go new file mode 100644 index 0000000000..7b56da4b5a --- /dev/null +++ b/internal/project/validatepackagename.go @@ -0,0 +1,98 @@ +package project + +import ( + "fmt" + "net/url" + "strings" + "unicode/utf8" +) + +type NameValidationResult int + +const ( + NameOk NameValidationResult = iota + EmptyName + NameTooLong + NameStartsWithDot + NameStartsWithUnderscore + NameContainsNonURISafeCharacters +) + +const maxPackageNameLength = 214 + +/** + * Validates package name using rules defined at https://docs.npmjs.com/files/package.json + * + * @internal + */ +func ValidatePackageName(packageName string) (result NameValidationResult, name string, isScopeName bool) { + return validatePackageNameWorker(packageName /*supportScopedPackage*/, true) +} + +func validatePackageNameWorker(packageName string, supportScopedPackage bool) (result NameValidationResult, name string, isScopeName bool) { + packageNameLen := len(packageName) + if packageNameLen == 0 { + return EmptyName, "", false + } + if packageNameLen > maxPackageNameLength { + return NameTooLong, "", false + } + firstChar, _ := utf8.DecodeRuneInString(packageName) + if firstChar == '.' { + return NameStartsWithDot, "", false + } + if firstChar == '_' { + return NameStartsWithUnderscore, "", false + } + // check if name is scope package like: starts with @ and has one '/' in the middle + // scoped packages are not currently supported + if supportScopedPackage { + if withoutScope, found := strings.CutPrefix(packageName, "@"); found { + scope, scopedPackageName, found := strings.Cut(withoutScope, "/") + if found && len(scope) > 0 && len(scopedPackageName) > 0 && strings.Index(scopedPackageName, "/") == -1 { + scopeResult, _, _ := validatePackageNameWorker(scope /*supportScopedPackage*/, false) + if scopeResult != NameOk { + return scopeResult, scope, true + } + packageResult, _, _ := validatePackageNameWorker(scopedPackageName /*supportScopedPackage*/, false) + if packageResult != NameOk { + return packageResult, scopedPackageName, false + } + return NameOk, "", false + } + } + } + if url.QueryEscape(packageName) != packageName { + return NameContainsNonURISafeCharacters, "", false + } + return NameOk, "", false +} + +/** @internal */ +func RenderPackageNameValidationFailure(typing string, result NameValidationResult, name string, isScopeName bool) string { + var kind string + if isScopeName { + kind = "Scope" + } else { + kind = "Package" + } + if name == "" { + name = typing + } + switch result { + case EmptyName: + return fmt.Sprintf("'%s':: %s name '%s' cannot be empty", typing, kind, name) + case NameTooLong: + return fmt.Sprintf("'%s':: %s name '%s' should be less than %d characters", typing, kind, name, maxPackageNameLength) + case NameStartsWithDot: + return fmt.Sprintf("'%s':: %s name '%s' cannot start with '.'", typing, kind, name) + case NameStartsWithUnderscore: + return fmt.Sprintf("'%s':: %s name '%s' cannot start with '_'", typing, kind, name) + case NameContainsNonURISafeCharacters: + return fmt.Sprintf("'%s':: %s name '%s' contains non URI safe characters", typing, kind, name) + case NameOk: + panic("Unexpected Ok result") + default: + panic("Unknown package name validation result") + } +} diff --git a/internal/project/validatepackagename_test.go b/internal/project/validatepackagename_test.go new file mode 100644 index 0000000000..4d1e6762f5 --- /dev/null +++ b/internal/project/validatepackagename_test.go @@ -0,0 +1,107 @@ +package project_test + +import ( + "testing" + + "github.com/microsoft/typescript-go/internal/project" + "gotest.tools/v3/assert" +) + +func TestValidatePackageName(t *testing.T) { + t.Parallel() + t.Run("name cannot be too long", func(t *testing.T) { + t.Parallel() + packageName := "a" + for range 8 { + packageName += packageName + } + status, _, _ := project.ValidatePackageName(packageName) + assert.Equal(t, status, project.NameTooLong) + }) + t.Run("package name cannot start with dot", func(t *testing.T) { + t.Parallel() + status, _, _ := project.ValidatePackageName(".foo") + assert.Equal(t, status, project.NameStartsWithDot) + }) + t.Run("package name cannot start with underscore", func(t *testing.T) { + t.Parallel() + status, _, _ := project.ValidatePackageName("_foo") + assert.Equal(t, status, project.NameStartsWithUnderscore) + }) + t.Run("package non URI safe characters are not supported", func(t *testing.T) { + t.Parallel() + status, _, _ := project.ValidatePackageName(" scope ") + assert.Equal(t, status, project.NameContainsNonURISafeCharacters) + status, _, _ = project.ValidatePackageName("; say ‘Hello from TypeScript!’ #") + assert.Equal(t, status, project.NameContainsNonURISafeCharacters) + status, _, _ = project.ValidatePackageName("a/b/c") + assert.Equal(t, status, project.NameContainsNonURISafeCharacters) + }) + t.Run("scoped package name is supported", func(t *testing.T) { + t.Parallel() + status, _, _ := project.ValidatePackageName("@scope/bar") + assert.Equal(t, status, project.NameOk) + }) + t.Run("scoped name in scoped package name cannot start with dot", func(t *testing.T) { + t.Parallel() + status, name, isScopeName := project.ValidatePackageName("@.scope/bar") + assert.Equal(t, status, project.NameStartsWithDot) + assert.Equal(t, name, ".scope") + assert.Equal(t, isScopeName, true) + status, name, isScopeName = project.ValidatePackageName("@.scope/.bar") + assert.Equal(t, status, project.NameStartsWithDot) + assert.Equal(t, name, ".scope") + assert.Equal(t, isScopeName, true) + }) + t.Run("scoped name in scoped package name cannot start with dot", func(t *testing.T) { + t.Parallel() + status, name, isScopeName := project.ValidatePackageName("@_scope/bar") + assert.Equal(t, status, project.NameStartsWithUnderscore) + assert.Equal(t, name, "_scope") + assert.Equal(t, isScopeName, true) + status, name, isScopeName = project.ValidatePackageName("@_scope/_bar") + assert.Equal(t, status, project.NameStartsWithUnderscore) + assert.Equal(t, name, "_scope") + assert.Equal(t, isScopeName, true) + }) + t.Run("scope name in scoped package name with non URI safe characters are not supported", func(t *testing.T) { + t.Parallel() + status, name, isScopeName := project.ValidatePackageName("@ scope /bar") + assert.Equal(t, status, project.NameContainsNonURISafeCharacters) + assert.Equal(t, name, " scope ") + assert.Equal(t, isScopeName, true) + status, name, isScopeName = project.ValidatePackageName("@; say ‘Hello from TypeScript!’ #/bar") + assert.Equal(t, status, project.NameContainsNonURISafeCharacters) + assert.Equal(t, name, "; say ‘Hello from TypeScript!’ #") + assert.Equal(t, isScopeName, true) + status, name, isScopeName = project.ValidatePackageName("@ scope / bar ") + assert.Equal(t, status, project.NameContainsNonURISafeCharacters) + assert.Equal(t, name, " scope ") + assert.Equal(t, isScopeName, true) + }) + t.Run("package name in scoped package name cannot start with dot", func(t *testing.T) { + t.Parallel() + status, name, isScopeName := project.ValidatePackageName("@scope/.bar") + assert.Equal(t, status, project.NameStartsWithDot) + assert.Equal(t, name, ".bar") + assert.Equal(t, isScopeName, false) + }) + t.Run("package name in scoped package name cannot start with underscore", func(t *testing.T) { + t.Parallel() + status, name, isScopeName := project.ValidatePackageName("@scope/_bar") + assert.Equal(t, status, project.NameStartsWithUnderscore) + assert.Equal(t, name, "_bar") + assert.Equal(t, isScopeName, false) + }) + t.Run("package name in scoped package name with non URI safe characters are not supported", func(t *testing.T) { + t.Parallel() + status, name, isScopeName := project.ValidatePackageName("@scope/ bar ") + assert.Equal(t, status, project.NameContainsNonURISafeCharacters) + assert.Equal(t, name, " bar ") + assert.Equal(t, isScopeName, false) + status, name, isScopeName = project.ValidatePackageName("@scope/; say ‘Hello from TypeScript!’ #") + assert.Equal(t, status, project.NameContainsNonURISafeCharacters) + assert.Equal(t, name, "; say ‘Hello from TypeScript!’ #") + assert.Equal(t, isScopeName, false) + }) +} diff --git a/internal/project/watch.go b/internal/project/watch.go index 2098f0fd82..351509c763 100644 --- a/internal/project/watch.go +++ b/internal/project/watch.go @@ -3,6 +3,7 @@ package project import ( "context" "fmt" + "maps" "slices" "strings" "time" @@ -17,24 +18,39 @@ const ( ) type watchedFiles[T any] struct { - client Client + p *Project getGlobs func(data T) []string watchKind lsproto.WatchKind data T globs []string watcherID WatcherHandle + watchType string } -func newWatchedFiles[T any](client Client, watchKind lsproto.WatchKind, getGlobs func(data T) []string) *watchedFiles[T] { +func newWatchedFiles[T any]( + p *Project, + watchKind lsproto.WatchKind, + getGlobs func(data T) []string, + watchType string, +) *watchedFiles[T] { return &watchedFiles[T]{ - client: client, + p: p, watchKind: watchKind, getGlobs: getGlobs, + watchType: watchType, } } -func (w *watchedFiles[T]) update(ctx context.Context, newData T) (updated bool, err error) { +func (w *watchedFiles[T]) update(ctx context.Context, newData T) { + if updated, err := w.updateWorker(ctx, newData); err != nil { + w.p.Log(fmt.Sprintf("Failed to update %s watch: %v\n%s", w.watchType, err, formatFileList(w.globs, "\t", hr))) + } else if updated { + w.p.Logf("%s watches updated %s:\n%s", w.watchType, w.watcherID, formatFileList(w.globs, "\t", hr)) + } +} + +func (w *watchedFiles[T]) updateWorker(ctx context.Context, newData T) (updated bool, err error) { newGlobs := w.getGlobs(newData) w.data = newData if slices.Equal(w.globs, newGlobs) { @@ -43,11 +59,16 @@ func (w *watchedFiles[T]) update(ctx context.Context, newData T) (updated bool, w.globs = newGlobs if w.watcherID != "" { - if err = w.client.UnwatchFiles(ctx, w.watcherID); err != nil { + if err = w.p.host.Client().UnwatchFiles(ctx, w.watcherID); err != nil { return false, err } } + w.watcherID = "" + if len(newGlobs) == 0 { + return true, nil + } + watchers := make([]*lsproto.FileSystemWatcher, 0, len(newGlobs)) for _, glob := range newGlobs { watchers = append(watchers, &lsproto.FileSystemWatcher{ @@ -57,7 +78,7 @@ func (w *watchedFiles[T]) update(ctx context.Context, newData T) (updated bool, Kind: &w.watchKind, }) } - watcherID, err := w.client.WatchFiles(ctx, watchers) + watcherID, err := w.p.host.Client().WatchFiles(ctx, watchers) if err != nil { return false, err } @@ -65,7 +86,11 @@ func (w *watchedFiles[T]) update(ctx context.Context, newData T) (updated bool, return true, nil } -func createGlobMapper(host ProjectHost) func(data map[tspath.Path]string) []string { +func globMapperForTypingsInstaller(data map[tspath.Path]string) []string { + return slices.Sorted(maps.Values(data)) +} + +func createResolutionLookupGlobMapper(host ProjectHost) func(data map[tspath.Path]string) []string { rootDir := host.GetCurrentDirectory() rootPath := tspath.ToPath(rootDir, "", host.FS().UseCaseSensitiveFileNames()) rootPathComponents := tspath.GetPathComponents(string(rootPath), "") diff --git a/internal/testutil/projecttestutil/projecttestutil.go b/internal/testutil/projecttestutil/projecttestutil.go index fc6f9f9cb0..bf6edf7072 100644 --- a/internal/testutil/projecttestutil/projecttestutil.go +++ b/internal/testutil/projecttestutil/projecttestutil.go @@ -4,18 +4,30 @@ import ( "context" "fmt" "io" + "slices" "strings" "sync" "github.com/microsoft/typescript-go/internal/bundled" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/project" + "github.com/microsoft/typescript-go/internal/tspath" "github.com/microsoft/typescript-go/internal/vfs" "github.com/microsoft/typescript-go/internal/vfs/vfstest" ) //go:generate go tool github.com/matryer/moq -stub -fmt goimports -pkg projecttestutil -out clientmock_generated.go ../../project Client +type TestTypingsInstallerOptions struct { + TypesRegistry []string + PackageToFile map[string]string +} + +type TestTypingsInstaller struct { + project.TypingsInstallerOptions + TestTypingsInstallerOptions +} + type ProjectServiceHost struct { fs vfs.FS mu sync.Mutex @@ -23,13 +35,24 @@ type ProjectServiceHost struct { output strings.Builder logger *project.Logger ClientMock *ClientMock + testOptions *TestTypingsInstallerOptions + ServiceOptions *project.ServiceOptions } +const ( + TestTypingsLocation = "/home/src/Library/Caches/typescript" + TestLibLocation = "/home/src/tslibs/TS/Lib" +) + // DefaultLibraryPath implements project.ProjectServiceHost. func (p *ProjectServiceHost) DefaultLibraryPath() string { return p.defaultLibraryPath } +func (p *ProjectServiceHost) TypingsLocation() string { + return TestTypingsLocation +} + // FS implements project.ProjectServiceHost. func (p *ProjectServiceHost) FS() vfs.FS { return p.fs @@ -63,15 +86,128 @@ func (p *ProjectServiceHost) ReplaceFS(files map[string]any) { var _ project.ServiceHost = (*ProjectServiceHost)(nil) -func Setup(files map[string]any) (*project.Service, *ProjectServiceHost) { +func Setup(files map[string]any, testOptions *TestTypingsInstaller) (*project.Service, *ProjectServiceHost) { host := newProjectServiceHost(files) - service := project.NewService(host, project.ServiceOptions{ + if testOptions != nil { + host.testOptions = &testOptions.TestTypingsInstallerOptions + } + var throttleLimit int + if testOptions != nil && testOptions.ThrottleLimit != 0 { + throttleLimit = testOptions.ThrottleLimit + } else { + throttleLimit = 5 + } + host.ServiceOptions = &project.ServiceOptions{ Logger: host.logger, WatchEnabled: true, - }) + TypingsInstallerOptions: project.TypingsInstallerOptions{ + ThrottleLimit: throttleLimit, + + NpmInstall: host.NpmInstall, + InstallStatus: make(chan project.TypingsInstallerStatus), + }, + } + service := project.NewService(host, *host.ServiceOptions) return service, host } +func (p *ProjectServiceHost) NpmInstall(cwd string, npmInstallArgs []string) ([]byte, error) { + if p.testOptions == nil { + return nil, nil + } + + lenNpmInstallArgs := len(npmInstallArgs) + if lenNpmInstallArgs < 3 { + panic(fmt.Sprintf("Unexpected npm install: %s %v", cwd, npmInstallArgs)) + } + + if lenNpmInstallArgs == 3 && npmInstallArgs[2] == "types-registry@latest" { + // Write typings file + err := p.FS().WriteFile(tspath.CombinePaths(cwd, "node_modules/types-registry/index.json"), p.createTypesRegistryFileContent(), false) + return nil, err + } + + for _, atTypesPackageTs := range npmInstallArgs[2 : lenNpmInstallArgs-2] { + // @types/packageName@TsVersionToUse + packageName := atTypesPackageTs[7 : len(atTypesPackageTs)-len(project.TsVersionToUse)-1] + content, ok := p.testOptions.PackageToFile[packageName] + if !ok { + return nil, fmt.Errorf("content not provided for %s", packageName) + } + err := p.FS().WriteFile(tspath.CombinePaths(cwd, "node_modules/@types/"+packageName+"/index.d.ts"), content, false) + if err != nil { + return nil, err + } + } + return nil, nil +} + +var ( + typesRegistryConfigTextOnce sync.Once + typesRegistryConfigText string +) + +func TypesRegistryConfigText() string { + typesRegistryConfigTextOnce.Do(func() { + var result strings.Builder + for key, value := range TypesRegistryConfig() { + if result.Len() != 0 { + result.WriteString(",") + } + result.WriteString(fmt.Sprintf("\n \"%s\": \"%s\"", key, value)) + + } + typesRegistryConfigText = result.String() + }) + return typesRegistryConfigText +} + +var ( + typesRegistryConfigOnce sync.Once + typesRegistryConfig map[string]string +) + +func TypesRegistryConfig() map[string]string { + typesRegistryConfigOnce.Do(func() { + typesRegistryConfig = map[string]string{ + "latest": "1.3.0", + "ts2.0": "1.0.0", + "ts2.1": "1.0.0", + "ts2.2": "1.2.0", + "ts2.3": "1.3.0", + "ts2.4": "1.3.0", + "ts2.5": "1.3.0", + "ts2.6": "1.3.0", + "ts2.7": "1.3.0", + } + }) + return typesRegistryConfig +} + +func (p *ProjectServiceHost) createTypesRegistryFileContent() string { + var builder strings.Builder + builder.WriteString("{\n \"entries\": {") + for index, entry := range p.testOptions.TypesRegistry { + appendTypesRegistryConfig(&builder, index, entry) + } + index := len(p.testOptions.TypesRegistry) + for key := range p.testOptions.PackageToFile { + if !slices.Contains(p.testOptions.TypesRegistry, key) { + appendTypesRegistryConfig(&builder, index, key) + index++ + } + } + builder.WriteString("\n }\n}") + return builder.String() +} + +func appendTypesRegistryConfig(builder *strings.Builder, index int, entry string) { + if index > 0 { + builder.WriteString(",") + } + builder.WriteString(fmt.Sprintf("\n \"%s\": {%s\n }", entry, TypesRegistryConfigText())) +} + func newProjectServiceHost(files map[string]any) *ProjectServiceHost { fs := bundled.WrapFS(vfstest.FromMap(files, false /*useCaseSensitiveFileNames*/)) host := &ProjectServiceHost{ diff --git a/internal/tsoptions/parsedcommandline.go b/internal/tsoptions/parsedcommandline.go index 544aafa096..02978664cf 100644 --- a/internal/tsoptions/parsedcommandline.go +++ b/internal/tsoptions/parsedcommandline.go @@ -62,6 +62,14 @@ func (p *ParsedCommandLine) CompilerOptions() *core.CompilerOptions { return p.ParsedConfig.CompilerOptions } +func (p *ParsedCommandLine) SetTypeAcquisition(o *core.TypeAcquisition) { + p.ParsedConfig.TypeAcquisition = o +} + +func (p *ParsedCommandLine) TypeAcquisition() *core.TypeAcquisition { + return p.ParsedConfig.TypeAcquisition +} + // All file names matched by files, include, and exclude patterns func (p *ParsedCommandLine) FileNames() []string { return p.ParsedConfig.FileNames diff --git a/internal/tsoptions/tsconfigparsing.go b/internal/tsoptions/tsconfigparsing.go index 44782554e6..90fa4de234 100644 --- a/internal/tsoptions/tsconfigparsing.go +++ b/internal/tsoptions/tsconfigparsing.go @@ -102,8 +102,8 @@ func (c *configFileSpecs) matchesExclude(fileName string, comparePathsOptions ts if len(c.validatedExcludeSpecs) == 0 { return false } - excludePattern := getRegularExpressionForWildcard(c.validatedExcludeSpecs, comparePathsOptions.CurrentDirectory, "exclude") - excludeRegex := getRegexFromPattern(excludePattern, comparePathsOptions.UseCaseSensitiveFileNames) + excludePattern := vfs.GetRegularExpressionForWildcard(c.validatedExcludeSpecs, comparePathsOptions.CurrentDirectory, "exclude") + excludeRegex := vfs.GetRegexFromPattern(excludePattern, comparePathsOptions.UseCaseSensitiveFileNames) if match, err := excludeRegex.MatchString(fileName); err == nil && match { return true } @@ -120,9 +120,9 @@ func (c *configFileSpecs) matchesInclude(fileName string, comparePathsOptions ts return false } for _, spec := range c.validatedIncludeSpecs { - includePattern := getPatternFromSpec(spec, comparePathsOptions.CurrentDirectory, "files") + includePattern := vfs.GetPatternFromSpec(spec, comparePathsOptions.CurrentDirectory, "files") if includePattern != "" { - includeRegex := getRegexFromPattern(includePattern, comparePathsOptions.UseCaseSensitiveFileNames) + includeRegex := vfs.GetRegexFromPattern(includePattern, comparePathsOptions.UseCaseSensitiveFileNames) if match, err := includeRegex.MatchString(fileName); err == nil && match { return true } @@ -1536,15 +1536,15 @@ func getFileNamesFromConfigSpecs( var jsonOnlyIncludeRegexes []*regexp2.Regexp if len(validatedIncludeSpecs) > 0 { - files := readDirectory(host, basePath, basePath, core.Flatten(supportedExtensionsWithJsonIfResolveJsonModule), validatedExcludeSpecs, validatedIncludeSpecs, nil) + files := vfs.ReadDirectory(host, basePath, basePath, core.Flatten(supportedExtensionsWithJsonIfResolveJsonModule), validatedExcludeSpecs, validatedIncludeSpecs, nil) for _, file := range files { if tspath.FileExtensionIs(file, tspath.ExtensionJson) { if jsonOnlyIncludeRegexes == nil { includes := core.Filter(validatedIncludeSpecs, func(include string) bool { return strings.HasSuffix(include, tspath.ExtensionJson) }) - includeFilePatterns := core.Map(getRegularExpressionsForWildcards(includes, basePath, "files"), func(pattern string) string { return fmt.Sprintf("^%s$", pattern) }) + includeFilePatterns := core.Map(vfs.GetRegularExpressionsForWildcards(includes, basePath, "files"), func(pattern string) string { return fmt.Sprintf("^%s$", pattern) }) if includeFilePatterns != nil { jsonOnlyIncludeRegexes = core.Map(includeFilePatterns, func(pattern string) *regexp2.Regexp { - return getRegexFromPattern(pattern, host.UseCaseSensitiveFileNames()) + return vfs.GetRegexFromPattern(pattern, host.UseCaseSensitiveFileNames()) }) } else { jsonOnlyIncludeRegexes = nil @@ -1633,3 +1633,28 @@ func GetSupportedExtensionsWithJsonIfResolveJsonModule(compilerOptions *core.Com } return slices.Concat(supportedExtensions, [][]string{{tspath.ExtensionJson}}) } + +// Reads the config file and reports errors. +func GetParsedCommandLineOfConfigFile(configFileName string, options *core.CompilerOptions, sys ParseConfigHost, extendedConfigCache map[tspath.Path]*ExtendedConfigCacheEntry) (*ParsedCommandLine, []*ast.Diagnostic) { + errors := []*ast.Diagnostic{} + configFileText, errors := tryReadFile(configFileName, sys.FS().ReadFile, errors) + if len(errors) > 0 { + // these are unrecoverable errors--exit to report them as diagnostics + return nil, errors + } + + cwd := sys.GetCurrentDirectory() + tsConfigSourceFile := NewTsconfigSourceFileFromFilePath(configFileName, tspath.ToPath(configFileName, cwd, sys.FS().UseCaseSensitiveFileNames()), configFileText) + // tsConfigSourceFile.resolvedPath = tsConfigSourceFile.FileName() + // tsConfigSourceFile.originalFileName = tsConfigSourceFile.FileName() + return ParseJsonSourceFileConfigFileContent( + tsConfigSourceFile, + sys, + tspath.GetNormalizedAbsolutePath(tspath.GetDirectoryPath(configFileName), cwd), + options, + tspath.GetNormalizedAbsolutePath(configFileName, cwd), + nil, + nil, + extendedConfigCache, + ), nil +} diff --git a/internal/tsoptions/wildcarddirectories.go b/internal/tsoptions/wildcarddirectories.go index c906d14bf2..33068745ac 100644 --- a/internal/tsoptions/wildcarddirectories.go +++ b/internal/tsoptions/wildcarddirectories.go @@ -6,6 +6,7 @@ import ( "github.com/dlclark/regexp2" "github.com/microsoft/typescript-go/internal/tspath" + "github.com/microsoft/typescript-go/internal/vfs" ) func getWildcardDirectories(include []string, exclude []string, comparePathsOptions tspath.ComparePathsOptions) map[string]bool { @@ -26,7 +27,7 @@ func getWildcardDirectories(include []string, exclude []string, comparePathsOpti return nil } - rawExcludeRegex := getRegularExpressionForWildcard(exclude, comparePathsOptions.CurrentDirectory, "exclude") + rawExcludeRegex := vfs.GetRegularExpressionForWildcard(exclude, comparePathsOptions.CurrentDirectory, "exclude") var excludeRegex *regexp.Regexp if rawExcludeRegex != "" { options := "" @@ -129,7 +130,7 @@ func getWildcardDirectoryFromSpec(spec string, useCaseSensitiveFileNames bool) * if lastSepIndex := strings.LastIndexByte(spec, tspath.DirectorySeparator); lastSepIndex != -1 { lastSegment := spec[lastSepIndex+1:] - if isImplicitGlob(lastSegment) { + if vfs.IsImplicitGlob(lastSegment) { path := tspath.RemoveTrailingDirectorySeparator(spec) return &wildcardDirectoryMatch{ Key: toCanonicalKey(path, useCaseSensitiveFileNames), diff --git a/internal/tspath/extension.go b/internal/tspath/extension.go index a1d4c9c2b4..243530c067 100644 --- a/internal/tspath/extension.go +++ b/internal/tspath/extension.go @@ -105,6 +105,10 @@ func IsDeclarationFileName(fileName string) bool { return GetDeclarationFileExtension(fileName) != "" } +func ExtensionIsOneOf(ext string, extensions []string) bool { + return slices.Contains(extensions, ext) +} + func GetDeclarationFileExtension(fileName string) string { base := GetBaseFileName(fileName) for _, ext := range supportedDeclarationExtensions { diff --git a/internal/tsoptions/utilities.go b/internal/vfs/utilities.go similarity index 86% rename from internal/tsoptions/utilities.go rename to internal/vfs/utilities.go index 3a9da6e1b5..e496bf6867 100644 --- a/internal/tsoptions/utilities.go +++ b/internal/vfs/utilities.go @@ -1,4 +1,4 @@ -package tsoptions +package vfs import ( "fmt" @@ -8,11 +8,9 @@ import ( "sync" "github.com/dlclark/regexp2" - "github.com/microsoft/typescript-go/internal/ast" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/stringutil" "github.com/microsoft/typescript-go/internal/tspath" - "github.com/microsoft/typescript-go/internal/vfs" ) type FileMatcherPatterns struct { @@ -33,7 +31,7 @@ const ( usageExclude usage = "exclude" ) -func getRegularExpressionsForWildcards(specs []string, basePath string, usage usage) []string { +func GetRegularExpressionsForWildcards(specs []string, basePath string, usage usage) []string { if len(specs) == 0 { return nil } @@ -42,8 +40,8 @@ func getRegularExpressionsForWildcards(specs []string, basePath string, usage us }) } -func getRegularExpressionForWildcard(specs []string, basePath string, usage usage) string { - patterns := getRegularExpressionsForWildcards(specs, basePath, usage) +func GetRegularExpressionForWildcard(specs []string, basePath string, usage usage) string { + patterns := GetRegularExpressionsForWildcards(specs, basePath, usage) if len(patterns) == 0 { return "" } @@ -80,7 +78,7 @@ var isImplicitGlobRegex = regexp2.MustCompile(`[.*?]`, regexp2.None) // An "includes" path "foo" is implicitly a glob "foo/** /*" (without the space) if its last component has no extension, // and does not contain any glob characters itself. -func isImplicitGlob(lastPathComponent string) bool { +func IsImplicitGlob(lastPathComponent string) bool { match, err := isImplicitGlobRegex.MatchString(lastPathComponent) if err != nil { return false @@ -151,7 +149,7 @@ var wildcardMatchers = map[usage]WildcardMatcher{ usageExclude: excludeMatcher, } -func getPatternFromSpec( +func GetPatternFromSpec( spec string, basePath string, usage usage, @@ -186,7 +184,7 @@ func getSubPatternFromSpec( // We need to remove to create our regex correctly. components[0] = tspath.RemoveTrailingDirectorySeparator(components[0]) - if isImplicitGlob(lastComponent) { + if IsImplicitGlob(lastComponent) { components = append(components, "**", "*") } @@ -302,10 +300,10 @@ func getFileMatcherPatterns(path string, excludes []string, includes []string, u absolutePath := tspath.CombinePaths(currentDirectory, path) return FileMatcherPatterns{ - includeFilePatterns: core.Map(getRegularExpressionsForWildcards(includes, absolutePath, "files"), func(pattern string) string { return "^" + pattern + "$" }), - includeFilePattern: getRegularExpressionForWildcard(includes, absolutePath, "files"), - includeDirectoryPattern: getRegularExpressionForWildcard(includes, absolutePath, "directories"), - excludePattern: getRegularExpressionForWildcard(excludes, absolutePath, "exclude"), + includeFilePatterns: core.Map(GetRegularExpressionsForWildcards(includes, absolutePath, "files"), func(pattern string) string { return "^" + pattern + "$" }), + includeFilePattern: GetRegularExpressionForWildcard(includes, absolutePath, "files"), + includeDirectoryPattern: GetRegularExpressionForWildcard(includes, absolutePath, "directories"), + excludePattern: GetRegularExpressionForWildcard(excludes, absolutePath, "exclude"), basePaths: getBasePaths(path, includes, useCaseSensitiveFileNames), } } @@ -320,7 +318,7 @@ var ( regexp2Cache = make(map[regexp2CacheKey]*regexp2.Regexp) ) -func getRegexFromPattern(pattern string, useCaseSensitiveFileNames bool) *regexp2.Regexp { +func GetRegexFromPattern(pattern string, useCaseSensitiveFileNames bool) *regexp2.Regexp { flags := regexp2.ECMAScript if !useCaseSensitiveFileNames { flags |= regexp2.IgnoreCase @@ -364,7 +362,7 @@ type visitor struct { includeDirectoryRegex *regexp2.Regexp extensions []string useCaseSensitiveFileNames bool - host vfs.FS + host FS visited core.Set[string] results [][]string } @@ -420,22 +418,22 @@ func (v *visitor) visitDirectory( } // path is the directory of the tsconfig.json -func matchFiles(path string, extensions []string, excludes []string, includes []string, useCaseSensitiveFileNames bool, currentDirectory string, depth *int, host vfs.FS) []string { +func matchFiles(path string, extensions []string, excludes []string, includes []string, useCaseSensitiveFileNames bool, currentDirectory string, depth *int, host FS) []string { path = tspath.NormalizePath(path) currentDirectory = tspath.NormalizePath(currentDirectory) patterns := getFileMatcherPatterns(path, excludes, includes, useCaseSensitiveFileNames, currentDirectory) var includeFileRegexes []*regexp2.Regexp if patterns.includeFilePatterns != nil { - includeFileRegexes = core.Map(patterns.includeFilePatterns, func(pattern string) *regexp2.Regexp { return getRegexFromPattern(pattern, useCaseSensitiveFileNames) }) + includeFileRegexes = core.Map(patterns.includeFilePatterns, func(pattern string) *regexp2.Regexp { return GetRegexFromPattern(pattern, useCaseSensitiveFileNames) }) } var includeDirectoryRegex *regexp2.Regexp if patterns.includeDirectoryPattern != "" { - includeDirectoryRegex = getRegexFromPattern(patterns.includeDirectoryPattern, useCaseSensitiveFileNames) + includeDirectoryRegex = GetRegexFromPattern(patterns.includeDirectoryPattern, useCaseSensitiveFileNames) } var excludeRegex *regexp2.Regexp if patterns.excludePattern != "" { - excludeRegex = getRegexFromPattern(patterns.excludePattern, useCaseSensitiveFileNames) + excludeRegex = GetRegexFromPattern(patterns.excludePattern, useCaseSensitiveFileNames) } // Associate an array of results with each include regex. This keeps results in order of the "include" order. @@ -466,31 +464,6 @@ func matchFiles(path string, extensions []string, excludes []string, includes [] return core.Flatten(results) } -func readDirectory(host vfs.FS, currentDir string, path string, extensions []string, excludes []string, includes []string, depth *int) []string { +func ReadDirectory(host FS, currentDir string, path string, extensions []string, excludes []string, includes []string, depth *int) []string { return matchFiles(path, extensions, excludes, includes, host.UseCaseSensitiveFileNames(), currentDir, depth, host) } - -// Reads the config file and reports errors. -func GetParsedCommandLineOfConfigFile(configFileName string, options *core.CompilerOptions, sys ParseConfigHost, extendedConfigCache map[tspath.Path]*ExtendedConfigCacheEntry) (*ParsedCommandLine, []*ast.Diagnostic) { - errors := []*ast.Diagnostic{} - configFileText, errors := tryReadFile(configFileName, sys.FS().ReadFile, errors) - if len(errors) > 0 { - // these are unrecoverable errors--exit to report them as diagnostics - return nil, errors - } - - cwd := sys.GetCurrentDirectory() - tsConfigSourceFile := NewTsconfigSourceFileFromFilePath(configFileName, tspath.ToPath(configFileName, cwd, sys.FS().UseCaseSensitiveFileNames()), configFileText) - // tsConfigSourceFile.resolvedPath = tsConfigSourceFile.FileName() - // tsConfigSourceFile.originalFileName = tsConfigSourceFile.FileName() - return ParseJsonSourceFileConfigFileContent( - tsConfigSourceFile, - sys, - tspath.GetNormalizedAbsolutePath(tspath.GetDirectoryPath(configFileName), cwd), - options, - tspath.GetNormalizedAbsolutePath(configFileName, cwd), - nil, - nil, - extendedConfigCache, - ), nil -}