diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4bb0c3f..ff7718a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -43,6 +43,11 @@ jobs: - name: Functional Tests id: functional + # We want to run these tests even if the unit tests fail, because + # it is useful to know if e.g. the unit tests fail due to one + # stackage endpoint failing, but the functional tests pass due to + # a backup working. + if: always() shell: bash run: NO_CLEANUP=1 cabal test functional diff --git a/README.md b/README.md index 4ee0b2f..75a143e 100644 --- a/README.md +++ b/README.md @@ -61,15 +61,42 @@ The procedure is as follows: ### The clc-stackage exe -Previously, this project was just a single (massive) cabal file that had to be manually updated. Usage was fairly simple: `cabal build clc-stackage --keep-going` to build the project, `--keep-going` so that as many packages as possible are built. +`clc-stackage` is an executable that will: -This has been updated so that `clc-stackage` is now an executable that will automatically generate the desired cabal file based on the results of querying stackage directly. This streamlines updates, provides a more flexible build process, and potentially has prettier output (with `--batch` arg): +1. Download the stackage snapshot from the stackage server. +2. Divide the snapshot into groups (determined by `--batch` argument). +3. For each group, generate a cabal file and attempt to build it. -![demo](example_output.png) +#### Querying stackage -In particular, the `clc-stackage` exe allows for splitting the entire package set into subset groups of size `N` with the `--batch N` option. Each group is then built sequentially. Not only can this be useful for situations where building the entire package set in one go is infeasible, but it also provides a "cache" functionality, that allows us to interrupt the program at any point (e.g. `CTRL-C`), and pick up where we left off. For example: +By default, `clc-stackage` queries https://www.stackage.org/ for snapshot information. In situations where this is not desirable (e.g. the server is not working, or we want to test a custom snapshot), the snapshot can be overridden: +```sh +$ ./bin/clc-stackage --snapshot-path=path/to/snapshot ``` + +This snapshot should be formatted similar to the `cabal.config` endpoint on the stackage server (e.g. https://www.stackage.org/nightly/cabal.config). That is, package lines should be formatted ` ==`: + +``` +abstract-deque ==0.3 +abstract-deque-tests ==0.3 +abstract-par ==0.3.3 +AC-Angle ==1.0 +acc ==0.2.0.3 +... +``` + +The stackage config itself is valid, so trailing commas and other extraneous lines are allowed (and ignored). + +#### Investigating failures + +By default (`--write-logs save-failures`), the build logs are saved to the `./output/logs/` directory, with `./output/logs/current-build/` streaming the current build logs. + +#### Group batching + +The `clc-stackage` exe allows for splitting the entire package set into subset groups of size `N` with the `--batch N` option. Each group is then built sequentially. Not only can this be useful for situations where building the entire package set in one go is infeasible, but it also provides a "cache" functionality, that allows us to interrupt the program at any point (e.g. `CTRL-C`), and pick up where we left off. For example: + +```sh $ ./bin/clc-stackage --batch 100 ``` @@ -77,7 +104,7 @@ This will split the entire downloaded package set into groups of size 100. Each See `./bin/clc-stackage --help` for more info. -#### Optimal performance +##### Optimal performance On the one hand, splitting the entire package set into `--batch` groups makes the output easier to understand and offers a nice workflow for interrupting/restarting the build. On the other hand, there is a question of what the best value of `N` is for `--batch N`, with respect to performance. diff --git a/app/Main.hs b/app/Main.hs index d4f375b..4f00334 100644 --- a/app/Main.hs +++ b/app/Main.hs @@ -2,26 +2,15 @@ module Main (main) where import CLC.Stackage.Runner qualified as Runner import CLC.Stackage.Utils.Logging qualified as Logging -import Data.Text qualified as T -import Data.Time.LocalTime qualified as Local import System.Console.Terminal.Size qualified as TermSize -import System.IO (hPutStrLn, stderr) main :: IO () main = do mWidth <- (fmap . fmap) TermSize.width TermSize.size + let hLogger = Logging.mkDefaultLogger case mWidth of - Just w -> Runner.run $ mkLogger w + Just w -> Runner.run $ hLogger {Logging.terminalWidth = w} Nothing -> do - let hLogger = mkLogger 80 - Logging.putTimeInfoStr hLogger False "Failed detecting terminal width" + Logging.putTimeInfoStr hLogger "Failed detecting terminal width" Runner.run hLogger - where - mkLogger w = - Logging.MkHandle - { Logging.getLocalTime = Local.zonedTimeToLocalTime <$> Local.getZonedTime, - Logging.logStrErrLn = hPutStrLn stderr . T.unpack, - Logging.logStrLn = putStrLn . T.unpack, - Logging.terminalWidth = w - } diff --git a/cabal.project b/cabal.project index 45adba7..08f105a 100644 --- a/cabal.project +++ b/cabal.project @@ -17,7 +17,6 @@ program-options -Wprepositive-qualified-module -Wredundant-constraints -Wunused-binds - -Wunused-packages -Wunused-type-patterns -Wno-unticked-promoted-constructors diff --git a/clc-stackage.cabal b/clc-stackage.cabal index 2e4d2a1..52662a7 100644 --- a/clc-stackage.cabal +++ b/clc-stackage.cabal @@ -24,7 +24,6 @@ common common-lang if os(windows) cpp-options: -DWINDOWS - build-depends: base >=4.16.0.0 && <4.22 default-language: GHC2021 library utils @@ -35,15 +34,19 @@ library utils CLC.Stackage.Utils.JSON CLC.Stackage.Utils.Logging CLC.Stackage.Utils.OS + CLC.Stackage.Utils.Package CLC.Stackage.Utils.Paths build-depends: , aeson >=2.0 && <2.3 , aeson-pretty ^>=0.8.9 + , base >=4.16.0.0 && <4.22 , bytestring >=0.10.12.0 && <0.13 + , deepseq >=1.4.6.0 && <1.6 , directory ^>=1.3.5.0 , file-io ^>=0.1.0.0 - , filepath >=1.4.2.1 && <1.6 + , filepath >=1.5.0.0 && <1.6 + , os-string ^>=2.0.0 , pretty-terminal ^>=0.1.0.0 , text >=1.2.3.2 && <2.2 , time >=1.9.3 && <1.15 @@ -55,13 +58,16 @@ library parser exposed-modules: CLC.Stackage.Parser CLC.Stackage.Parser.API - CLC.Stackage.Parser.Data.Response - CLC.Stackage.Parser.Query + CLC.Stackage.Parser.API.CabalConfig + CLC.Stackage.Parser.API.Common + CLC.Stackage.Parser.API.JSON build-depends: , aeson + , base , bytestring , containers >=0.6.3.1 && <0.9 + , deepseq , filepath , http-client >=0.5.9 && <0.8 , http-client-tls ^>=0.3 @@ -70,6 +76,7 @@ library parser , utils hs-source-dirs: src/parser + ghc-options: -Wunused-packages library builder import: common-lang @@ -77,12 +84,11 @@ library builder CLC.Stackage.Builder CLC.Stackage.Builder.Batch CLC.Stackage.Builder.Env - CLC.Stackage.Builder.Package CLC.Stackage.Builder.Process CLC.Stackage.Builder.Writer build-depends: - , aeson + , base , containers , directory , filepath @@ -91,6 +97,7 @@ library builder , utils hs-source-dirs: src/builder + ghc-options: -Wunused-packages library runner import: common-lang @@ -102,6 +109,7 @@ library runner build-depends: , aeson + , base , builder , containers , directory @@ -114,56 +122,60 @@ library runner , utils hs-source-dirs: src/runner + ghc-options: -Wunused-packages executable clc-stackage import: common-lang main-is: Main.hs build-depends: + , base , runner , terminal-size ^>=0.3.4 - , text - , time , utils hs-source-dirs: ./app - ghc-options: -threaded -with-rtsopts=-N + ghc-options: -threaded -with-rtsopts=-N -Wunused-packages library test-utils import: common-lang - exposed-modules: - Test.Utils - + exposed-modules: Test.Utils build-depends: , base , tasty >=1.1.0.3 && <1.6 , tasty-golden ^>=2.3.1.1 hs-source-dirs: test/utils + ghc-options: -Wunused-packages test-suite unit import: common-lang type: exitcode-stdio-1.0 main-is: Main.hs other-modules: + Unit.CLC.Stackage.Parser.API Unit.CLC.Stackage.Runner.Env Unit.CLC.Stackage.Runner.Report + Unit.CLC.Stackage.Utils.Package Unit.Prelude build-depends: , base , builder , containers + , deepseq , filepath + , http-client-tls + , parser , runner , tasty , tasty-golden - , tasty-hunit >=0.9 && <0.11 + , tasty-hunit >=0.9 && <0.11 , test-utils , time , utils hs-source-dirs: test/unit - ghc-options: -threaded -with-rtsopts=-N + ghc-options: -threaded -with-rtsopts=-N -Wunused-packages test-suite functional import: common-lang @@ -177,11 +189,15 @@ test-suite functional , env-guard ^>=0.2 , filepath , runner - , test-utils , tasty , tasty-golden + , test-utils , text , time , utils hs-source-dirs: test/functional + +-- For some reason -Wunused-packages is complaining about clc-stackage +-- being an unnecessary dep for the functional test suite...hence it is +-- removed from cabal.project and added manually to other targets. diff --git a/dev.md b/dev.md index b718132..cd87162 100644 --- a/dev.md +++ b/dev.md @@ -96,9 +96,11 @@ The executable that actually runs. This is a very thin wrapper over `runner`, wh 1. `ghc-version` in [.github/workflows/ci.yaml](.github/workflows/ci.yaml). 2. [README.md](README.md). -5. Optional: Update `clc-stackage.cabal`'s dependencies (i.e. `cabal outdated`). +5. Update functional tests as needed i.e. exact package versions in `*golden` and `test/functional/snapshot.txt`. -6. Optional: Update nix inputs (`nix flake update`). +6. Optional: Update `clc-stackage.cabal`'s dependencies (i.e. `cabal outdated`). + +7. Optional: Update nix inputs (`nix flake update`). ## Testing diff --git a/example_output.png b/example_output.png deleted file mode 100644 index 9b6f8fe..0000000 Binary files a/example_output.png and /dev/null differ diff --git a/src/builder/CLC/Stackage/Builder/Batch.hs b/src/builder/CLC/Stackage/Builder/Batch.hs index 8d5fe13..ac3dfe3 100644 --- a/src/builder/CLC/Stackage/Builder/Batch.hs +++ b/src/builder/CLC/Stackage/Builder/Batch.hs @@ -10,7 +10,7 @@ import CLC.Stackage.Builder.Env packagesToBuild ), ) -import CLC.Stackage.Builder.Package (Package) +import CLC.Stackage.Utils.Package (Package) import Data.Bifunctor (Bifunctor (first)) import Data.List qualified as L import Data.List.NonEmpty (NonEmpty ((:|)), (<|)) diff --git a/src/builder/CLC/Stackage/Builder/Env.hs b/src/builder/CLC/Stackage/Builder/Env.hs index 3fb5475..a6c565e 100644 --- a/src/builder/CLC/Stackage/Builder/Env.hs +++ b/src/builder/CLC/Stackage/Builder/Env.hs @@ -1,54 +1,16 @@ -- | Provides the environment for building. module CLC.Stackage.Builder.Env ( BuildEnv (..), - CabalVerbosity (..), - cabalVerbosityToArg, - Jobs (..), - jobsToArg, Progress (..), WriteLogs (..), ) where -import CLC.Stackage.Builder.Package (Package) import CLC.Stackage.Utils.Logging qualified as Logging +import CLC.Stackage.Utils.Package (Package) import Data.IORef (IORef) import Data.List.NonEmpty (NonEmpty) import Data.Set (Set) -import Data.Word (Word8) - --- | Cabal's --verbose flag -data CabalVerbosity - = -- | V0 - CabalVerbosity0 - | -- | V1 - CabalVerbosity1 - | -- | V2 - CabalVerbosity2 - | -- | V3 - CabalVerbosity3 - deriving stock (Eq, Show) - -cabalVerbosityToArg :: CabalVerbosity -> String -cabalVerbosityToArg CabalVerbosity0 = "--verbose=0" -cabalVerbosityToArg CabalVerbosity1 = "--verbose=1" -cabalVerbosityToArg CabalVerbosity2 = "--verbose=2" -cabalVerbosityToArg CabalVerbosity3 = "--verbose=3" - --- | Number of build jobs. -data Jobs - = -- | Literal number of jobs. - JobsN Word8 - | -- | String "$ncpus" - JobsNCpus - | -- | Job semaphore. Requires GHC 9.8 and Cabal 3.12 - JobsSemaphore - deriving stock (Eq, Show) - -jobsToArg :: Jobs -> String -jobsToArg (JobsN n) = "--jobs=" ++ show n -jobsToArg JobsNCpus = "--jobs=$ncpus" -jobsToArg JobsSemaphore = "--semaphore" data Progress = MkProgress { -- | Dependencies that built successfully. @@ -74,8 +36,8 @@ data BuildEnv = MkBuildEnv batch :: Maybe Int, -- | Build arguments for cabal. buildArgs :: [String], - -- | If true, colors logs. - colorLogs :: Bool, + -- | Optional path to cabal executable. + cabalPath :: FilePath, -- | If true, the first group that fails to completely build stops -- clc-stackage. Defaults to false. groupFailFast :: Bool, diff --git a/src/builder/CLC/Stackage/Builder/Package.hs b/src/builder/CLC/Stackage/Builder/Package.hs deleted file mode 100644 index 9c2c558..0000000 --- a/src/builder/CLC/Stackage/Builder/Package.hs +++ /dev/null @@ -1,65 +0,0 @@ -{-# LANGUAGE ViewPatterns #-} - --- | Provides the type representing a package with version. -module CLC.Stackage.Builder.Package - ( Package (..), - fromText, - toText, - toDepText, - toDirName, - ) -where - -import CLC.Stackage.Utils.Paths qualified as Paths -import Data.Aeson (FromJSON, ToJSON) -import Data.String (IsString (fromString)) -import Data.Text (Text) -import Data.Text qualified as T -import GHC.Generics (Generic) -import System.OsPath (OsPath) - --- | Package data. -data Package = MkPackage - { name :: Text, - version :: Text - } - deriving stock (Eq, Generic, Ord, Show) - deriving anyclass (FromJSON, ToJSON) - -instance IsString Package where - fromString s = case fromText (T.pack s) of - Nothing -> - error $ - mconcat - [ "String '", - s, - "' did no match expected package format: ==" - ] - Just p -> p - -fromText :: Text -> Maybe Package -fromText txt = case T.breakOn delim txt of - (xs, T.stripPrefix delim -> Just ys) - -- point exists but version is empty - | T.null ys -> Nothing - -- correct - | otherwise -> Just $ MkPackage xs ys - -- point does not exist - _ -> Nothing - --- | Text representation of the package e.g. 'foo ==1.2.3'. -toText :: Package -> Text -toText p = p.name <> delim <> p.version - -delim :: Text -delim = " ==" - --- | Text representation suitable for cabal file build-depends. -toDepText :: Package -> Text -toDepText = (", " <>) . toText - --- | Returns an OsPath name based on this package i.e. the OsPath --- representation of 'toText'. Used when naming an error directory for a --- package that fails. -toDirName :: Package -> IO OsPath -toDirName = Paths.encodeUtf . T.unpack . toText diff --git a/src/builder/CLC/Stackage/Builder/Process.hs b/src/builder/CLC/Stackage/Builder/Process.hs index 9bf88c3..55342f5 100644 --- a/src/builder/CLC/Stackage/Builder/Process.hs +++ b/src/builder/CLC/Stackage/Builder/Process.hs @@ -8,7 +8,7 @@ where import CLC.Stackage.Builder.Batch (PackageGroup (unPackageGroup)) import CLC.Stackage.Builder.Env ( BuildEnv - ( colorLogs, + ( cabalPath, groupFailFast, hLogger, progress, @@ -18,10 +18,10 @@ import CLC.Stackage.Builder.Env WriteLogs (WriteLogsCurrent, WriteLogsNone, WriteLogsSaveFailures), ) import CLC.Stackage.Builder.Env qualified as Env -import CLC.Stackage.Builder.Package qualified as Package import CLC.Stackage.Builder.Writer qualified as Writer import CLC.Stackage.Utils.IO qualified as IO import CLC.Stackage.Utils.Logging qualified as Logging +import CLC.Stackage.Utils.Package qualified as Package import CLC.Stackage.Utils.Paths qualified as Paths import Control.Exception (throwIO) import Control.Monad (when) @@ -45,7 +45,7 @@ buildProject env idx pkgs = do let buildNoLogs :: IO ExitCode buildNoLogs = withGeneratedDir $ - (\(ec, _, _) -> ec) <$> P.readProcessWithExitCode "cabal" env.buildArgs "" + (\(ec, _, _) -> ec) <$> P.readProcessWithExitCode env.cabalPath env.buildArgs "" buildLogs :: Bool -> IO ExitCode buildLogs saveFailures = do @@ -53,7 +53,7 @@ buildProject env idx pkgs = do IO.withBinaryFileWriteMode stdoutPath $ \stdoutHandle -> IO.withBinaryFileWriteMode stderrPath $ \stderrHandle -> do - let createProc = P.proc "cabal" env.buildArgs + let createProc = P.proc env.cabalPath env.buildArgs createProc' = createProc { P.std_out = P.UseHandle stdoutHandle, @@ -100,11 +100,11 @@ buildProject env idx pkgs = do ExitSuccess -> do -- save results modifyIORef' env.progress.successesRef addPackages - Logging.putTimeSuccessStr env.hLogger env.colorLogs msg + Logging.putTimeSuccessStr env.hLogger msg ExitFailure _ -> do -- save results modifyIORef' env.progress.failuresRef addPackages - Logging.putTimeErrStr env.hLogger env.colorLogs msg + Logging.putTimeErrStr env.hLogger msg -- throw error if fail fast when env.groupFailFast $ throwIO exitCode @@ -115,7 +115,7 @@ buildProject env idx pkgs = do mconcat [ T.pack $ show idx, ": ", - T.intercalate ", " (Package.toText <$> pkgsList) + T.intercalate ", " (Package.toDisplayName <$> pkgsList) ] pkgsList = NE.toList pkgs.unPackageGroup pkgsSet = Set.fromList pkgsList diff --git a/src/builder/CLC/Stackage/Builder/Writer.hs b/src/builder/CLC/Stackage/Builder/Writer.hs index 4bb2f24..48787ea 100644 --- a/src/builder/CLC/Stackage/Builder/Writer.hs +++ b/src/builder/CLC/Stackage/Builder/Writer.hs @@ -5,9 +5,9 @@ module CLC.Stackage.Builder.Writer where import CLC.Stackage.Builder.Batch (PackageGroup (unPackageGroup)) -import CLC.Stackage.Builder.Package (Package) -import CLC.Stackage.Builder.Package qualified as Package import CLC.Stackage.Utils.IO qualified as IO +import CLC.Stackage.Utils.Package (Package) +import CLC.Stackage.Utils.Package qualified as Package import CLC.Stackage.Utils.Paths qualified as Paths import Data.List.NonEmpty qualified as NE import Data.Text (Text) @@ -33,7 +33,7 @@ writeCabalProjectLocal pkgs = IO.writeBinaryFile path constraintsSrc path = Paths.generatedCabalProjectLocalPath constraintsSrc = TEnc.encodeUtf8 constraintsTxt constraintsTxt = T.unlines $ "constraints:" : constraints - constraints = (\p -> " " <> Package.toText p <> ",") <$> pkgs + constraints = (\p -> " " <> Package.toCabalConstraintsText p) <$> pkgs -- | Writes the package set to a cabal file for building. This will be called -- for each group we want to build. @@ -60,7 +60,7 @@ mkCabalFile pkgs = " default-language: Haskell2010" ] where - pkgsTxt = (\p -> pkgsIndent <> Package.toDepText p) <$> NE.toList pkgs.unPackageGroup + pkgsTxt = (\p -> pkgsIndent <> Package.toCabalDepText p) <$> NE.toList pkgs.unPackageGroup -- build-depends is indented 4, then 2 for the package itself. pkgsIndent :: Text diff --git a/src/parser/CLC/Stackage/Parser.hs b/src/parser/CLC/Stackage/Parser.hs index 4c021c7..b92bf86 100644 --- a/src/parser/CLC/Stackage/Parser.hs +++ b/src/parser/CLC/Stackage/Parser.hs @@ -1,4 +1,3 @@ -{-# LANGUAGE CPP #-} {-# LANGUAGE QuasiQuotes #-} module CLC.Stackage.Parser @@ -11,15 +10,19 @@ module CLC.Stackage.Parser ) where -import CLC.Stackage.Parser.Data.Response - ( PackageResponse (name, version), - StackageResponse (packages), +import CLC.Stackage.Parser.API + ( StackageResponse (packages), ) -import CLC.Stackage.Parser.Query qualified as Query +import CLC.Stackage.Parser.API qualified as API +import CLC.Stackage.Parser.API.CabalConfig qualified as CabalConfig import CLC.Stackage.Utils.IO qualified as IO import CLC.Stackage.Utils.JSON qualified as JSON +import CLC.Stackage.Utils.Logging qualified as Logging import CLC.Stackage.Utils.OS (Os (Linux, Osx, Windows)) import CLC.Stackage.Utils.OS qualified as OS +import CLC.Stackage.Utils.Package (Package) +import CLC.Stackage.Utils.Package qualified as Package +import Control.Monad (when) import Data.Aeson (FromJSON, ToJSON) import Data.Foldable (for_) import Data.Set (Set) @@ -27,16 +30,17 @@ import Data.Set qualified as Set import Data.Text (Text) import Data.Text qualified as T import GHC.Generics (Generic) -import System.OsPath (osp) +import System.OsPath (OsPath, osp) -- | Retrieves the list of packages, based on -- 'CLC.Stackage.Parser.API.stackageUrl'. -getPackageList :: IO [PackageResponse] -getPackageList = getPackageListByOs OS.currentOs +getPackageList :: Logging.Handle -> Maybe OsPath -> IO [Package] +getPackageList hLogger msnapshotPath = + getPackageListByOs hLogger msnapshotPath OS.currentOs -- | Prints the package list to a file. -printPackageList :: Bool -> Maybe Os -> IO () -printPackageList incVers mOs = do +printPackageList :: Maybe Os -> IO () +printPackageList mOs = do case mOs of Just os -> printOsList os Nothing -> for_ [minBound .. maxBound] printOsList @@ -46,26 +50,37 @@ printPackageList incVers mOs = do file Windows = [osp|pkgs_windows.txt|] printOsList os = do - pkgs <- getPackageListByOsFmt incVers os + pkgs <- getPackageListByOsFmt os let txt = T.unlines pkgs IO.writeFileUtf8 (file os) txt -- | Retrieves the package list formatted to text. -getPackageListByOsFmt :: Bool -> Os -> IO [Text] -getPackageListByOsFmt incVers = (fmap . fmap) toText . getPackageListByOs - where - toText r = - if incVers - then r.name <> "-" <> r.version - else r.name +getPackageListByOsFmt :: Os -> IO [Text] +getPackageListByOsFmt = + (fmap . fmap) Package.toDisplayName + . getPackageListByOs Logging.mkDefaultLogger Nothing -- | Helper in case we want to see what the package set for a given OS is. -getPackageListByOs :: Os -> IO [PackageResponse] -getPackageListByOs os = do +getPackageListByOs :: Logging.Handle -> Maybe OsPath -> Os -> IO [Package] +getPackageListByOs hLogger msnapshotPath os = do excludedPkgs <- getExcludedPkgs os let filterExcluded = flip Set.notMember excludedPkgs . (.name) - response <- Query.getStackage + response <- case msnapshotPath of + Nothing -> API.getStackage hLogger + Just snapshotPath -> + CabalConfig.parseCabalConfig + <$> IO.readFileUtf8 snapshotPath + + let numPackages = length response.packages + when (numPackages < 2000) $ do + let msg = + mconcat + [ "Only found ", + T.pack $ show numPackages, + " packages. Is that right?" + ] + Logging.putTimeWarnStr hLogger msg let packages = filter filterExcluded response.packages diff --git a/src/parser/CLC/Stackage/Parser/API.hs b/src/parser/CLC/Stackage/Parser/API.hs index b8c0f24..6b19704 100644 --- a/src/parser/CLC/Stackage/Parser/API.hs +++ b/src/parser/CLC/Stackage/Parser/API.hs @@ -1,35 +1,52 @@ -- | REST API for stackage.org. module CLC.Stackage.Parser.API - ( withResponse, + ( -- * Querying stackage + StackageResponse (..), + getStackage, + + -- ** Exceptions + StackageException (..), + ExceptionReason (..), + + -- * Misc stackageSnapshot, ) where -import Network.HTTP.Client (BodyReader, Request, Response) -import Network.HTTP.Client qualified as HttpClient +import CLC.Stackage.Parser.API.CabalConfig qualified as CabalConfig +import CLC.Stackage.Parser.API.Common + ( ExceptionReason + ( ReasonDecodeJson, + ReasonDecodeUtf8, + ReasonReadBody, + ReasonStatus + ), + StackageException (MkStackageException), + StackageResponse (MkStackageResponse, packages), + ) +import CLC.Stackage.Parser.API.JSON qualified as JSON +import CLC.Stackage.Utils.Exception qualified as Ex +import CLC.Stackage.Utils.Logging qualified as Logging +import Control.Exception (Exception (displayException)) +import Data.Text qualified as T import Network.HTTP.Client.TLS qualified as TLS --- | Hits the stackage endpoint, invoking the callback on the result. -withResponse :: (Response BodyReader -> IO a) -> IO a -withResponse onResponse = do +-- | Returns the 'StackageResponse' corresponding to the given snapshot. +getStackage :: Logging.Handle -> IO StackageResponse +getStackage hLogger = do manager <- TLS.newTlsManager - req <- getRequest - HttpClient.withResponse req manager onResponse + Ex.tryAny (JSON.getStackage manager stackageSnapshot) >>= \case + Right r1 -> pure $ r1 + Left jsonEx -> do + let msg = + mconcat + [ "Json endpoint failed. Trying cabal config next: ", + T.pack $ displayException jsonEx + ] -getRequest :: IO Request -getRequest = updateReq <$> mkReq - where - mkReq = HttpClient.parseRequest stackageUrl - updateReq r = - r - { HttpClient.requestHeaders = - [ ("Accept", "application/json;charset=utf-8,application/json") - ] - } + Logging.putTimeWarnStr hLogger msg --- | Url for the stackage snapshot. -stackageUrl :: String -stackageUrl = "https://stackage.org/" <> stackageSnapshot + CabalConfig.getStackage manager stackageSnapshot -- | Stackage snapshot. Note that picking a "good" snapshot is something of -- an art i.e. not all valid snapshots return json output at the diff --git a/src/parser/CLC/Stackage/Parser/API/CabalConfig.hs b/src/parser/CLC/Stackage/Parser/API/CabalConfig.hs new file mode 100644 index 0000000..de7a323 --- /dev/null +++ b/src/parser/CLC/Stackage/Parser/API/CabalConfig.hs @@ -0,0 +1,87 @@ +module CLC.Stackage.Parser.API.CabalConfig + ( -- * Primary + getStackage, + + -- * Misc + parseCabalConfig, + ) +where + +import CLC.Stackage.Parser.API.Common + ( ExceptionReason + ( ReasonDecodeUtf8, + ReasonReadBody, + ReasonStatus + ), + StackageException (MkStackageException), + StackageResponse (MkStackageResponse), + getStatusCode, + ) +import CLC.Stackage.Utils.Exception qualified as Ex +import CLC.Stackage.Utils.Package qualified as Package +import Control.Exception (throwIO) +import Control.Monad (when) +import Data.Maybe (catMaybes) +import Data.Text (Text) +import Data.Text qualified as T +import Data.Text.Encoding qualified as TEnc +import Network.HTTP.Client (BodyReader, Manager, Request, Response) +import Network.HTTP.Client qualified as HttpClient + +-- | Given http manager and snapshot string, queries the cabal config +-- endpoint. This is intended as a backup, for when the primary endpoint fails. +getStackage :: Manager -> String -> IO StackageResponse +getStackage manager stackageSnapshot = do + req <- getRequest + HttpClient.withResponse req manager readStackageResponse + where + readStackageResponse :: Response BodyReader -> IO StackageResponse + readStackageResponse res = do + let bodyReader = HttpClient.responseBody res + status = HttpClient.responseStatus res + statusCode = getStatusCode res + mkEx = MkStackageException stackageSnapshot + + when (statusCode /= 200) $ + throwIO $ + mkEx (ReasonStatus status) + + bodyBs <- + Ex.mapThrowLeft + (mkEx . ReasonReadBody) + =<< Ex.tryAny (mconcat <$> HttpClient.brConsume bodyReader) + + bodyTxt <- + Ex.mapThrowLeft + (mkEx . ReasonDecodeUtf8 bodyBs) + $ TEnc.decodeUtf8' bodyBs + + pure $ parseCabalConfig bodyTxt + + getRequest :: IO Request + getRequest = HttpClient.parseRequest stackageUrl + + -- Url for the stackage snapshot. + stackageUrl :: String + stackageUrl = + "https://stackage.org/" + <> stackageSnapshot + <> "/cabal.config" + +parseCabalConfig :: Text -> StackageResponse +parseCabalConfig = + MkStackageResponse + . catMaybes + . fmap parseCabalConfigLine + . T.lines + +-- | Parses a line like ' =='. +parseCabalConfigLine :: Text -> Maybe Package.Package +parseCabalConfigLine txt = do + -- Strip leading 'constraints:' keyword, if it exists. + let s = case T.stripPrefix "constraints:" txt' of + Nothing -> txt' + Just rest -> T.stripStart rest + Package.fromCabalConstraintsText s + where + txt' = T.stripStart txt diff --git a/src/parser/CLC/Stackage/Parser/Query.hs b/src/parser/CLC/Stackage/Parser/API/Common.hs similarity index 64% rename from src/parser/CLC/Stackage/Parser/Query.hs rename to src/parser/CLC/Stackage/Parser/API/Common.hs index 7343caa..fe2b5c9 100644 --- a/src/parser/CLC/Stackage/Parser/Query.hs +++ b/src/parser/CLC/Stackage/Parser/API/Common.hs @@ -1,55 +1,37 @@ -{-# LANGUAGE CPP #-} -{-# LANGUAGE QuasiQuotes #-} +-- | Types and functions common to stackage JSON and CabalConfig APIs. +module CLC.Stackage.Parser.API.Common + ( -- * Types + StackageResponse (..), -module CLC.Stackage.Parser.Query - ( -- * Querying stackage - getStackage, - - -- ** Exceptions + -- * Exception StackageException (..), ExceptionReason (..), + + -- * Misc + getStatusCode, ) where -import CLC.Stackage.Parser.API - ( stackageSnapshot, - withResponse, - ) -import CLC.Stackage.Parser.Data.Response (StackageResponse) -import CLC.Stackage.Utils.Exception qualified as Ex -import CLC.Stackage.Utils.JSON qualified as JSON +import CLC.Stackage.Utils.Package (Package) +import Control.DeepSeq (NFData) import Control.Exception ( Exception (displayException), SomeException, - throwIO, ) -import Control.Monad (when) import Data.ByteString (ByteString) +import Data.Text.Encoding.Error (UnicodeException) +import GHC.Generics (Generic) import Network.HTTP.Client (Response) import Network.HTTP.Client qualified as HttpClient import Network.HTTP.Types.Status (Status) import Network.HTTP.Types.Status qualified as Status --- | Returns the 'StackageResponse' corresponding to the given snapshot. -getStackage :: IO StackageResponse -getStackage = withResponse $ \res -> do - let bodyReader = HttpClient.responseBody res - status = HttpClient.responseStatus res - statusCode = getStatusCode res - mkEx = MkStackageException stackageSnapshot - - when (statusCode /= 200) $ - throwIO $ - mkEx (ReasonStatus status) - - bodyBs <- - Ex.mapThrowLeft - (mkEx . ReasonReadBody) - =<< Ex.tryAny (mconcat <$> HttpClient.brConsume bodyReader) - - Ex.mapThrowLeft - (mkEx . ReasonDecodeJson bodyBs) - (JSON.decode bodyBs) +-- | Stackage response. This type unifies different stackage responses. +newtype StackageResponse = MkStackageResponse + { packages :: [Package] + } + deriving stock (Eq, Generic, Show) + deriving anyclass (NFData) -- | Exception reason. data ExceptionReason @@ -60,6 +42,9 @@ data ExceptionReason | -- | Exception decoding JSON. The first string is the json we attempted -- to decode. The second is the error message. ReasonDecodeJson ByteString String + | -- | Exception decoding JSON. The first string is the bytestring we + -- attempted to decode. The second is the error message. + ReasonDecodeUtf8 ByteString UnicodeException deriving stock (Show) -- | General network exception. @@ -99,10 +84,17 @@ instance Exception StackageException where ] ReasonDecodeJson jsonBs err -> mconcat - [ "Could not decode JSON:\n\n", - show jsonBs, - "\n\nError: ", - err + [ "Could not decode JSON: ", + err, + "This is likely due to the endpoint returning HTML, not JSON. Bytes: ", + show jsonBs + ] + ReasonDecodeUtf8 bs err -> + mconcat + [ "Could not decode UTF-8: ", + displayException err, + ". Bytes: ", + show bs ] where snapshot = ex.snapshot diff --git a/src/parser/CLC/Stackage/Parser/API/JSON.hs b/src/parser/CLC/Stackage/Parser/API/JSON.hs new file mode 100644 index 0000000..9ee4a2d --- /dev/null +++ b/src/parser/CLC/Stackage/Parser/API/JSON.hs @@ -0,0 +1,106 @@ +module CLC.Stackage.Parser.API.JSON + ( -- * Query json endpoint + getStackage, + ) +where + +import CLC.Stackage.Parser.API.Common + ( ExceptionReason (ReasonDecodeJson, ReasonReadBody, ReasonStatus), + StackageException (MkStackageException), + getStatusCode, + ) +import CLC.Stackage.Parser.API.Common qualified as Common +import CLC.Stackage.Utils.Exception qualified as Ex +import CLC.Stackage.Utils.JSON qualified as JSON +import CLC.Stackage.Utils.Package qualified as Package +import Control.Exception (throwIO) +import Control.Monad (when) +import Data.Aeson (FromJSON, ToJSON) +import Data.Text (Text) +import GHC.Generics (Generic) +import Network.HTTP.Client (BodyReader, Manager, Request, Response) +import Network.HTTP.Client qualified as HttpClient + +-- | Given http manager and snapshot string, queries the primary json +-- endpoint. +getStackage :: Manager -> String -> IO Common.StackageResponse +getStackage manager stackageSnapshot = do + req <- getRequest + HttpClient.withResponse req manager readStackageResponse + where + readStackageResponse :: Response BodyReader -> IO Common.StackageResponse + readStackageResponse res = do + let bodyReader = HttpClient.responseBody res + status = HttpClient.responseStatus res + statusCode = getStatusCode res + mkEx = MkStackageException stackageSnapshot + + when (statusCode /= 200) $ + throwIO $ + mkEx (ReasonStatus status) + + bodyBs <- + Ex.mapThrowLeft + (mkEx . ReasonReadBody) + =<< Ex.tryAny (mconcat <$> HttpClient.brConsume bodyReader) + + Ex.mapThrowLeft + (mkEx . ReasonDecodeJson bodyBs) + (toSnapshotCommon <$> JSON.decode bodyBs) + + getRequest :: IO Request + getRequest = updateReq <$> mkReq + where + mkReq = HttpClient.parseRequest stackageUrl + updateReq r = + r + { HttpClient.requestHeaders = + [ ("Accept", "application/json;charset=utf-8,application/json") + ] + } + + -- Url for the stackage snapshot. + stackageUrl :: String + stackageUrl = "https://stackage.org/" <> stackageSnapshot + +toSnapshotCommon :: StackageResponse -> Common.StackageResponse +toSnapshotCommon (MkStackageResponse _ pkgs) = + Common.MkStackageResponse + { packages = toPackageCommon <$> pkgs + } + +toPackageCommon :: PackageResponse -> Package.Package +toPackageCommon pr = + Package.MkPackage + { name = pr.name, + version = Package.PackageVersionText pr.version + } + +-- | Response returned by primary stackage endpoint e.g. +-- @stackage.org\/lts-20.14@. +data StackageResponse = MkStackageResponse + { snapshot :: SnapshotResponse, + packages :: [PackageResponse] + } + deriving stock (Eq, Generic, Show) + deriving anyclass (FromJSON, ToJSON) + +-- | Stackage snapshot data. +data SnapshotResponse = MkSnapshotResponse + { ghc :: Text, + created :: Text, + name :: Text, + compiler :: Text + } + deriving stock (Eq, Generic, Show) + deriving anyclass (FromJSON, ToJSON) + +-- | Package in a stackage snapshot. +data PackageResponse = MkPackageResponse + { origin :: Text, + name :: Text, + version :: Text, + synopsis :: Text + } + deriving stock (Eq, Generic, Show) + deriving anyclass (FromJSON, ToJSON) diff --git a/src/parser/CLC/Stackage/Parser/Data/Response.hs b/src/parser/CLC/Stackage/Parser/Data/Response.hs deleted file mode 100644 index f270e44..0000000 --- a/src/parser/CLC/Stackage/Parser/Data/Response.hs +++ /dev/null @@ -1,40 +0,0 @@ --- | Types returned by stackage API. -module CLC.Stackage.Parser.Data.Response - ( StackageResponse (..), - SnapshotResponse (..), - PackageResponse (..), - ) -where - -import Data.Aeson (FromJSON, ToJSON) -import Data.Text (Text) -import GHC.Generics (Generic) - --- | Response returned by primary stackage endpoint e.g. --- @stackage.org\/lts-20.14@. -data StackageResponse = MkStackageResponse - { snapshot :: SnapshotResponse, - packages :: [PackageResponse] - } - deriving stock (Eq, Generic, Show) - deriving anyclass (FromJSON, ToJSON) - --- | Stackage snapshot data. -data SnapshotResponse = MkSnapshotResponse - { ghc :: Text, - created :: Text, - name :: Text, - compiler :: Text - } - deriving stock (Eq, Generic, Show) - deriving anyclass (FromJSON, ToJSON) - --- | Package in a stackage snapshot. -data PackageResponse = MkPackageResponse - { origin :: Text, - name :: Text, - version :: Text, - synopsis :: Text - } - deriving stock (Eq, Generic, Show) - deriving anyclass (FromJSON, ToJSON) diff --git a/src/runner/CLC/Stackage/Runner.hs b/src/runner/CLC/Stackage/Runner.hs index c7d1eb4..c2bcfb2 100644 --- a/src/runner/CLC/Stackage/Runner.hs +++ b/src/runner/CLC/Stackage/Runner.hs @@ -11,11 +11,11 @@ import CLC.Stackage.Builder.Env ( BuildEnv (progress), Progress (failuresRef), ) -import CLC.Stackage.Builder.Package (Package) import CLC.Stackage.Builder.Writer qualified as Writer import CLC.Stackage.Runner.Env (RunnerEnv (completePackageSet)) import CLC.Stackage.Runner.Env qualified as Env import CLC.Stackage.Utils.Logging qualified as Logging +import CLC.Stackage.Utils.Package (Package) import Control.Exception (bracket, throwIO) import Control.Monad (when) import Data.Foldable (for_) diff --git a/src/runner/CLC/Stackage/Runner/Args.hs b/src/runner/CLC/Stackage/Runner/Args.hs index 4c0cf81..9652b04 100644 --- a/src/runner/CLC/Stackage/Runner/Args.hs +++ b/src/runner/CLC/Stackage/Runner/Args.hs @@ -6,15 +6,9 @@ module CLC.Stackage.Runner.Args where import CLC.Stackage.Builder.Env - ( CabalVerbosity - ( CabalVerbosity0, - CabalVerbosity1, - CabalVerbosity2, - CabalVerbosity3 - ), - Jobs (JobsN, JobsNCpus, JobsSemaphore), - WriteLogs (WriteLogsCurrent, WriteLogsNone, WriteLogsSaveFailures), + ( WriteLogs (WriteLogsCurrent, WriteLogsNone, WriteLogsSaveFailures), ) +import Data.String qualified as Str import Options.Applicative ( Mod, Parser, @@ -31,11 +25,13 @@ import Options.Applicative (<**>), ) import Options.Applicative qualified as OA +import Options.Applicative.Help (Doc) import Options.Applicative.Help.Chunk (Chunk (Chunk)) import Options.Applicative.Help.Chunk qualified as Chunk import Options.Applicative.Help.Pretty qualified as Pretty -import Options.Applicative.Types (ArgPolicy (Intersperse)) -import Text.Read qualified as TR +import Options.Applicative.Types (ArgPolicy (Intersperse), ReadM) +import System.OsPath (OsPath) +import System.OsPath qualified as OsP -- | Log coloring option. data ColorLogs @@ -49,16 +45,18 @@ data Args = MkArgs { -- | If given, batches packages together so we build more than one. -- Defaults to batching everything together in the same group. batch :: Maybe Int, - -- | Cabal's --verbosity flag. - cabalVerbosity :: Maybe CabalVerbosity, + -- | Global options to pass to cabal e.g. --store-dir. + cabalGlobalOpts :: [String], + -- | Options to pass to cabal e.g. --semaphore. + cabalOpts :: [String], + -- | Optional path to cabal executable. + cabalPath :: Maybe OsPath, -- | Determines if we color the logs. If 'Nothing', attempts to detect -- if colors are supported. colorLogs :: ColorLogs, -- | If true, the first group that fails to completely build stops -- clc-stackage. groupFailFast :: Bool, - -- | Number of build jobs. - jobs :: Maybe Jobs, -- | Disables the cache, which otherwise saves the outcome of a run in a -- json file. The cache is used for resuming a run that was interrupted. noCache :: Bool, @@ -69,6 +67,9 @@ data Args = MkArgs packageFailFast :: Bool, -- | Whether to retry packages that failed. retryFailures :: Bool, + -- | Optional path to snapshot file. If given, we use the file's contents + -- as the package set, rather than the stackage server. + snapshotPath :: Maybe OsPath, -- | Determines what logs to write. writeLogs :: Maybe WriteLogs } @@ -112,34 +113,59 @@ getArgs = OA.execParser parserInfoArgs mconcat [ "This will build everything in one package group, and pass ", "--keep-going to cabal." - ] + ], + Chunk.paragraph "Examples:", + mkExample + [ "# Basic example", + "$ clc-stackage" + ], + mkExample + [ "# Batch with groups of 100 and some cabal options", + "$ clc-stackage --batch 100 --cabal-options='--semaphore --verbose=1'" + ], + mkExample + [ "# Run with custom cabal", + "$ clc-stackage --cabal-path=path/to/cabal --cabal-global-options='--store-dir=path/to/store'" + ], + mkExample + [ "# Run with custom snapshot", + "$ clc-stackage --snapshot-path=path/to/snapshot-file" + ] ] + mkExample :: [String] -> Chunk Doc + mkExample = + Chunk.vcatChunks + . fmap (fmap (Pretty.indent 2) . Chunk.stringChunk) parseCliArgs :: Parser Args parseCliArgs = ( do batch <- parseBatch - cabalVerbosity <- parseCabalVerbosity + cabalGlobalOpts <- parseCabalGlobalOpts + cabalOpts <- parseCabalOpts + cabalPath <- parseCabalPath colorLogs <- parseColorLogs groupFailFast <- parseGroupFailFast - jobs <- parseJobs noCache <- parseNoCache noCleanup <- parseNoCleanup packageFailFast <- parsePackageFailFast retryFailures <- parseRetryFailures + snapshotPath <- parseSnapshotPath writeLogs <- parseWriteLogs pure $ MkArgs { batch, - cabalVerbosity, + cabalGlobalOpts, + cabalOpts, + cabalPath, colorLogs, groupFailFast, - jobs, noCache, noCleanup, packageFailFast, retryFailures, + snapshotPath, writeLogs } ) @@ -164,27 +190,49 @@ parseBatch = ] ) -parseCabalVerbosity :: Parser (Maybe CabalVerbosity) -parseCabalVerbosity = +parseCabalGlobalOpts :: Parser [String] +parseCabalGlobalOpts = + OA.option + readOpts + ( mconcat + [ OA.long "cabal-global-options", + OA.metavar "ARGS...", + OA.value [], + mkHelp $ + mconcat + [ "Global arguments to pass to cabal e.g. '--store-dir=path/to/store'. ", + "These precede the build command." + ] + ] + ) + where + readOpts = Str.words <$> OA.str + +parseCabalOpts :: Parser [String] +parseCabalOpts = + OA.option + readOpts + ( mconcat + [ OA.long "cabal-options", + OA.metavar "ARGS...", + OA.value [], + mkHelp "Quoted arguments to pass to cabal e.g. '--semaphore --verbose=1'." + ] + ) + where + readOpts = Str.words <$> OA.str + +parseCabalPath :: Parser (Maybe OsPath) +parseCabalPath = OA.optional $ OA.option - readCabalVerbosity + readOsPath ( mconcat - [ OA.long "cabal-verbosity", - OA.metavar "(0 | 1 | 2 | 3)", - mkHelp - "Cabal's --verbose flag. Uses cabal's default if not given (1)." + [ OA.long "cabal-path", + OA.metavar "PATH", + mkHelp "Optional path to cabal executable." ] ) - where - readCabalVerbosity = - OA.str >>= \case - "0" -> pure CabalVerbosity0 - "1" -> pure CabalVerbosity1 - "2" -> pure CabalVerbosity2 - "3" -> pure CabalVerbosity3 - other -> - fail $ "Expected one of (0 | 1 | 2 | 3), received: " ++ other parseColorLogs :: Parser ColorLogs parseColorLogs = @@ -220,44 +268,6 @@ parseGroupFailFast = "clc-stackage." ] -parseJobs :: Parser (Maybe Jobs) -parseJobs = - OA.optional $ - OA.option - readJobs - ( mconcat - [ OA.short 'j', - OA.long "jobs", - OA.metavar "(NAT | $ncpus | semaphore)", - mkHelp $ - mconcat - [ "Controls the number of build jobs i.e. the flag passed to ", - "cabal's --jobs option. Can be a natural number in [1, 255] ", - "or the literal string '$ncpus', meaning all cpus. The ", - "literal 'semaphore' will instead use cabal's --semaphore ", - "option. This requires GHC 9.8+ and Cabal 3.12+. No option ", - "uses cabal's default i.e. $ncpus." - ] - ] - ) - where - readJobs = - OA.str >>= \case - "$ncpus" -> pure JobsNCpus - "semaphore" -> pure JobsSemaphore - other -> case TR.readMaybe @Int other of - Just n -> - if n > 0 && n < 256 - then pure $ JobsN $ fromIntegral n - else fail $ "Expected NAT in [1, 255], received: " ++ other - Nothing -> - fail $ - mconcat - [ "Expected one of (NAT in [1, 255] | $ncpus | semaphore), ", - "received: ", - other - ] - parseNoCache :: Parser Bool parseNoCache = OA.switch @@ -308,6 +318,28 @@ parseRetryFailures = ] ) +parseSnapshotPath :: Parser (Maybe OsPath) +parseSnapshotPath = + OA.optional $ + OA.option + readOsPath + ( mconcat + [ OA.long "snapshot-path", + OA.metavar "PATH", + mkHelp $ + mconcat + [ "Optional path to snapshot file. If given, this overrides ", + "the stackage snapshot; that is, we use the file's contents, ", + "rather than the stackage server. The file should be ", + "formatted similar to ", + "https://www.stackage.org//cabal.config i.e. each ", + "line should be ' ==' e.g. 'lens ==5.3.4'. Note ", + "that the snapshot is still filtered according to ", + "excluded_pkgs.json." + ] + ] + ) + parseWriteLogs :: Parser (Maybe WriteLogs) parseWriteLogs = OA.optional $ @@ -340,6 +372,13 @@ parseWriteLogs = other ] +readOsPath :: ReadM OsPath +readOsPath = do + fp <- OA.str + case OsP.encodeUtf fp of + Just osp -> pure osp + Nothing -> fail $ "Failed encoding to ospath: " ++ fp + mkHelp :: String -> Mod f a mkHelp = OA.helpDoc diff --git a/src/runner/CLC/Stackage/Runner/Env.hs b/src/runner/CLC/Stackage/Runner/Env.hs index 2a52541..2bcd26b 100644 --- a/src/runner/CLC/Stackage/Runner/Env.hs +++ b/src/runner/CLC/Stackage/Runner/Env.hs @@ -1,3 +1,5 @@ +{-# LANGUAGE QuasiQuotes #-} + module CLC.Stackage.Runner.Env ( RunnerEnv (..), setup, @@ -13,7 +15,6 @@ import CLC.Stackage.Builder.Env ( BuildEnv ( MkBuildEnv, batch, - colorLogs, groupFailFast, hLogger, packagesToBuild, @@ -27,11 +28,10 @@ import CLC.Stackage.Builder.Env ), ) import CLC.Stackage.Builder.Env qualified as Builder.Env -import CLC.Stackage.Builder.Package (Package (MkPackage, name, version)) import CLC.Stackage.Parser qualified as Parser -import CLC.Stackage.Parser.Data.Response (PackageResponse (name, version)) import CLC.Stackage.Runner.Args - ( ColorLogs + ( Args (snapshotPath), + ColorLogs ( ColorLogsDetect, ColorLogsOff, ColorLogsOn @@ -40,16 +40,18 @@ import CLC.Stackage.Runner.Args import CLC.Stackage.Runner.Args qualified as Args import CLC.Stackage.Runner.Report (Results (MkResults)) import CLC.Stackage.Runner.Report qualified as Report +import CLC.Stackage.Utils.Exception qualified as Ex import CLC.Stackage.Utils.IO qualified as IO import CLC.Stackage.Utils.Logging qualified as Logging +import CLC.Stackage.Utils.Package (Package (MkPackage, name, version)) import CLC.Stackage.Utils.Paths qualified as Paths import Control.Exception (throwIO) -import Control.Monad (unless) +import Control.Monad (join, unless) import Data.Bool (Bool (False, True), not) import Data.Foldable (Foldable (foldl')) import Data.IORef (newIORef, readIORef) import Data.List.NonEmpty (NonEmpty ((:|))) -import Data.Maybe (Maybe (Just, Nothing), fromMaybe, maybe) +import Data.Maybe (Maybe (Just, Nothing), fromMaybe) import Data.Set (Set) import Data.Set qualified as Set import Data.Text qualified as T @@ -57,7 +59,9 @@ import Data.Time (LocalTime) import System.Console.Pretty (supportsPretty) import System.Directory.OsPath qualified as Dir import System.Exit (ExitCode (ExitSuccess)) -import Prelude (IO, mconcat, pure, show, ($), (++), (.), (<$>), (<>)) +import System.OsPath (osp) +import System.OsPath qualified as OsP +import Prelude (IO, Monad ((>>=)), mconcat, pure, show, ($), (.), (<$>), (<>)) -- | Args used for building all packages. data RunnerEnv = MkRunnerEnv @@ -84,43 +88,56 @@ data RunnerEnv = MkRunnerEnv -- | Creates an environment based on cli args and cache data. The parameter -- modifies the package set returned by stackage. setup :: Logging.Handle -> ([Package] -> [Package]) -> IO RunnerEnv -setup hLogger modifyPackages = do - startTime <- hLogger.getLocalTime +setup hLoggerRaw modifyPackages = do + startTime <- hLoggerRaw.getLocalTime cliArgs <- Args.getArgs + colorLogs <- + case cliArgs.colorLogs of + ColorLogsOff -> pure False + ColorLogsOn -> pure True + ColorLogsDetect -> supportsPretty + + -- Update logger with CLI color param. + let hLogger = hLoggerRaw {Logging.color = colorLogs} + -- Set up build args for cabal, filling in missing defaults let buildArgs = - ["build"] - ++ cabalVerboseArg - ++ jobsArg - ++ keepGoingArg - - cabalVerboseArg = toArgs Builder.Env.cabalVerbosityToArg cliArgs.cabalVerbosity - jobsArg = toArgs Builder.Env.jobsToArg cliArgs.jobs + join + [ cliArgs.cabalGlobalOpts, + ["build"], + keepGoingArg, + cliArgs.cabalOpts + ] -- when packageFailFast is false, add keep-going so that we build as many -- packages in the group. keepGoingArg = ["--keep-going" | not cliArgs.packageFailFast] + cabalPathRaw <- case cliArgs.cabalPath of + Nothing -> pure [osp|cabal|] + Just p -> Paths.canonicalizePath p + + cabalPath <- + Dir.findExecutable cabalPathRaw >>= \case + -- TODO: It would be nice to avoid the decode here and keep everything + -- in OsPath, though that is blocked until process support OsPath. + Just p -> OsP.decodeUtf p + Nothing -> Ex.throwText $ "Cabal not found: " <> T.pack (show cabalPathRaw) + successesRef <- newIORef Set.empty failuresRef <- newIORef Set.empty - colorLogs <- - case cliArgs.colorLogs of - ColorLogsOff -> pure False - ColorLogsOn -> pure True - ColorLogsDetect -> supportsPretty - cache <- if cliArgs.noCache then pure Nothing - else Report.readCache hLogger colorLogs + else Report.readCache hLogger -- (entire set, packages to build) (completePackageSet, pkgsList) <- case cache of Nothing -> do -- if no cache exists, query stackage - pkgsResponses <- Parser.getPackageList + pkgsResponses <- Parser.getPackageList hLogger cliArgs.snapshotPath let completePackageSet = responseToPkgs <$> pkgsResponses pkgs = modifyPackages completePackageSet pure (completePackageSet, pkgs) @@ -145,7 +162,7 @@ setup hLogger modifyPackages = do packagesToBuild <- case pkgsList of (p : ps) -> pure (p :| ps) [] -> do - Logging.putTimeInfoStr hLogger colorLogs "Cache exists but has no packages to test." + Logging.putTimeInfoStr hLogger "Cache exists but has no packages to test." throwIO ExitSuccess let progress = @@ -158,7 +175,7 @@ setup hLogger modifyPackages = do MkBuildEnv { batch = cliArgs.batch, buildArgs, - colorLogs, + cabalPath, groupFailFast = cliArgs.groupFailFast, hLogger, packagesToBuild, @@ -187,9 +204,6 @@ setup hLogger modifyPackages = do version = p.version } - toArgs :: (a -> b) -> Maybe a -> [b] - toArgs f = maybe [] (\x -> [f x]) - -- | Prints summary and writes results to disk. teardown :: RunnerEnv -> IO () teardown env = do @@ -209,16 +223,17 @@ teardown env = do Report.saveReport report + let colorLogs = env.buildEnv.hLogger.color env.buildEnv.hLogger.logStrLn $ T.unlines [ "", "", - Logging.colorGreen env.buildEnv.colorLogs $ "- Successes: " <> successStr report, - Logging.colorRed env.buildEnv.colorLogs $ "- Failures: " <> failureStr report, - Logging.colorMagenta env.buildEnv.colorLogs $ "- Untested: " <> untestedStr report, + Logging.colorGreen colorLogs $ "- Successes: " <> successStr report, + Logging.colorRed colorLogs $ "- Failures: " <> failureStr report, + Logging.colorMagenta colorLogs $ "- Untested: " <> untestedStr report, "", - Logging.colorBlue env.buildEnv.colorLogs $ "- Start: " <> report.startTime, - Logging.colorBlue env.buildEnv.colorLogs $ "- End: " <> report.endTime + Logging.colorBlue colorLogs $ "- Start: " <> report.startTime, + Logging.colorBlue colorLogs $ "- End: " <> report.endTime ] where successStr r = fmtPercent r.stats.numSuccesses r.stats.successRate diff --git a/src/runner/CLC/Stackage/Runner/Report.hs b/src/runner/CLC/Stackage/Runner/Report.hs index 9d7aaed..39f00bc 100644 --- a/src/runner/CLC/Stackage/Runner/Report.hs +++ b/src/runner/CLC/Stackage/Runner/Report.hs @@ -16,10 +16,10 @@ module CLC.Stackage.Runner.Report ) where -import CLC.Stackage.Builder.Package (Package) import CLC.Stackage.Utils.IO qualified as IO import CLC.Stackage.Utils.JSON qualified as JSON import CLC.Stackage.Utils.Logging qualified as Logging +import CLC.Stackage.Utils.Package (Package) import CLC.Stackage.Utils.Paths qualified as Paths import Control.Exception (throwIO) import Data.Aeson (AesonException (AesonException), FromJSON, ToJSON) @@ -104,19 +104,19 @@ mkReport results startTime endTime = dv n = floor $ 100 * (fromIntegral n / numAllTested) -- | Reads results data, if the cache exists. -readCache :: Logging.Handle -> Bool -> IO (Maybe Results) -readCache handle colorLogs = do +readCache :: Logging.Handle -> IO (Maybe Results) +readCache handle = do catchPathStr <- T.pack <$> OsPath.decodeUtf Paths.cachePath Dir.doesFileExist Paths.cachePath >>= \case False -> do - Logging.putTimeInfoStr handle colorLogs $ "Cached results do not exist: " <> catchPathStr + Logging.putTimeInfoStr handle $ "Cached results do not exist: " <> catchPathStr pure Nothing True -> do contents <- IO.readBinaryFile Paths.cachePath case JSON.decode contents of Left err -> throwIO $ AesonException err Right r -> do - Logging.putTimeInfoStr handle colorLogs $ "Using cached results: " <> catchPathStr + Logging.putTimeInfoStr handle $ "Using cached results: " <> catchPathStr pure $ Just r -- | Saves the current progress data as the next prior run. diff --git a/src/utils/CLC/Stackage/Utils/Exception.hs b/src/utils/CLC/Stackage/Utils/Exception.hs index 2db6c4f..fb9c05e 100644 --- a/src/utils/CLC/Stackage/Utils/Exception.hs +++ b/src/utils/CLC/Stackage/Utils/Exception.hs @@ -1,9 +1,14 @@ -- | Provides utils for exceptions. module CLC.Stackage.Utils.Exception - ( try, + ( -- * Utilities + try, tryAny, throwLeft, mapThrowLeft, + + -- * Types + TextException (..), + throwText, ) where @@ -15,6 +20,8 @@ import Control.Exception ) import Control.Exception qualified as Ex import Data.Bifunctor (first) +import Data.Text (Text) +import Data.Text qualified as T mapThrowLeft :: (Exception e2) => (e1 -> e2) -> Either e1 a -> IO a mapThrowLeft f = throwLeft . first f @@ -42,3 +49,12 @@ isSyncException e = case fromException (toException e) of Just (SomeAsyncException _) -> False Nothing -> True + +newtype TextException = MkTextException Text + deriving stock (Eq, Show) + +instance Exception TextException where + displayException (MkTextException t) = T.unpack t + +throwText :: Text -> IO a +throwText = throwIO . MkTextException diff --git a/src/utils/CLC/Stackage/Utils/Logging.hs b/src/utils/CLC/Stackage/Utils/Logging.hs index 1b8a42f..217d93a 100644 --- a/src/utils/CLC/Stackage/Utils/Logging.hs +++ b/src/utils/CLC/Stackage/Utils/Logging.hs @@ -1,10 +1,12 @@ module CLC.Stackage.Utils.Logging ( -- * Logging Handler Handle (..), + mkDefaultLogger, -- * Printing with timestamps putTimeInfoStr, putTimeSuccessStr, + putTimeWarnStr, putTimeErrStr, -- ** ANSI Colors @@ -22,13 +24,17 @@ import Data.Text (Text) import Data.Text qualified as T import Data.Time.Format qualified as Format import Data.Time.LocalTime (LocalTime) +import Data.Time.LocalTime qualified as Local import Data.Word (Word16) -import System.Console.Pretty (Color (Blue, Green, Magenta, Red)) +import System.Console.Pretty (Color (Blue, Green, Magenta, Red, Yellow)) import System.Console.Pretty qualified as Pretty +import System.IO (hPutStrLn, stderr) -- | Simple handle for logging, for testing output. data Handle = MkHandle - { -- | Retrieve local time. + { -- | If true, colors the logs. + color :: Bool, + -- | Retrieve local time. getLocalTime :: IO LocalTime, -- | Log stderr. logStrErrLn :: Text -> IO (), @@ -39,26 +45,37 @@ data Handle = MkHandle } -- | 'putStrLn' with a timestamp and info prefix. -putTimeInfoStr :: Handle -> Bool -> Text -> IO () -putTimeInfoStr hLogger b s = do +putTimeInfoStr :: Handle -> Text -> IO () +putTimeInfoStr hLogger s = do timeStr <- getLocalTimeString hLogger - hLogger.logStrLn $ colorBlue b $ "[" <> timeStr <> "][Info] " <> s' + hLogger.logStrLn $ colorBlue hLogger.color $ "[" <> timeStr <> "][Info] " <> s' where s' = truncateIfNeeded hLogger.terminalWidth s -- | 'putStrLn' with a timestamp and info prefix. -putTimeSuccessStr :: Handle -> Bool -> Text -> IO () -putTimeSuccessStr hLogger b s = do +putTimeSuccessStr :: Handle -> Text -> IO () +putTimeSuccessStr hLogger s = do timeStr <- getLocalTimeString hLogger - hLogger.logStrLn $ colorGreen b $ "[" <> timeStr <> "][Success] " <> s' + hLogger.logStrLn $ colorGreen hLogger.color $ "[" <> timeStr <> "][Success] " <> s' where s' = truncateIfNeeded hLogger.terminalWidth s +-- | 'putStrLn' with a timestamp and warn prefix. +putTimeWarnStr :: Handle -> Text -> IO () +putTimeWarnStr hLogger s = do + timeStr <- getLocalTimeString hLogger + hLogger.logStrErrLn $ colorYellow hLogger.color $ "[" <> timeStr <> "][Warn] " <> s' + where + -- Allow this to be longer than the terminal width, since this is + -- generally used as a one-off message, and the warning may include useful + -- error info. + s' = truncateIfNeeded 200 s + -- | 'putStrErrLn' with a timestamp and error prefix. -putTimeErrStr :: Handle -> Bool -> Text -> IO () -putTimeErrStr hLogger b s = do +putTimeErrStr :: Handle -> Text -> IO () +putTimeErrStr hLogger s = do timeStr <- getLocalTimeString hLogger - hLogger.logStrErrLn $ colorRed b $ "[" <> timeStr <> "][Error] " <> s' + hLogger.logStrErrLn $ colorRed hLogger.color $ "[" <> timeStr <> "][Error] " <> s' where s' = truncateIfNeeded hLogger.terminalWidth s @@ -82,6 +99,9 @@ colorGreen b = colorIf b Green colorRed :: Bool -> Text -> Text colorRed b = colorIf b Red +colorYellow :: Bool -> Text -> Text +colorYellow b = colorIf b Yellow + colorIf :: Bool -> Color -> Text -> Text colorIf True = Pretty.color colorIf False = const id @@ -119,3 +139,13 @@ constLen = 11 -- e.g. [2024-10-14 15:14:00] timeStrLen :: Word16 timeStrLen = 21 + +mkDefaultLogger :: Handle +mkDefaultLogger = + MkHandle + { color = False, + getLocalTime = Local.zonedTimeToLocalTime <$> Local.getZonedTime, + logStrErrLn = hPutStrLn stderr . T.unpack, + logStrLn = putStrLn . T.unpack, + terminalWidth = 80 + } diff --git a/src/utils/CLC/Stackage/Utils/Package.hs b/src/utils/CLC/Stackage/Utils/Package.hs new file mode 100644 index 0000000..7e91e08 --- /dev/null +++ b/src/utils/CLC/Stackage/Utils/Package.hs @@ -0,0 +1,177 @@ +{-# LANGUAGE ViewPatterns #-} + +-- | Provides the type representing a package with version. +module CLC.Stackage.Utils.Package + ( -- * Package + Package (..), + + -- ** Creation + fromCabalConstraintsText, + + -- ** Elimination + toCabalDepText, + toCabalConstraintsText, + toDirName, + toDisplayName, + + -- * Version + PackageVersion (..), + ) +where + +import CLC.Stackage.Utils.Paths qualified as Paths +import Control.Applicative (Alternative ((<|>))) +import Control.DeepSeq (NFData) +import Data.Aeson (FromJSON (parseJSON), ToJSON (toJSON)) +import Data.Aeson qualified as Asn +import Data.Char qualified as Ch +import Data.String (IsString (fromString)) +import Data.Text (Text) +import Data.Text qualified as T +import GHC.Generics (Generic) +import System.OsPath (OsPath) + +-- | Wrapper for package version. +data PackageVersion + = -- | Basic version text e.g. "2.3". + PackageVersionText Text + | -- | Represents an installed lib e.g. "foo installed" from cabal + -- constraints. This is included in the from/toJSON instances, for + -- writing/reading the report. + -- + -- Generally speaking, this is only used when clc-stackage falls back + -- to the cabal.config endpoint, or is used with an explicit + -- --snapshot-path argument. + PackageVersionInstalled + deriving stock (Eq, Generic, Ord, Show) + deriving anyclass (NFData) + +instance FromJSON PackageVersion where + parseJSON = Asn.withText "PackageVersion" $ \case + "installed" -> pure PackageVersionInstalled + other -> pure $ PackageVersionText other + +instance ToJSON PackageVersion where + toJSON (PackageVersionText t) = toJSON t + toJSON PackageVersionInstalled = "installed" + +-- | Package data. +data Package = MkPackage + { name :: Text, + version :: PackageVersion + } + deriving stock (Eq, Generic, Ord, Show) + deriving anyclass (FromJSON, NFData, ToJSON) + +instance IsString Package where + fromString s = case fromCabalConstraintsText (T.pack s) of + Nothing -> + error $ + mconcat + [ "String '", + s, + "' did no match expected package format: ==" + ] + Just p -> p + +-- | Text representation suitable for cabal file build-depends. +toCabalDepText :: Package -> Text +toCabalDepText = (", " <>) . toTextNoInstalled + +-- | Text representation suitable for cabal file constraints. +toCabalConstraintsText :: Package -> Text +toCabalConstraintsText = (<> ",") . toTextInstalled + +-- | Returns an OsPath name based on this package i.e. the OsPath +-- representation of 'toText'. Used when naming an error directory for a +-- package that fails. +toDirName :: Package -> IO OsPath +toDirName = Paths.encodeUtf . T.unpack . toDisplayName + +-- | Slightly nicer display name e.g. "mtl-installed", "aeson-1.2.3". +toDisplayName :: Package -> Text +toDisplayName (MkPackage name vers) = txt + where + txt = name <> "-" <> v + v = case vers of + PackageVersionText t -> t + PackageVersionInstalled -> "installed" + +-- | Text representation of the package respecting "installed" versions e.g. +-- +-- - "aeson ==1.2.3" +-- - "mtl installed" +-- +-- This output is suitable for cabal constraints, _not_ a cabal file. +toTextInstalled :: Package -> Text +toTextInstalled p = p.name <> versToTextInstalled p.version + +-- | Text representation of the package, dropping "installed" versions e.g. +-- +-- - "aeson ==1.2.3" +-- - "mtl" +-- +-- This output is suitable for cabal file, _not_ constraints. +toTextNoInstalled :: Package -> Text +toTextNoInstalled p = p.name <> versToTextNoInstalled p.version + +-- | Attempts to parse the text to the package. Some flexibility e.g. +-- supports: +-- +-- - "aeson ==2.0.0," +-- - "mtl installed" +fromCabalConstraintsText :: Text -> Maybe Package +fromCabalConstraintsText = packageParser . T.stripStart + +-- NOTE: [*Parsers] +-- +-- DIY parser, where each function parses only as much as it needs, then +-- returns the rest to be fed into the next parser. Following megaparsec's +-- lead, each parser assumes that it is at the start of relevant text +-- (i.e. no leading whitespace), and consumes trailing whitespace. +-- +-- Hence the "rest" that is returned must have its leading whitespace stripped, +-- so that the next parser can make the same assumption. + +packageParser :: Text -> Maybe Package +packageParser txt = do + (name, r1) <- nameParser txt + (vers, _) <- versionTextParser r1 <|> versionInstalledParser r1 + pure $ MkPackage name vers + +-- Split on whitepspace or equals e.g. "mtl installed", "aeson ==1.2.3". +nameParser :: Text -> Maybe (Text, Text) +nameParser txt + | T.null name = Nothing + | otherwise = Just (name, T.stripStart rest) + where + (name, rest) = T.break isNameChar txt + isNameChar c = c == ' ' || c == '=' + +-- Parse "installed". +versionInstalledParser :: Text -> Maybe (PackageVersion, Text) +versionInstalledParser txt = do + rest <- T.stripPrefix "installed" txt + pure (PackageVersionInstalled, T.stripStart rest) + +-- Parse e.g. "==1.2.3". +versionTextParser :: Text -> Maybe (PackageVersion, Text) +versionTextParser txt = do + r1 <- T.stripPrefix delim txt + let (vers, r2) = T.span isVersChar (T.stripStart r1) + if not (T.null vers) + then Just (PackageVersionText vers, T.stripStart r2) + else Nothing + where + isVersChar c = Ch.isDigit c || c == '.' + +versToTextInstalled :: PackageVersion -> Text +versToTextInstalled (PackageVersionText t) = " " <> delim <> t +versToTextInstalled PackageVersionInstalled = " installed" + +versToTextNoInstalled :: PackageVersion -> Text +versToTextNoInstalled (PackageVersionText t) = " " <> delim <> t +versToTextNoInstalled PackageVersionInstalled = "" + +delim :: Text +delim = "==" diff --git a/src/utils/CLC/Stackage/Utils/Paths.hs b/src/utils/CLC/Stackage/Utils/Paths.hs index 1623164..a8101b1 100644 --- a/src/utils/CLC/Stackage/Utils/Paths.hs +++ b/src/utils/CLC/Stackage/Utils/Paths.hs @@ -11,6 +11,7 @@ module CLC.Stackage.Utils.Paths generatedCabalProjectLocalPath, -- * Utils + canonicalizePath, OsPath.encodeUtf, decodeUtfLenient, unsafeDecodeUtf, @@ -20,8 +21,10 @@ where import GHC.IO.Encoding.Failure (CodingFailureMode (TransliterateCodingFailure)) import GHC.IO.Encoding.UTF16 qualified as UTF16 import GHC.IO.Encoding.UTF8 qualified as UTF8 +import System.Directory.OsPath qualified as Dir import System.OsPath (OsPath, osp, ()) import System.OsPath qualified as OsPath +import System.OsString qualified as OsStr -- | Leniently decodes OsPath to String. decodeUtfLenient :: OsPath -> String @@ -37,6 +40,21 @@ unsafeDecodeUtf p = case OsPath.decodeUtf p of Just fp -> fp Nothing -> error $ "Error decoding ospath: " <> show p +-- | Calls canonicalizePath, after manually expanding tilde (~) to the home +-- directory. The latter usually shouldn't be needed, as the shell normally +-- performs such expansions before the string makes it to the program. +-- But when it is part of an argument e.g. +-- +-- --cabal-path=~/... +-- +-- it is not expanded. +canonicalizePath :: OsPath -> IO OsPath +canonicalizePath p = case OsStr.stripPrefix [osp|~/|] p of + Nothing -> Dir.canonicalizePath p + Just rest -> do + home <- Dir.getHomeDirectory + Dir.canonicalizePath $ home rest + -- | Output directory. outputDir :: OsPath outputDir = [osp|output|] diff --git a/test/functional/Main.hs b/test/functional/Main.hs index b530e3d..bf90e24 100644 --- a/test/functional/Main.hs +++ b/test/functional/Main.hs @@ -2,11 +2,11 @@ module Main (main) where -import CLC.Stackage.Builder.Package (Package (name)) import CLC.Stackage.Runner qualified as Runner import CLC.Stackage.Utils.IO qualified as IO import CLC.Stackage.Utils.Logging qualified as Logging import CLC.Stackage.Utils.OS (Os (Windows), currentOs) +import CLC.Stackage.Utils.Package (Package (name)) import CLC.Stackage.Utils.Paths qualified as Paths import Data.ByteString (ByteString) import Data.ByteString.Char8 qualified as C8 @@ -36,7 +36,8 @@ main = testGroup "Functional" [ testSmall getNoCleanup, - testSmallBatch getNoCleanup + testSmallBatch getNoCleanup, + testSmallSnapshotPath getNoCleanup ] testSmall :: IO Bool -> TestTree @@ -61,6 +62,22 @@ testSmallBatch getNoCleanup = runGolden getNoCleanup params testName = [osp|testSmallBatch|] } +testSmallSnapshotPath :: IO Bool -> TestTree +testSmallSnapshotPath getNoCleanup = runGolden getNoCleanup params + where + params = + MkGoldenParams + { args = ["--snapshot-path", snapshotPath], + runner = runSmall, + testDesc, + testName = [osp|testSmallSnapshotPath|] + } + testDesc = "Finishes clc-stackage with small package list and --snapshot-path" + + snapshotPath = + Paths.unsafeDecodeUtf $ + [osp|test|] [osp|functional|] [osp|snapshot.txt|] + -- | Tests building only a few packages runSmall :: IO [ByteString] runSmall = do @@ -113,7 +130,8 @@ mkHLogger = do let hLogger = Logging.MkHandle - { Logging.getLocalTime = pure mkLocalTime, + { Logging.color = False, + Logging.getLocalTime = pure mkLocalTime, Logging.logStrLn = \s -> modifyIORef' logsRef (s :), Logging.logStrErrLn = \s -> modifyIORef' logsRef (s :), Logging.terminalWidth = 80 diff --git a/test/functional/goldens/testSmallBatch_posix.golden b/test/functional/goldens/testSmallBatch_posix.golden index e088b47..e8ce0c3 100644 --- a/test/functional/goldens/testSmallBatch_posix.golden +++ b/test/functional/goldens/testSmallBatch_posix.golden @@ -1,7 +1,7 @@ [2020-05-31 12:00:00][Info] Cached results do not exist: output/cache.json -[2020-05-31 12:00:00][Success] 3: cborg ==0.2.10.0, clock ==0.8.4 -[2020-05-31 12:00:00][Success] 2: mtl ==2.3.1, optics-core ==0.4.1.1 -[2020-05-31 12:00:00][Success] 1: profunctors ==5.6.2 +[2020-05-31 12:00:00][Success] 3: cborg-0.2.10.0, clock-0.8.4 +[2020-05-31 12:00:00][Success] 2: mtl-2.3.1, optics-core-0.4.1.1 +[2020-05-31 12:00:00][Success] 1: profunctors-5.6.2 - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmallBatch_windows.golden b/test/functional/goldens/testSmallBatch_windows.golden index e867167..5569fee 100644 --- a/test/functional/goldens/testSmallBatch_windows.golden +++ b/test/functional/goldens/testSmallBatch_windows.golden @@ -1,7 +1,7 @@ [2020-05-31 12:00:00][Info] Cached results do not exist: output\cache.json -[2020-05-31 12:00:00][Success] 3: cborg ==0.2.10.0, clock ==0.8.4 -[2020-05-31 12:00:00][Success] 2: mtl ==2.3.1, optics-core ==0.4.1.1 -[2020-05-31 12:00:00][Success] 1: profunctors ==5.6.2 +[2020-05-31 12:00:00][Success] 3: cborg-0.2.10.0, clock-0.8.4 +[2020-05-31 12:00:00][Success] 2: mtl-2.3.1, optics-core-0.4.1.1 +[2020-05-31 12:00:00][Success] 1: profunctors-5.6.2 - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmallSnapshotPath_posix.golden b/test/functional/goldens/testSmallSnapshotPath_posix.golden new file mode 100644 index 0000000..731f562 --- /dev/null +++ b/test/functional/goldens/testSmallSnapshotPath_posix.golden @@ -0,0 +1,12 @@ +[2020-05-31 12:00:00][Info] Cached results do not exist: output/cache.json +[2020-05-31 12:00:00][Warn] Only found 8 packages. Is that right? +[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, mtl-installed... + + +- Successes: 5 (100%) +- Failures: 0 (0%) +- Untested: 0 (0%) + +- Start: 2020-05-31 12:00:00 +- End: 2020-05-31 12:00:00 + diff --git a/test/functional/goldens/testSmallSnapshotPath_windows.golden b/test/functional/goldens/testSmallSnapshotPath_windows.golden new file mode 100644 index 0000000..6ba7a5a --- /dev/null +++ b/test/functional/goldens/testSmallSnapshotPath_windows.golden @@ -0,0 +1,12 @@ +[2020-05-31 12:00:00][Info] Cached results do not exist: output\cache.json +[2020-05-31 12:00:00][Warn] Only found 8 packages. Is that right? +[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, mtl-installed... + + +- Successes: 5 (100%) +- Failures: 0 (0%) +- Untested: 0 (0%) + +- Start: 2020-05-31 12:00:00 +- End: 2020-05-31 12:00:00 + diff --git a/test/functional/goldens/testSmall_posix.golden b/test/functional/goldens/testSmall_posix.golden index 1da2cea..9e6ac70 100644 --- a/test/functional/goldens/testSmall_posix.golden +++ b/test/functional/goldens/testSmall_posix.golden @@ -1,5 +1,5 @@ [2020-05-31 12:00:00][Info] Cached results do not exist: output/cache.json -[2020-05-31 12:00:00][Success] 1: cborg ==0.2.10.0, clock ==0.8.4, mtl ==2.3... +[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, mtl-2.3.1, op... - Successes: 5 (100%) diff --git a/test/functional/goldens/testSmall_windows.golden b/test/functional/goldens/testSmall_windows.golden index 6c55ac1..519cd25 100644 --- a/test/functional/goldens/testSmall_windows.golden +++ b/test/functional/goldens/testSmall_windows.golden @@ -1,5 +1,5 @@ [2020-05-31 12:00:00][Info] Cached results do not exist: output\cache.json -[2020-05-31 12:00:00][Success] 1: cborg ==0.2.10.0, clock ==0.8.4, mtl ==2.3... +[2020-05-31 12:00:00][Success] 1: cborg-0.2.10.0, clock-0.8.4, mtl-2.3.1, op... - Successes: 5 (100%) diff --git a/test/functional/snapshot.txt b/test/functional/snapshot.txt new file mode 100644 index 0000000..f1f48cb --- /dev/null +++ b/test/functional/snapshot.txt @@ -0,0 +1,12 @@ +# This is the list of packages from the functional tests' modifyPackages +# (chosen for consistency) with a few others thrown in, to test that filtering +# works the same for both querying stackage and reading this file. This list +# is intentionally kept small to make maintenance easier. +aeson ==2.2.3.0 +cborg ==0.2.10.0 +clock ==0.8.4 +kan-extensions ==5.2.6 +mtl installed +optics-core ==0.4.1.1 +profunctors ==5.6.2 +servant ==0.20.2 diff --git a/test/unit/Main.hs b/test/unit/Main.hs index 5851630..f2188bc 100644 --- a/test/unit/Main.hs +++ b/test/unit/Main.hs @@ -2,8 +2,10 @@ module Main (main) where import Test.Tasty (defaultMain, localOption, testGroup) import Test.Tasty.Golden (DeleteOutputFile (OnPass)) -import Unit.CLC.Stackage.Runner.Env qualified as Env -import Unit.CLC.Stackage.Runner.Report qualified as Report +import Unit.CLC.Stackage.Parser.API qualified as Parser.API +import Unit.CLC.Stackage.Runner.Env qualified as Runner.Env +import Unit.CLC.Stackage.Runner.Report qualified as Runner.Report +import Unit.CLC.Stackage.Utils.Package qualified as Utils.Package main :: IO () main = @@ -11,6 +13,8 @@ main = localOption OnPass $ testGroup "Unit" - [ Env.tests, - Report.tests + [ Parser.API.tests, + Runner.Env.tests, + Runner.Report.tests, + Utils.Package.tests ] diff --git a/test/unit/Unit/CLC/Stackage/Parser/API.hs b/test/unit/Unit/CLC/Stackage/Parser/API.hs new file mode 100644 index 0000000..2f84921 --- /dev/null +++ b/test/unit/Unit/CLC/Stackage/Parser/API.hs @@ -0,0 +1,40 @@ +{-# OPTIONS_GHC -Wno-missing-import-lists #-} + +module Unit.CLC.Stackage.Parser.API (tests) where + +import CLC.Stackage.Parser.API qualified as API +import CLC.Stackage.Parser.API.CabalConfig qualified as CabalConfig +import CLC.Stackage.Parser.API.JSON qualified as JSON +import CLC.Stackage.Utils.Exception qualified as Ex +import Control.DeepSeq (NFData, rnf) +import Control.Exception (displayException, evaluate) +import Network.HTTP.Client.TLS qualified as TLS +import Unit.Prelude + +tests :: TestTree +tests = + testGroup + "CLC.Stackage.Parser.API" + [ testCabalConfigEndpoint, + testJsonEndpoint + ] + +testCabalConfigEndpoint :: TestTree +testCabalConfigEndpoint = testCase desc $ do + manager <- TLS.newTlsManager + eval =<< CabalConfig.getStackage manager API.stackageSnapshot + where + desc = "Queries stackage cabal.config endpoint" + +testJsonEndpoint :: TestTree +testJsonEndpoint = testCase desc $ do + manager <- TLS.newTlsManager + eval =<< JSON.getStackage manager API.stackageSnapshot + where + desc = "Queries stackage json endpoint" + +eval :: (NFData a) => a -> IO () +eval x = + Ex.tryAny (evaluate $ rnf x) >>= \case + Right _ -> pure () + Left ex -> assertFailure $ "Exception during eval: " ++ displayException ex diff --git a/test/unit/Unit/CLC/Stackage/Runner/Env.hs b/test/unit/Unit/CLC/Stackage/Runner/Env.hs index 9667e68..22deef9 100644 --- a/test/unit/Unit/CLC/Stackage/Runner/Env.hs +++ b/test/unit/Unit/CLC/Stackage/Runner/Env.hs @@ -13,7 +13,7 @@ import Unit.Prelude tests :: TestTree tests = testGroup - "Sequential.Env" + "CLC.Stackage.Runner.Env" [ testResults, newCacheTests ] diff --git a/test/unit/Unit/CLC/Stackage/Runner/Report.hs b/test/unit/Unit/CLC/Stackage/Runner/Report.hs index ed9a128..f4b1d35 100644 --- a/test/unit/Unit/CLC/Stackage/Runner/Report.hs +++ b/test/unit/Unit/CLC/Stackage/Runner/Report.hs @@ -27,7 +27,7 @@ import Unit.Prelude tests :: TestTree tests = testGroup - "Sequential.Report" + "CLC.Stackage.Runner.Report" [ testMkReport, testResultJsonEncode, testReportJsonEncode diff --git a/test/unit/Unit/CLC/Stackage/Utils/Package.hs b/test/unit/Unit/CLC/Stackage/Utils/Package.hs new file mode 100644 index 0000000..6f121dd --- /dev/null +++ b/test/unit/Unit/CLC/Stackage/Utils/Package.hs @@ -0,0 +1,66 @@ +{-# OPTIONS_GHC -Wno-missing-import-lists #-} + +module Unit.CLC.Stackage.Utils.Package (tests) where + +import CLC.Stackage.Utils.Package + ( Package (MkPackage, name, version), + PackageVersion (PackageVersionInstalled, PackageVersionText), + ) +import CLC.Stackage.Utils.Package qualified as Package +import Unit.Prelude + +tests :: TestTree +tests = + testGroup + "CLC.Stackage.Utils.Package" + [ testFromCabalConstraintsTextSuccesses, + testToCabalDepText, + testToCabalConstraintsText, + testToDisplayName + ] + +testFromCabalConstraintsTextSuccesses :: TestTree +testFromCabalConstraintsTextSuccesses = testCase desc $ do + Just e1 @=? Package.fromCabalConstraintsText "aeson ==2.0.1" + Just e1 @=? Package.fromCabalConstraintsText "aeson ==2.0.1," + Just e1 @=? Package.fromCabalConstraintsText " aeson == 2.0.1 , " + Just e2 @=? Package.fromCabalConstraintsText "mtl installed" + Just e2 @=? Package.fromCabalConstraintsText "mtl installed," + Just e2 @=? Package.fromCabalConstraintsText " mtl installed , " + where + desc = "fromCabalConstraintsText successes" + +testToCabalDepText :: TestTree +testToCabalDepText = testCase desc $ do + ", aeson ==2.0.1" @=? Package.toCabalDepText e1 + ", mtl" @=? Package.toCabalDepText e2 + where + desc = "toCabalDepText" + +testToCabalConstraintsText :: TestTree +testToCabalConstraintsText = testCase desc $ do + "aeson ==2.0.1," @=? Package.toCabalConstraintsText e1 + "mtl installed," @=? Package.toCabalConstraintsText e2 + where + desc = "toCabalConstraintsText" + +testToDisplayName :: TestTree +testToDisplayName = testCase desc $ do + "aeson-2.0.1" @=? Package.toDisplayName e1 + "mtl-installed" @=? Package.toDisplayName e2 + where + desc = "toDisplayName" + +e1 :: Package +e1 = + MkPackage + { name = "aeson", + version = PackageVersionText "2.0.1" + } + +e2 :: Package +e2 = + MkPackage + { name = "mtl", + version = PackageVersionInstalled + } diff --git a/test/unit/Unit/Prelude.hs b/test/unit/Unit/Prelude.hs index 585d9b2..a360cb9 100644 --- a/test/unit/Unit/Prelude.hs +++ b/test/unit/Unit/Prelude.hs @@ -10,7 +10,7 @@ import CLC.Stackage.Builder.Env ( MkBuildEnv, batch, buildArgs, - colorLogs, + cabalPath, groupFailFast, hLogger, packagesToBuild, @@ -65,11 +65,12 @@ mkBuildEnv = do MkBuildEnv { batch = Nothing, buildArgs = [], - colorLogs = True, + cabalPath = "cabal", groupFailFast = False, hLogger = Logging.MkHandle - { getLocalTime = pure mkLocalTime, + { color = False, + getLocalTime = pure mkLocalTime, logStrErrLn = const (pure ()), logStrLn = const (pure ()), terminalWidth = 80