From 989eaae63bff09a596d37c2b2bc5abf1a51d6036 Mon Sep 17 00:00:00 2001 From: Amir Roosta Date: Sun, 2 Feb 2020 14:42:25 +0100 Subject: [PATCH 1/3] Add ThresholdAct option to MSBuild task - This will enable the option to choose if the build should fail when the coverage is below the threshold or just give a warning about it --- Documentation/MSBuildIntegration.md | 11 ++++++++ src/coverlet.console/Program.cs | 9 ++++++- src/coverlet.core/Enums/ThresholdAction.cs | 8 ++++++ .../CoverageResultTask.cs | 25 ++++++++++++++++++- .../coverlet.msbuild.props | 1 + .../coverlet.msbuild.targets | 1 + 6 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 src/coverlet.core/Enums/ThresholdAction.cs diff --git a/Documentation/MSBuildIntegration.md b/Documentation/MSBuildIntegration.md index b8b80ad63..b823ff874 100644 --- a/Documentation/MSBuildIntegration.md +++ b/Documentation/MSBuildIntegration.md @@ -105,6 +105,17 @@ The following command will compare the threshold value with the overall total co dotnet test /p:CollectCoverage=true /p:Threshold=80 /p:ThresholdType=line /p:ThresholdStat=total ``` +You can also specify what action to take when the the coverage is below the threshold value using the `/p:ThresholdAct` option. the accepted values are: + +* `fail`: which fails the build and is the default option +* `warning`: which will not stop the build and only gives warning + +The following will give give a warning (but let's the build to continue) when the threshold value is below 80: + +```bash +dotnet test /p:CollectCoverage=true /p:Threshold=80 /p:ThresholdAct=warning +``` + ## Excluding From Coverage ### Attributes diff --git a/src/coverlet.console/Program.cs b/src/coverlet.console/Program.cs index 608786fcb..81cb977f2 100644 --- a/src/coverlet.console/Program.cs +++ b/src/coverlet.console/Program.cs @@ -37,6 +37,7 @@ static int Main(string[] args) CommandOption threshold = app.Option("--threshold", "Exits with error if the coverage % is below value.", CommandOptionType.SingleValue); CommandOption thresholdTypes = app.Option("--threshold-type", "Coverage type to apply the threshold to.", CommandOptionType.MultipleValue); CommandOption thresholdStat = app.Option("--threshold-stat", "Coverage statistic used to enforce the threshold value.", CommandOptionType.SingleValue); + CommandOption thresholdAct = app.Option("--threshold-act", "The action to take when coverage is below the threshold value. Defaults to fail the build", CommandOptionType.SingleValue); CommandOption excludeFilters = app.Option("--exclude", "Filter expressions to exclude specific modules and types.", CommandOptionType.MultipleValue); CommandOption includeFilters = app.Option("--include", "Filter expressions to include only specific modules and types.", CommandOptionType.MultipleValue); CommandOption excludedSourceFiles = app.Option("--exclude-by-file", "Glob patterns specifying source files to exclude.", CommandOptionType.MultipleValue); @@ -105,6 +106,7 @@ static int Main(string[] args) var dThreshold = threshold.HasValue() ? double.Parse(threshold.Value()) : 0; var dThresholdTypes = thresholdTypes.HasValue() ? thresholdTypes.Values : new List(new string[] { "line", "branch", "method" }); var dThresholdStat = thresholdStat.HasValue() ? Enum.Parse(thresholdStat.Value(), true) : Enum.Parse("minimum", true); + var dThresholdAct = thresholdAct.HasValue() ? Enum.Parse(thresholdAct.Value(), true) : Enum.Parse("fail", true); logger.LogInformation("\nCalculating coverage result..."); @@ -223,7 +225,12 @@ static int Main(string[] args) exceptionMessageBuilder.AppendLine($"The {dThresholdStat.ToString().ToLower()} method coverage is below the specified {dThreshold}"); } - throw new Exception(exceptionMessageBuilder.ToString()); + if (dThresholdAct == ThresholdAction.Fail) + { + throw new Exception(exceptionMessageBuilder.ToString()); + } + + logger.LogWarning(exceptionMessageBuilder.ToString()); } return exitCode; diff --git a/src/coverlet.core/Enums/ThresholdAction.cs b/src/coverlet.core/Enums/ThresholdAction.cs new file mode 100644 index 000000000..88e664328 --- /dev/null +++ b/src/coverlet.core/Enums/ThresholdAction.cs @@ -0,0 +1,8 @@ +namespace Coverlet.Core.Enums +{ + internal enum ThresholdAction + { + Fail, + Warning + } +} diff --git a/src/coverlet.msbuild.tasks/CoverageResultTask.cs b/src/coverlet.msbuild.tasks/CoverageResultTask.cs index 2cf0f3691..b46cec01c 100644 --- a/src/coverlet.msbuild.tasks/CoverageResultTask.cs +++ b/src/coverlet.msbuild.tasks/CoverageResultTask.cs @@ -20,6 +20,7 @@ public class CoverageResultTask : Task private double _threshold; private string _thresholdType; private string _thresholdStat; + private string _thresholdAct; private string _coverletMultiTargetFrameworksCurrentTFM; private ITaskItem _instrumenterState; private MSBuildLogger _logger; @@ -59,6 +60,13 @@ public string ThresholdStat set { _thresholdStat = value; } } + [Required] + public string ThresholdAct + { + get { return _thresholdAct; } + set { _thresholdAct = value; } + } + [Required] public ITaskItem InstrumenterState { @@ -148,6 +156,7 @@ public override bool Execute() var thresholdTypeFlags = ThresholdTypeFlags.None; var thresholdStat = ThresholdStatistic.Minimum; + var thresholdAct = ThresholdAction.Fail; foreach (var thresholdType in _thresholdType.Split(',').Select(t => t.Trim())) { @@ -174,6 +183,15 @@ public override bool Execute() thresholdStat = ThresholdStatistic.Total; } + if (_thresholdAct.Equals("fail", StringComparison.OrdinalIgnoreCase)) + { + thresholdAct = ThresholdAction.Fail; + } + else if (_thresholdAct.Equals("warning", StringComparison.OrdinalIgnoreCase)) + { + thresholdAct = ThresholdAction.Warning; + } + var coverageTable = new ConsoleTable("Module", "Line", "Branch", "Method"); var summary = new CoverageSummary(); int numModules = result.Modules.Count; @@ -230,7 +248,12 @@ public override bool Execute() exceptionMessageBuilder.AppendLine($"The {thresholdStat.ToString().ToLower()} method coverage is below the specified {_threshold}"); } - throw new Exception(exceptionMessageBuilder.ToString()); + if(thresholdAct == ThresholdAction.Fail) + { + throw new Exception(exceptionMessageBuilder.ToString()); + } + + _logger.LogWarning(exceptionMessageBuilder.ToString()); } } catch (Exception ex) diff --git a/src/coverlet.msbuild.tasks/coverlet.msbuild.props b/src/coverlet.msbuild.tasks/coverlet.msbuild.props index e6906ccc3..19adda142 100644 --- a/src/coverlet.msbuild.tasks/coverlet.msbuild.props +++ b/src/coverlet.msbuild.tasks/coverlet.msbuild.props @@ -15,6 +15,7 @@ 0 line,branch,method minimum + fail $(MSBuildThisFileDirectory) diff --git a/src/coverlet.msbuild.tasks/coverlet.msbuild.targets b/src/coverlet.msbuild.tasks/coverlet.msbuild.targets index eecec4ee4..e73b1ecfb 100644 --- a/src/coverlet.msbuild.tasks/coverlet.msbuild.targets +++ b/src/coverlet.msbuild.tasks/coverlet.msbuild.targets @@ -39,6 +39,7 @@ Threshold="$(Threshold)" ThresholdType="$(ThresholdType)" ThresholdStat="$(ThresholdStat)" + ThresholdAct="$(ThresholdAct)" InstrumenterState="$(InstrumenterState)" CoverletMultiTargetFrameworksCurrentTFM="$(_coverletMultiTargetFrameworksCurrentTFM)" /> From 3b5bc1e3936252d83bbebdafe65a1161258731cb Mon Sep 17 00:00:00 2001 From: Amir Roosta Date: Wed, 5 Feb 2020 15:46:06 +0100 Subject: [PATCH 2/3] Fix review issues --- Documentation/MSBuildIntegration.md | 6 +++--- src/coverlet.console/Program.cs | 14 +++++++++----- src/coverlet.msbuild.tasks/CoverageResultTask.cs | 11 +++++++---- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/Documentation/MSBuildIntegration.md b/Documentation/MSBuildIntegration.md index b823ff874..029da0fc7 100644 --- a/Documentation/MSBuildIntegration.md +++ b/Documentation/MSBuildIntegration.md @@ -105,12 +105,12 @@ The following command will compare the threshold value with the overall total co dotnet test /p:CollectCoverage=true /p:Threshold=80 /p:ThresholdType=line /p:ThresholdStat=total ``` -You can also specify what action to take when the the coverage is below the threshold value using the `/p:ThresholdAct` option. the accepted values are: +You can also specify what action to take when the the coverage is below the threshold value using the `/p:ThresholdAct` option. The accepted values are: * `fail`: which fails the build and is the default option -* `warning`: which will not stop the build and only gives warning +* `warning`: which will not stop the build and only gives a warning console message -The following will give give a warning (but let's the build to continue) when the threshold value is below 80: +The following will give a warning (but let's the build to continue) when the threshold value is below 80: ```bash dotnet test /p:CollectCoverage=true /p:Threshold=80 /p:ThresholdAct=warning diff --git a/src/coverlet.console/Program.cs b/src/coverlet.console/Program.cs index 81cb977f2..c463e6a9b 100644 --- a/src/coverlet.console/Program.cs +++ b/src/coverlet.console/Program.cs @@ -205,10 +205,10 @@ static int Main(string[] args) { exitCode += (int)CommandExitCodes.TestFailed; } + thresholdTypeFlags = result.GetThresholdTypesBelowThreshold(summary, dThreshold, thresholdTypeFlags, dThresholdStat); if (thresholdTypeFlags != ThresholdTypeFlags.None) { - exitCode += (int)CommandExitCodes.CoverageBelowThreshold; var exceptionMessageBuilder = new StringBuilder(); if ((thresholdTypeFlags & ThresholdTypeFlags.Line) != ThresholdTypeFlags.None) { @@ -225,12 +225,16 @@ static int Main(string[] args) exceptionMessageBuilder.AppendLine($"The {dThresholdStat.ToString().ToLower()} method coverage is below the specified {dThreshold}"); } - if (dThresholdAct == ThresholdAction.Fail) + switch (dThresholdAct) { - throw new Exception(exceptionMessageBuilder.ToString()); + case ThresholdAction.Warning: + logger.LogWarning(exceptionMessageBuilder.ToString()); + break; + case ThresholdAction.Fail: + default: + exitCode += (int)CommandExitCodes.CoverageBelowThreshold; + throw new Exception(exceptionMessageBuilder.ToString()); } - - logger.LogWarning(exceptionMessageBuilder.ToString()); } return exitCode; diff --git a/src/coverlet.msbuild.tasks/CoverageResultTask.cs b/src/coverlet.msbuild.tasks/CoverageResultTask.cs index b46cec01c..25d847a7e 100644 --- a/src/coverlet.msbuild.tasks/CoverageResultTask.cs +++ b/src/coverlet.msbuild.tasks/CoverageResultTask.cs @@ -248,12 +248,15 @@ public override bool Execute() exceptionMessageBuilder.AppendLine($"The {thresholdStat.ToString().ToLower()} method coverage is below the specified {_threshold}"); } - if(thresholdAct == ThresholdAction.Fail) + switch (thresholdAct) { - throw new Exception(exceptionMessageBuilder.ToString()); + case ThresholdAction.Warning: + _logger.LogWarning(exceptionMessageBuilder.ToString()); + break; + case ThresholdAction.Fail: + default: + throw new Exception(exceptionMessageBuilder.ToString()); } - - _logger.LogWarning(exceptionMessageBuilder.ToString()); } } catch (Exception ex) From d90a3f866d513f02986067371dffd8d328c51c17 Mon Sep 17 00:00:00 2001 From: Amir Roosta Date: Wed, 10 Jul 2024 20:29:28 +0200 Subject: [PATCH 3/3] Sync threshold action logic with master branch updates --- Documentation/MSBuildIntegration.md | 4 ++-- src/coverlet.console/Program.cs | 20 +++++++++++++--- src/coverlet.core/Enums/ThresholdAction.cs | 5 +++- .../CoverageResultTask.cs | 24 ++++++++++++++++++- .../coverlet.msbuild.props | 1 + .../CoverageResultTaskTests.cs | 1 + 6 files changed, 48 insertions(+), 7 deletions(-) diff --git a/Documentation/MSBuildIntegration.md b/Documentation/MSBuildIntegration.md index 06bd8f098..84a07da62 100644 --- a/Documentation/MSBuildIntegration.md +++ b/Documentation/MSBuildIntegration.md @@ -132,12 +132,12 @@ The following command will compare the threshold value with the overall total co dotnet test /p:CollectCoverage=true /p:Threshold=80 /p:ThresholdType=line /p:ThresholdStat=total ``` -You can also specify what action to take when the the coverage is below the threshold value using the `/p:ThresholdAct` option. The accepted values are: +You can also specify what action to take when the coverage is below the threshold value using the `/p:ThresholdAct` option. The accepted values are: * `fail`: which fails the build and is the default option * `warning`: which will not stop the build and only gives a warning console message -The following will give a warning (but let's the build to continue) when the threshold value is below 80: +The following will give a warning, but allows the build to continue when the threshold value is below 80: ```bash dotnet test /p:CollectCoverage=true /p:Threshold=80 /p:ThresholdAct=warning diff --git a/src/coverlet.console/Program.cs b/src/coverlet.console/Program.cs index 864df117f..8aaa2e659 100644 --- a/src/coverlet.console/Program.cs +++ b/src/coverlet.console/Program.cs @@ -36,6 +36,7 @@ static int Main(string[] args) var threshold = new Option("--threshold", "Exits with error if the coverage % is below value.") { Arity = ArgumentArity.ZeroOrOne }; var thresholdTypes = new Option>("--threshold-type", () => new List(new string[] { "line", "branch", "method" }), "Coverage type to apply the threshold to.").FromAmong("line", "branch", "method"); var thresholdStat = new Option("--threshold-stat", () => ThresholdStatistic.Minimum, "Coverage statistic used to enforce the threshold value.") { Arity = ArgumentArity.ZeroOrOne }; + var thresholdAct = new Option("--threshold-act", () => ThresholdAction.Fail, "The action to take when coverage is below the threshold value. Defaults to failing the build.") { Arity = ArgumentArity.ZeroOrOne }; var excludeFilters = new Option("--exclude", "Filter expressions to exclude specific modules and types.") { Arity = ArgumentArity.ZeroOrMore, AllowMultipleArgumentsPerToken = true }; var includeFilters = new Option("--include", "Filter expressions to include only specific modules and types.") { Arity = ArgumentArity.ZeroOrMore, AllowMultipleArgumentsPerToken = true }; var excludedSourceFiles = new Option("--exclude-by-file", "Glob patterns specifying source files to exclude.") { Arity = ArgumentArity.ZeroOrMore, AllowMultipleArgumentsPerToken = true }; @@ -61,6 +62,7 @@ static int Main(string[] args) threshold, thresholdTypes, thresholdStat, + thresholdAct, excludeFilters, includeFilters, excludedSourceFiles, @@ -89,6 +91,7 @@ static int Main(string[] args) string thresholdValue = context.ParseResult.GetValueForOption(threshold); List thresholdTypesValue = context.ParseResult.GetValueForOption(thresholdTypes); ThresholdStatistic thresholdStatValue = context.ParseResult.GetValueForOption(thresholdStat); + ThresholdAction thresholdActValue = context.ParseResult.GetValueForOption(thresholdAct); string[] excludeFiltersValue = context.ParseResult.GetValueForOption(excludeFilters); string[] includeFiltersValue = context.ParseResult.GetValueForOption(includeFilters); string[] excludedSourceFilesValue = context.ParseResult.GetValueForOption(excludedSourceFiles); @@ -115,6 +118,7 @@ static int Main(string[] args) thresholdValue, thresholdTypesValue, thresholdStatValue, + thresholdActValue, excludeFiltersValue, includeFiltersValue, excludedSourceFilesValue, @@ -142,6 +146,7 @@ private static Task HandleCommand(string moduleOrAppDirectory, string threshold, List thresholdTypes, ThresholdStatistic thresholdStat, + ThresholdAction thresholdAct, string[] excludeFilters, string[] includeFilters, string[] excludedSourceFiles, @@ -380,12 +385,21 @@ string sourceMappingFile { exceptionMessageBuilder.AppendLine($"The {thresholdStat.ToString().ToLower()} method coverage is below the specified {thresholdTypeFlagValues[ThresholdTypeFlags.Method]}"); } - throw new Exception(exceptionMessageBuilder.ToString()); + + switch (thresholdAct) + { + case ThresholdAction.Warning: + logger.LogWarning(exceptionMessageBuilder.ToString()); + break; + case ThresholdAction.Fail: + exitCode += (int)CommandExitCodes.CoverageBelowThreshold; + throw new Exception(exceptionMessageBuilder.ToString()); + default: + throw new ArgumentOutOfRangeException(nameof(thresholdAct), thresholdAct, "Unhandled threshold action"); + } } return Task.FromResult(exitCode); - - } catch (Win32Exception we) when (we.Source == "System.Diagnostics.Process") diff --git a/src/coverlet.core/Enums/ThresholdAction.cs b/src/coverlet.core/Enums/ThresholdAction.cs index 88e664328..acf7af56a 100644 --- a/src/coverlet.core/Enums/ThresholdAction.cs +++ b/src/coverlet.core/Enums/ThresholdAction.cs @@ -1,4 +1,7 @@ -namespace Coverlet.Core.Enums +// Copyright (c) Toni Solarin-Sodara +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Coverlet.Core.Enums { internal enum ThresholdAction { diff --git a/src/coverlet.msbuild.tasks/CoverageResultTask.cs b/src/coverlet.msbuild.tasks/CoverageResultTask.cs index 87ba40c7d..305dfd163 100644 --- a/src/coverlet.msbuild.tasks/CoverageResultTask.cs +++ b/src/coverlet.msbuild.tasks/CoverageResultTask.cs @@ -37,6 +37,9 @@ public class CoverageResultTask : BaseTask [Required] public string ThresholdStat { get; set; } + [Required] + public string ThresholdAct { get; set; } + [Required] public ITaskItem InstrumenterState { get; set; } @@ -190,6 +193,16 @@ public override bool Execute() thresholdStat = ThresholdStatistic.Total; } + ThresholdAction thresholdAct = ThresholdAction.Fail; + if (ThresholdAct.Equals("fail", StringComparison.OrdinalIgnoreCase)) + { + thresholdAct = ThresholdAction.Fail; + } + else if (ThresholdAct.Equals("warning", StringComparison.OrdinalIgnoreCase)) + { + thresholdAct = ThresholdAction.Warning; + } + var coverageTable = new ConsoleTable("Module", "Line", "Branch", "Method"); var summary = new CoverageSummary(); @@ -248,7 +261,16 @@ public override bool Execute() $"The {thresholdStat.ToString().ToLower()} method coverage is below the specified {thresholdTypeFlagValues[ThresholdTypeFlags.Method]}"); } - throw new Exception(exceptionMessageBuilder.ToString()); + switch (thresholdAct) + { + case ThresholdAction.Warning: + _logger.LogWarning(exceptionMessageBuilder.ToString()); + break; + case ThresholdAction.Fail: + throw new Exception(exceptionMessageBuilder.ToString()); + default: + throw new ArgumentOutOfRangeException(nameof(thresholdAct), thresholdAct, "Unhandled threshold action"); + } } } catch (Exception ex) diff --git a/src/coverlet.msbuild.tasks/coverlet.msbuild.props b/src/coverlet.msbuild.tasks/coverlet.msbuild.props index 9403e7702..08a789aa5 100644 --- a/src/coverlet.msbuild.tasks/coverlet.msbuild.props +++ b/src/coverlet.msbuild.tasks/coverlet.msbuild.props @@ -16,6 +16,7 @@ 0 line,branch,method minimum + fail diff --git a/test/coverlet.msbuild.tasks.tests/CoverageResultTaskTests.cs b/test/coverlet.msbuild.tasks.tests/CoverageResultTaskTests.cs index 547fcdbfb..93643948c 100644 --- a/test/coverlet.msbuild.tasks.tests/CoverageResultTaskTests.cs +++ b/test/coverlet.msbuild.tasks.tests/CoverageResultTaskTests.cs @@ -98,6 +98,7 @@ public void Execute_StateUnderTest_WithInstrumentationState_Fake() Threshold = "50", ThresholdType = "total", ThresholdStat = "total", + ThresholdAct = "fail", InstrumenterState = InstrumenterState }; coverageResultTask.BuildEngine = _buildEngine.Object;