diff --git a/Documentation/Changelog.md b/Documentation/Changelog.md index 5dfebc247..7e3f47b0b 100644 --- a/Documentation/Changelog.md +++ b/Documentation/Changelog.md @@ -6,7 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased -## Release date 2024-01-20 +### Improvements +- Implement solution based merging for data collector [#1307](https://github.com/coverlet-coverage/coverlet/issues/1307) + +## Release date 2025-01-20 ### Packages coverlet.msbuild 6.0.4 coverlet.console 6.0.4 diff --git a/Documentation/Troubleshooting.md b/Documentation/Troubleshooting.md index 5a3e682cc..941115117 100644 --- a/Documentation/Troubleshooting.md +++ b/Documentation/Troubleshooting.md @@ -265,3 +265,12 @@ To enable exceptions log for in-process data collectors ```shell set COVERLET_DATACOLLECTOR_INPROC_EXCEPTIONLOG_ENABLED=1 ``` + +## Enable collector post processing debugging +Post processing is uses for automatically merging coverage reports with the `ReportMerging` option enabled. You can live attach and debug the post processor `COVERLET_DATACOLLECTOR_POSTPROCESSOR_DEBUG`. + +You will be asked to attach a debugger through UI popup. + +```shell + set COVERLET_DATACOLLECTOR_POSTPROCESSOR_DEBUG=1 +``` diff --git a/Documentation/VSTestIntegration.md b/Documentation/VSTestIntegration.md index 3c8be75e1..0354e119a 100644 --- a/Documentation/VSTestIntegration.md +++ b/Documentation/VSTestIntegration.md @@ -121,6 +121,7 @@ These are a list of options that are supported by coverlet. These can be specifi | DoesNotReturnAttribute | Methods marked with these attributes are known not to return, statements following them will be excluded from coverage | | DeterministicReport | Generates deterministic report in context of deterministic build. Take a look at [documentation](DeterministicBuild.md) for further informations. | ExcludeAssembliesWithoutSources | Specifies whether to exclude assemblies without source. Options are either MissingAll, MissingAny or None. Default is MissingAll.| +| ReportMerging | Automatically merge coverage reports if coverage is calculated for multiple projects. Default is false.| How to specify these options via runsettings? @@ -143,6 +144,7 @@ How to specify these options via runsettings? true false MissingAll,MissingAny,None + false diff --git a/src/coverlet.collector/ArtifactPostProcessor/CoverletCoveragePostProcessor.cs b/src/coverlet.collector/ArtifactPostProcessor/CoverletCoveragePostProcessor.cs new file mode 100644 index 000000000..5151cc362 --- /dev/null +++ b/src/coverlet.collector/ArtifactPostProcessor/CoverletCoveragePostProcessor.cs @@ -0,0 +1,162 @@ +// Copyright (c) Toni Solarin-Sodara +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Xml; +using Coverlet.Collector.Utilities; +using Coverlet.Core; +using Coverlet.Core.Abstractions; +using Coverlet.Core.Reporters; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using Newtonsoft.Json; +using System.Diagnostics; +using Coverlet.Core.Helpers; + +namespace coverlet.collector.ArtifactPostProcessor +{ + public class CoverletCoveragePostProcessor : IDataCollectorAttachmentProcessor + { + private CoverageResult _coverageResult; + private ReportFormatParser _reportFormatParser; + private IMessageLogger _logger; + + public bool SupportsIncrementalProcessing => true; + + public IEnumerable GetExtensionUris() => new[] { new Uri(CoverletConstants.DefaultUri) }; + + public Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, + ICollection attachments, IProgress progressReporter, + IMessageLogger logger, CancellationToken cancellationToken) + { + _reportFormatParser ??= new ReportFormatParser(); + _coverageResult ??= new CoverageResult(); + _coverageResult.Modules ??= new Modules(); + _logger = logger; + + string[] formats = _reportFormatParser.ParseReportFormats(configurationElement); + bool deterministic = _reportFormatParser.ParseDeterministicReport(configurationElement); + bool useSourceLink = _reportFormatParser.ParseUseSourceLink(configurationElement); + bool reportMerging = _reportFormatParser.ParseReportMerging(configurationElement); + + AttachDebugger(); + + if (!reportMerging) return Task.FromResult(attachments); + + IList reporters = CreateReporters(formats).ToList(); + + if (attachments.Count > 1) + { + _coverageResult.Parameters = new CoverageParameters() {DeterministicReport = deterministic, UseSourceLink = useSourceLink }; + + var fileAttachments = attachments.SelectMany(x => x.Attachments.Where(IsFileAttachment)).ToList(); + string mergeFilePath = Path.GetDirectoryName(fileAttachments.First().Uri.LocalPath); + + MergeExistingJsonReports(attachments); + + RemoveObsoleteReports(fileAttachments); + + AttachmentSet mergedFileAttachment = WriteCoverageReports(reporters, mergeFilePath, _coverageResult); + + attachments = new List { mergedFileAttachment }; + } + + return Task.FromResult(attachments); + } + + private static void RemoveObsoleteReports(List fileAttachments) + { + fileAttachments.ForEach(x => + { + string directory = Path.GetDirectoryName(x.Uri.LocalPath); + if (! string.IsNullOrEmpty(directory) && Directory.Exists(directory)) + Directory.Delete(directory, true); + }); + } + + private void MergeExistingJsonReports(IEnumerable attachments) + { + foreach (AttachmentSet attachmentSet in attachments) + { + attachmentSet.Attachments.Where(IsFileWithJsonExt).ToList().ForEach(x => + MergeWithCoverageResult(x.Uri.LocalPath, _coverageResult) + ); + } + } + + private AttachmentSet WriteCoverageReports(IEnumerable reporters, string directory, CoverageResult coverageResult) + { + var attachment = new AttachmentSet(new Uri(CoverletConstants.DefaultUri), string.Empty); + foreach (IReporter reporter in reporters) + { + string report = GetCoverageReport(coverageResult, reporter); + var file = new FileInfo(Path.Combine(directory, Path.ChangeExtension(CoverletConstants.DefaultFileName, reporter.Extension))); + file.Directory?.Create(); + File.WriteAllText(file.FullName, report); + attachment.Attachments.Add(new UriDataAttachment(new Uri(file.FullName),string.Empty)); + } + return attachment; + } + + private static bool IsFileWithJsonExt(UriDataAttachment x) + { + return IsFileAttachment(x) && Path.GetExtension(x.Uri.AbsolutePath).Equals(".json"); + } + + private static bool IsFileAttachment(UriDataAttachment x) + { + return x.Uri.IsFile; + } + + private void MergeWithCoverageResult(string filePath, CoverageResult coverageResult) + { + string json = File.ReadAllText(filePath); + coverageResult.Merge(JsonConvert.DeserializeObject(json)); + } + + private string GetCoverageReport(CoverageResult coverageResult, IReporter reporter) + { + try + { + // empty source root translator returns the original path for deterministic report + return reporter.Report(coverageResult, new SourceRootTranslator()); + } + catch (Exception ex) + { + throw new CoverletDataCollectorException( + $"{CoverletConstants.DataCollectorName}: Failed to get coverage report", ex); + } + } + + private void AttachDebugger() + { + if (int.TryParse(Environment.GetEnvironmentVariable("COVERLET_DATACOLLECTOR_POSTPROCESSOR_DEBUG"), out int result) && result == 1) + { + Debugger.Launch(); + Debugger.Break(); + } + } + + private IEnumerable CreateReporters(IEnumerable formats) + { + IEnumerable reporters = formats.Select(format => + { + var reporterFactory = new ReporterFactory(format); + if (!reporterFactory.IsValidFormat()) + { + _logger.SendMessage(TestMessageLevel.Warning, $"Invalid report format '{format}'"); + return null; + } + return reporterFactory.CreateReporter(); + }).Where(r => r != null); + + return reporters; + } + } +} diff --git a/src/coverlet.collector/DataCollection/CoverageManager.cs b/src/coverlet.collector/DataCollection/CoverageManager.cs index f0453501c..467414f18 100644 --- a/src/coverlet.collector/DataCollection/CoverageManager.cs +++ b/src/coverlet.collector/DataCollection/CoverageManager.cs @@ -26,24 +26,32 @@ internal class CoverageManager public CoverageManager(CoverletSettings settings, TestPlatformEqtTrace eqtTrace, TestPlatformLogger logger, ICoverageWrapper coverageWrapper, IInstrumentationHelper instrumentationHelper, IFileSystem fileSystem, ISourceRootTranslator sourceRootTranslator, ICecilSymbolHelper cecilSymbolHelper) : this(settings, - settings.ReportFormats.Select(format => - { - var reporterFactory = new ReporterFactory(format); - if (!reporterFactory.IsValidFormat()) - { - eqtTrace.Warning($"Invalid report format '{format}'"); - return null; - } - else - { - return reporterFactory.CreateReporter(); - } - }).Where(r => r != null).ToArray(), + CreateReporters(settings, eqtTrace), new CoverletLogger(eqtTrace, logger), coverageWrapper, instrumentationHelper, fileSystem, sourceRootTranslator, cecilSymbolHelper) { } + private static IReporter[] CreateReporters(CoverletSettings settings, TestPlatformEqtTrace eqtTrace) + { + if (settings.ReportMerging && ! settings.ReportFormats.Contains("json")) + settings.ReportFormats = settings.ReportFormats.Append("json").ToArray(); + + return settings.ReportFormats.Select(format => + { + var reporterFactory = new ReporterFactory(format); + if (!reporterFactory.IsValidFormat()) + { + eqtTrace.Warning($"Invalid report format '{format}'"); + return null; + } + else + { + return reporterFactory.CreateReporter(); + } + }).Where(r => r != null).ToArray(); + } + public CoverageManager(CoverletSettings settings, IReporter[] reporters, ILogger logger, ICoverageWrapper coverageWrapper, IInstrumentationHelper instrumentationHelper, IFileSystem fileSystem, ISourceRootTranslator sourceRootTranslator, ICecilSymbolHelper cecilSymbolHelper) { diff --git a/src/coverlet.collector/DataCollection/CoverletCoverageCollector.cs b/src/coverlet.collector/DataCollection/CoverletCoverageCollector.cs index c95ba217b..791861609 100644 --- a/src/coverlet.collector/DataCollection/CoverletCoverageCollector.cs +++ b/src/coverlet.collector/DataCollection/CoverletCoverageCollector.cs @@ -6,6 +6,7 @@ using System.Diagnostics; using System.Linq; using System.Xml; +using coverlet.collector.ArtifactPostProcessor; using Coverlet.Collector.Utilities; using Coverlet.Collector.Utilities.Interfaces; using Coverlet.Core.Abstractions; @@ -21,6 +22,7 @@ namespace Coverlet.Collector.DataCollection /// [DataCollectorTypeUri(CoverletConstants.DefaultUri)] [DataCollectorFriendlyName(CoverletConstants.FriendlyName)] + [DataCollectorAttachmentProcessor(typeof(CoverletCoveragePostProcessor))] public class CoverletCoverageCollector : DataCollector { private readonly TestPlatformEqtTrace _eqtTrace; diff --git a/src/coverlet.collector/DataCollection/CoverletSettings.cs b/src/coverlet.collector/DataCollection/CoverletSettings.cs index 0c80687f9..5080374a4 100644 --- a/src/coverlet.collector/DataCollection/CoverletSettings.cs +++ b/src/coverlet.collector/DataCollection/CoverletSettings.cs @@ -86,6 +86,11 @@ internal class CoverletSettings /// public string ExcludeAssembliesWithoutSources { get; set; } + /// + /// Report merging flag + /// + public bool ReportMerging { get; set; } + public override string ToString() { var builder = new StringBuilder(); @@ -104,6 +109,7 @@ public override string ToString() builder.AppendFormat("DoesNotReturnAttributes: '{0}'", string.Join(",", DoesNotReturnAttributes ?? Enumerable.Empty())); builder.AppendFormat("DeterministicReport: '{0}'", DeterministicReport); builder.AppendFormat("ExcludeAssembliesWithoutSources: '{0}'", ExcludeAssembliesWithoutSources); + builder.AppendFormat("ReportMerging: '{0}'", ReportMerging); return builder.ToString(); } diff --git a/src/coverlet.collector/DataCollection/CoverletSettingsParser.cs b/src/coverlet.collector/DataCollection/CoverletSettingsParser.cs index 733dacfcc..488687263 100644 --- a/src/coverlet.collector/DataCollection/CoverletSettingsParser.cs +++ b/src/coverlet.collector/DataCollection/CoverletSettingsParser.cs @@ -15,10 +15,12 @@ namespace Coverlet.Collector.DataCollection internal class CoverletSettingsParser { private readonly TestPlatformEqtTrace _eqtTrace; + private readonly ReportFormatParser _reportFormatParser; public CoverletSettingsParser(TestPlatformEqtTrace eqtTrace) { _eqtTrace = eqtTrace; + _reportFormatParser = new ReportFormatParser(); } /// @@ -48,9 +50,10 @@ public CoverletSettings Parse(XmlElement configurationElement, IEnumerable testModules) return testModules.FirstOrDefault(); } - /// - /// Parse report formats - /// - /// Configuration element - /// Report formats - private static string[] ParseReportFormats(XmlElement configurationElement) - { - string[] formats = Array.Empty(); - if (configurationElement != null) - { - XmlElement reportFormatElement = configurationElement[CoverletConstants.ReportFormatElementName]; - formats = SplitElement(reportFormatElement); - } - - return formats is null || formats.Length == 0 ? new[] { CoverletConstants.DefaultReportFormat } : formats; - } - /// /// Parse filters to include /// diff --git a/src/coverlet.collector/Utilities/CoverletConstants.cs b/src/coverlet.collector/Utilities/CoverletConstants.cs index 5ce4a79ef..0b8be29b5 100644 --- a/src/coverlet.collector/Utilities/CoverletConstants.cs +++ b/src/coverlet.collector/Utilities/CoverletConstants.cs @@ -27,5 +27,6 @@ internal static class CoverletConstants public const string DoesNotReturnAttributesElementName = "DoesNotReturnAttribute"; public const string DeterministicReport = "DeterministicReport"; public const string ExcludeAssembliesWithoutSources = "ExcludeAssembliesWithoutSources"; + public const string ReportMerging = "ReportMerging"; } } diff --git a/src/coverlet.collector/Utilities/ReportFormatParser.cs b/src/coverlet.collector/Utilities/ReportFormatParser.cs new file mode 100644 index 000000000..c18461581 --- /dev/null +++ b/src/coverlet.collector/Utilities/ReportFormatParser.cs @@ -0,0 +1,50 @@ +// Copyright (c) Toni Solarin-Sodara +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Xml; +using System.Linq; + +namespace Coverlet.Collector.Utilities +{ + internal class ReportFormatParser + { + internal string[] ParseReportFormats(XmlElement configurationElement) + { + string[] formats = Array.Empty(); + if (configurationElement != null) + { + XmlElement reportFormatElement = configurationElement[CoverletConstants.ReportFormatElementName]; + formats = SplitElement(reportFormatElement); + } + + return formats is null || formats.Length == 0 ? new[] { CoverletConstants.DefaultReportFormat } : formats; + } + + private static string[] SplitElement(XmlElement element) + { + return element?.InnerText?.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Where(value => !string.IsNullOrWhiteSpace(value)).Select(value => value.Trim()).ToArray(); + } + + internal bool ParseUseSourceLink(XmlElement configurationElement) + { + XmlElement useSourceLinkElement = configurationElement[CoverletConstants.UseSourceLinkElementName]; + bool.TryParse(useSourceLinkElement?.InnerText, out bool useSourceLink); + return useSourceLink; + } + + internal bool ParseDeterministicReport(XmlElement configurationElement) + { + XmlElement deterministicReportElement = configurationElement[CoverletConstants.DeterministicReport]; + bool.TryParse(deterministicReportElement?.InnerText, out bool deterministicReport); + return deterministicReport; + } + + internal bool ParseReportMerging(XmlElement configurationElement) + { + XmlElement mergeWithElement = configurationElement[CoverletConstants.ReportMerging]; + bool.TryParse(mergeWithElement?.InnerText, out bool mergeWith); + return mergeWith; + } + } +} diff --git a/src/coverlet.core/Helpers/SourceRootTranslator.cs b/src/coverlet.core/Helpers/SourceRootTranslator.cs index d5a0dfcc7..ad4323a6d 100644 --- a/src/coverlet.core/Helpers/SourceRootTranslator.cs +++ b/src/coverlet.core/Helpers/SourceRootTranslator.cs @@ -24,11 +24,18 @@ internal class SourceRootTranslator : ISourceRootTranslator private readonly Dictionary> _sourceToDeterministicPathMapping; private Dictionary _resolutionCacheFiles; + public SourceRootTranslator() + { + _sourceRootMapping = new Dictionary>(); + _sourceToDeterministicPathMapping = new Dictionary>(); + } + public SourceRootTranslator(ILogger logger, IFileSystem fileSystem) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); _sourceRootMapping = new Dictionary>(); + _sourceToDeterministicPathMapping = new Dictionary>(); } public SourceRootTranslator(string sourceMappingFile, ILogger logger, IFileSystem fileSystem)