From 0a5fa5d4c4e2294f1d20b592892296948241428a Mon Sep 17 00:00:00 2001 From: Mathieu Guindon Date: Fri, 14 Mar 2025 19:51:49 -0400 Subject: [PATCH] fix tag updates --- .../SynchronizationPipelineFactory.cs | 2 +- .../Sections/SyncTags/LoadGitHubTagsBlock.cs | 2 +- .../Sections/SyncTags/SyncTagsSection.cs | 185 ++++++++++++------ .../Sections/SyncXmldoc/SyncXmldocSection.cs | 2 +- .../Pipeline/SynchronizeTagsPipeline.cs | 10 +- .../Services/GitHubClientService.cs | 10 +- .../Services/rubberduckdb/TagServices.cs | 6 +- 7 files changed, 144 insertions(+), 73 deletions(-) diff --git a/rubberduckvba.Server/ContentSynchronization/Pipeline/Abstract/SynchronizationPipelineFactory.cs b/rubberduckvba.Server/ContentSynchronization/Pipeline/Abstract/SynchronizationPipelineFactory.cs index 43e03c0..4ef7648 100644 --- a/rubberduckvba.Server/ContentSynchronization/Pipeline/Abstract/SynchronizationPipelineFactory.cs +++ b/rubberduckvba.Server/ContentSynchronization/Pipeline/Abstract/SynchronizationPipelineFactory.cs @@ -54,7 +54,7 @@ public ISynchronizationPipeline Create(TParamete return parameters switch { XmldocSyncRequestParameters => new SynchronizeXmlPipeline(parameters, _logger, _content, _github, _merge, _staging, _markdown, tokenSource, _inspections, _quickfixes, _annotations, _annotationParser, _quickFixParser, _inspectionParser, _tagServices), - TagSyncRequestParameters => new SynchronizeTagsPipeline(parameters, _logger, _content, _github, _merge, _staging, tokenSource), + TagSyncRequestParameters => new SynchronizeTagsPipeline(parameters, _logger, _content, _tagServices, _github, _merge, _staging, tokenSource), _ => throw new NotSupportedException(), }; } diff --git a/rubberduckvba.Server/ContentSynchronization/Pipeline/Sections/SyncTags/LoadGitHubTagsBlock.cs b/rubberduckvba.Server/ContentSynchronization/Pipeline/Sections/SyncTags/LoadGitHubTagsBlock.cs index a6c1b65..9a86535 100644 --- a/rubberduckvba.Server/ContentSynchronization/Pipeline/Sections/SyncTags/LoadGitHubTagsBlock.cs +++ b/rubberduckvba.Server/ContentSynchronization/Pipeline/Sections/SyncTags/LoadGitHubTagsBlock.cs @@ -16,7 +16,7 @@ public LoadGitHubTagsBlock(PipelineSection parent, CancellationToke public override async Task TransformAsync(SyncContext input) { - var githubTags = await _github.GetAllTagsAsync(Context.RubberduckDbMain.Name); + var githubTags = await _github.GetAllTagsAsync(); var (gitHubMain, gitHubNext, gitHubOthers) = githubTags.GetLatestTags(); Context.LoadGitHubTags(gitHubMain, gitHubNext, gitHubOthers); diff --git a/rubberduckvba.Server/ContentSynchronization/Pipeline/Sections/SyncTags/SyncTagsSection.cs b/rubberduckvba.Server/ContentSynchronization/Pipeline/Sections/SyncTags/SyncTagsSection.cs index b8cec89..be05bae 100644 --- a/rubberduckvba.Server/ContentSynchronization/Pipeline/Sections/SyncTags/SyncTagsSection.cs +++ b/rubberduckvba.Server/ContentSynchronization/Pipeline/Sections/SyncTags/SyncTagsSection.cs @@ -2,84 +2,147 @@ using rubberduckvba.Server.ContentSynchronization.Pipeline.Sections.Context; using rubberduckvba.Server.Model; using rubberduckvba.Server.Services; +using rubberduckvba.Server.Services.rubberduckdb; using System.Threading.Tasks.Dataflow; namespace rubberduckvba.Server.ContentSynchronization.Pipeline.Sections.SyncTags; public class SyncTagsSection : PipelineSection { - public SyncTagsSection(IPipeline parent, CancellationTokenSource tokenSource, ILogger logger, IRubberduckDbService content, IGitHubClientService github, IStagingServices staging) + public SyncTagsSection(IPipeline parent, CancellationTokenSource tokenSource, ILogger logger, TagServices tagServices, IGitHubClientService github, IStagingServices staging) : base(parent, tokenSource, logger) { - ReceiveRequest = new ReceiveRequestBlock(this, tokenSource, logger); - BroadcastParameters = new BroadcastParametersBlock(this, tokenSource, logger); - AcquireDbMainTag = new AcquireDbMainTagGraphBlock(this, tokenSource, content, logger); - AcquireDbNextTag = new AcquireDbNextTagGraphBlock(this, tokenSource, content, logger); - JoinDbTags = new DataflowJoinBlock(this, tokenSource, logger, nameof(JoinDbTags)); - LoadDbTags = new LoadDbLatestTagsBlock(this, tokenSource, logger); - LoadGitHubTags = new LoadGitHubTagsBlock(this, tokenSource, github, logger); - JoinTags = new DataflowJoinBlock(this, tokenSource, logger, nameof(JoinTags)); - BroadcastTags = new BroadcastTagsBlock(this, tokenSource, logger); - StreamGitHubTags = new StreamGitHubTagsBlock(this, tokenSource, logger); - GetTagAssets = new GetTagAssetsBlock(this, tokenSource, github, logger); - TagBuffer = new TagBufferBlock(this, tokenSource, logger); - AccumulateProcessedTags = new AccumulateProcessedTagsBlock(this, tokenSource, logger); - SaveTags = new BulkSaveStagingBlock(this, tokenSource, staging, logger); + SynchronizeTags = new SynchronizeTagsBlock(this, tokenSource, logger, tagServices, github); + //ReceiveRequest = new ReceiveRequestBlock(this, tokenSource, logger); + //BroadcastParameters = new BroadcastParametersBlock(this, tokenSource, logger); + //AcquireDbMainTag = new AcquireDbMainTagGraphBlock(this, tokenSource, content, logger); + //AcquireDbNextTag = new AcquireDbNextTagGraphBlock(this, tokenSource, content, logger); + //JoinDbTags = new DataflowJoinBlock(this, tokenSource, logger, nameof(JoinDbTags)); + //LoadDbTags = new LoadDbLatestTagsBlock(this, tokenSource, logger); + //LoadGitHubTags = new LoadGitHubTagsBlock(this, tokenSource, github, logger); + //JoinTags = new DataflowJoinBlock(this, tokenSource, logger, nameof(JoinTags)); + //BroadcastTags = new BroadcastTagsBlock(this, tokenSource, logger); + //StreamGitHubTags = new StreamGitHubTagsBlock(this, tokenSource, logger); + //GetTagAssets = new GetTagAssetsBlock(this, tokenSource, github, logger); + //TagBuffer = new TagBufferBlock(this, tokenSource, logger); + //AccumulateProcessedTags = new AccumulateProcessedTagsBlock(this, tokenSource, logger); + //SaveTags = new BulkSaveStagingBlock(this, tokenSource, staging, logger); } - #region blocks - private ReceiveRequestBlock ReceiveRequest { get; } - private BroadcastParametersBlock BroadcastParameters { get; } - private AcquireDbMainTagGraphBlock AcquireDbMainTag { get; } - private AcquireDbNextTagGraphBlock AcquireDbNextTag { get; } - private DataflowJoinBlock JoinDbTags { get; } - private LoadDbLatestTagsBlock LoadDbTags { get; } - private LoadGitHubTagsBlock LoadGitHubTags { get; } - private DataflowJoinBlock JoinTags { get; } - private BroadcastTagsBlock BroadcastTags { get; } - private StreamGitHubTagsBlock StreamGitHubTags { get; } - private GetTagAssetsBlock GetTagAssets { get; } - private TagBufferBlock TagBuffer { get; } - private AccumulateProcessedTagsBlock AccumulateProcessedTags { get; } - private BulkSaveStagingBlock SaveTags { get; } + //#region blocks + private SynchronizeTagsBlock SynchronizeTags { get; } + //private ReceiveRequestBlock ReceiveRequest { get; } + //private BroadcastParametersBlock BroadcastParameters { get; } + //private AcquireDbMainTagGraphBlock AcquireDbMainTag { get; } + //private AcquireDbNextTagGraphBlock AcquireDbNextTag { get; } + //private DataflowJoinBlock JoinDbTags { get; } + //private LoadDbLatestTagsBlock LoadDbTags { get; } + //private LoadGitHubTagsBlock LoadGitHubTags { get; } + //private DataflowJoinBlock JoinTags { get; } + //private BroadcastTagsBlock BroadcastTags { get; } + //private StreamGitHubTagsBlock StreamGitHubTags { get; } + //private GetTagAssetsBlock GetTagAssets { get; } + //private TagBufferBlock TagBuffer { get; } + //private AccumulateProcessedTagsBlock AccumulateProcessedTags { get; } + //private BulkSaveStagingBlock SaveTags { get; } - public ITargetBlock InputBlock => ReceiveRequest.Block; - public Task OutputTask => SaveTags.Block.Completion; + public ITargetBlock InputBlock => SynchronizeTags.Block!; + public Task OutputTask => SynchronizeTags.Block.Completion; protected override IReadOnlyDictionary Blocks => new Dictionary { - [nameof(ReceiveRequest)] = ReceiveRequest.Block, - [nameof(BroadcastParameters)] = BroadcastParameters.Block, - [nameof(AcquireDbMainTag)] = AcquireDbMainTag.Block, - [nameof(AcquireDbNextTag)] = AcquireDbNextTag.Block, - [nameof(JoinDbTags)] = JoinDbTags.Block, - [nameof(LoadDbTags)] = LoadDbTags.Block, - [nameof(LoadGitHubTags)] = LoadGitHubTags.Block, - [nameof(JoinTags)] = JoinTags.Block, - [nameof(BroadcastTags)] = BroadcastTags.Block, - [nameof(StreamGitHubTags)] = StreamGitHubTags.Block, - [nameof(GetTagAssets)] = GetTagAssets.Block, - [nameof(TagBuffer)] = TagBuffer.Block, - [nameof(AccumulateProcessedTags)] = AccumulateProcessedTags.Block, - [nameof(SaveTags)] = SaveTags.Block, + [nameof(SynchronizeTags)] = SynchronizeTags.Block, + // [nameof(ReceiveRequest)] = ReceiveRequest.Block, + // [nameof(BroadcastParameters)] = BroadcastParameters.Block, + // [nameof(AcquireDbMainTag)] = AcquireDbMainTag.Block, + // [nameof(AcquireDbNextTag)] = AcquireDbNextTag.Block, + // [nameof(JoinDbTags)] = JoinDbTags.Block, + // [nameof(LoadDbTags)] = LoadDbTags.Block, + // [nameof(LoadGitHubTags)] = LoadGitHubTags.Block, + // [nameof(JoinTags)] = JoinTags.Block, + // [nameof(BroadcastTags)] = BroadcastTags.Block, + // [nameof(StreamGitHubTags)] = StreamGitHubTags.Block, + // [nameof(GetTagAssets)] = GetTagAssets.Block, + // [nameof(TagBuffer)] = TagBuffer.Block, + // [nameof(AccumulateProcessedTags)] = AccumulateProcessedTags.Block, + // [nameof(SaveTags)] = SaveTags.Block, }; - #endregion + //#endregion public override void CreateBlocks() { - ReceiveRequest.CreateBlock(); - BroadcastParameters.CreateBlock(ReceiveRequest); - AcquireDbMainTag.CreateBlock(BroadcastParameters); - AcquireDbNextTag.CreateBlock(BroadcastParameters); - JoinDbTags.CreateBlock(AcquireDbMainTag, AcquireDbNextTag); - LoadDbTags.CreateBlock(JoinDbTags); - LoadGitHubTags.CreateBlock(LoadDbTags); - JoinTags.CreateBlock(LoadDbTags, LoadGitHubTags); - BroadcastTags.CreateBlock(JoinTags); - StreamGitHubTags.CreateBlock(BroadcastTags); - GetTagAssets.CreateBlock(StreamGitHubTags); - TagBuffer.CreateBlock(GetTagAssets); - AccumulateProcessedTags.CreateBlock(TagBuffer); - SaveTags.CreateBlock(AccumulateProcessedTags); + SynchronizeTags.CreateBlock(); + //ReceiveRequest.CreateBlock(); + //BroadcastParameters.CreateBlock(ReceiveRequest); + //AcquireDbMainTag.CreateBlock(BroadcastParameters); + //AcquireDbNextTag.CreateBlock(BroadcastParameters); + //JoinDbTags.CreateBlock(AcquireDbMainTag, AcquireDbNextTag); + //LoadDbTags.CreateBlock(JoinDbTags); + //LoadGitHubTags.CreateBlock(LoadDbTags); + //JoinTags.CreateBlock(LoadDbTags, LoadGitHubTags); + //BroadcastTags.CreateBlock(JoinTags); + //StreamGitHubTags.CreateBlock(BroadcastTags); + //GetTagAssets.CreateBlock(StreamGitHubTags); + //TagBuffer.CreateBlock(GetTagAssets); + //AccumulateProcessedTags.CreateBlock(TagBuffer); + //SaveTags.CreateBlock(AccumulateProcessedTags); } } + +public class SynchronizeTagsBlock : ActionBlockBase +{ + private readonly IGitHubClientService _github; + private readonly TagServices _tagServices; + + public SynchronizeTagsBlock(PipelineSection parent, CancellationTokenSource tokenSource, ILogger logger, + TagServices tagServices, + IGitHubClientService github) + : base(parent, tokenSource, logger) + { + _tagServices = tagServices; + _github = github; + } + + protected override async Task ActionAsync(TagSyncRequestParameters input) + { + var getGithubTags = _github.GetAllTagsAsync(); + var dbMain = _tagServices.GetLatestTag(false); + var dbNext = _tagServices.GetLatestTag(true); + + var githubTags = await getGithubTags; + var (gitHubMain, gitHubNext, _) = githubTags.GetLatestTags(); + + var mergedMain = (dbMain ?? gitHubMain with { InstallerDownloads = gitHubMain.InstallerDownloads })!; + var mergedNext = (dbNext ?? gitHubNext with { InstallerDownloads = gitHubNext.InstallerDownloads })!; + + var inserts = new List(); + var updates = new List(); + + if (dbMain is null) + { + inserts.Add(mergedMain); + } + else + { + updates.Add(mergedMain); + } + + if (dbNext is null) + { + inserts.Add(mergedNext); + } + else + { + updates.Add(mergedNext); + } + + if (inserts.Any()) + { + _tagServices.Create(inserts); + } + if (updates.Any()) + { + _tagServices.Update(updates); + } + } +} \ No newline at end of file diff --git a/rubberduckvba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/SyncXmldocSection.cs b/rubberduckvba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/SyncXmldocSection.cs index a08a0b5..5940b9e 100644 --- a/rubberduckvba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/SyncXmldocSection.cs +++ b/rubberduckvba.Server/ContentSynchronization/Pipeline/Sections/SyncXmldoc/SyncXmldocSection.cs @@ -93,7 +93,7 @@ protected override async Task ActionAsync(SyncRequestParameters input) var dbMain = await _content.GetLatestTagAsync(RepositoryId.Rubberduck, includePreRelease: false); Context.LoadRubberduckDbMain(dbMain); - var githubTags = await _github.GetAllTagsAsync(dbMain.Name); + var githubTags = await _github.GetAllTagsAsync(); // LoadInspectionDefaultConfig var config = await _github.GetCodeAnalysisDefaultsConfigAsync(); Context.LoadInspectionDefaultConfig(config); diff --git a/rubberduckvba.Server/ContentSynchronization/Pipeline/SynchronizeTagsPipeline.cs b/rubberduckvba.Server/ContentSynchronization/Pipeline/SynchronizeTagsPipeline.cs index e8ed1ba..4501e83 100644 --- a/rubberduckvba.Server/ContentSynchronization/Pipeline/SynchronizeTagsPipeline.cs +++ b/rubberduckvba.Server/ContentSynchronization/Pipeline/SynchronizeTagsPipeline.cs @@ -3,6 +3,7 @@ using rubberduckvba.Server.ContentSynchronization.Pipeline.Sections.SyncTags; using rubberduckvba.Server.ContentSynchronization.XmlDoc.Abstract; using rubberduckvba.Server.Services; +using rubberduckvba.Server.Services.rubberduckdb; namespace rubberduckvba.Server.ContentSynchronization.Pipeline; @@ -12,14 +13,17 @@ public class SynchronizeTagsPipeline : PipelineBase, ISynchro private readonly IGitHubClientService _github; private readonly IXmlDocMerge _merge; private readonly IStagingServices _staging; + private readonly TagServices _tagServices; - public SynchronizeTagsPipeline(IRequestParameters parameters, ILogger logger, IRubberduckDbService content, IGitHubClientService github, IXmlDocMerge merge, IStagingServices staging, CancellationTokenSource tokenSource) + public SynchronizeTagsPipeline(IRequestParameters parameters, ILogger logger, + IRubberduckDbService content, TagServices tagServices, IGitHubClientService github, IXmlDocMerge merge, IStagingServices staging, CancellationTokenSource tokenSource) : base(new SyncContext(parameters), tokenSource, logger) { _content = content; _github = github; _merge = merge; _staging = staging; + _tagServices = tagServices; } public async Task> ExecuteAsync(SyncRequestParameters parameters, CancellationTokenSource tokenSource) @@ -31,14 +35,14 @@ public async Task> ExecuteAsync(SyncRequestParameters para } // 01. Create the pipeline sections - var synchronizeTags = new SyncTagsSection(this, tokenSource, Logger, _content, _github, _staging); + var synchronizeTags = new SyncTagsSection(this, tokenSource, Logger, _tagServices, _github, _staging); // 02. Wire up the pipeline AddSections(parameters, synchronizeTags); DisposeAfter(synchronizeTags.WhenAllBlocksCompleted); // 03. Light it up - Start(synchronizeTags.InputBlock, parameters); + Start(synchronizeTags.InputBlock, (TagSyncRequestParameters)parameters); // 04. await completion await synchronizeTags.OutputTask; diff --git a/rubberduckvba.Server/Services/GitHubClientService.cs b/rubberduckvba.Server/Services/GitHubClientService.cs index 479c131..76a1b7d 100644 --- a/rubberduckvba.Server/Services/GitHubClientService.cs +++ b/rubberduckvba.Server/Services/GitHubClientService.cs @@ -17,7 +17,7 @@ namespace rubberduckvba.Server.Services; public interface IGitHubClientService { Task ValidateTokenAsync(string token); - Task> GetAllTagsAsync(string? dbMainTagName); + Task> GetAllTagsAsync(); Task GetTagAsync(string? token, string name); Task> GetCodeAnalysisDefaultsConfigAsync(); } @@ -60,7 +60,7 @@ private class ReleaseComparer : IEqualityComparer return new ClaimsPrincipal(identity); } - public async Task> GetAllTagsAsync(string? dbMainTagName) + public async Task> GetAllTagsAsync() { var config = configuration.Value; var credentials = new Credentials(config.OrgToken); @@ -68,10 +68,10 @@ public async Task> GetAllTagsAsync(string? dbMainTagName) var getReleases = client.Repository.Release.GetAll(config.OwnerOrg, config.Rubberduck, new ApiOptions { PageCount = 1, PageSize = 10 }); - var getKnownMain = client.Repository.Release.Get(config.OwnerOrg, config.Rubberduck, dbMainTagName); - await Task.WhenAll(getReleases, getKnownMain); + var getLatest = client.Repository.Release.GetLatest(config.OwnerOrg, config.Rubberduck); + await Task.WhenAll(getReleases, getLatest); - var releases = (await getReleases).Append(await getKnownMain).ToHashSet(new ReleaseComparer()); + var releases = (await getReleases).Append(await getLatest).ToHashSet(new ReleaseComparer()); return (from release in releases let installer = release.Assets.SingleOrDefault(asset => asset.Name.EndsWith(".exe") && asset.Name.StartsWith("Rubberduck.Setup")) diff --git a/rubberduckvba.Server/Services/rubberduckdb/TagServices.cs b/rubberduckvba.Server/Services/rubberduckdb/TagServices.cs index 698d9f0..b20f524 100644 --- a/rubberduckvba.Server/Services/rubberduckdb/TagServices.cs +++ b/rubberduckvba.Server/Services/rubberduckdb/TagServices.cs @@ -32,9 +32,13 @@ public IEnumerable GetLatestTags(IEnumerable allTags) => allTags .SelectMany(tags => tags.Take(1)) .ToList(); - public TagGraph GetLatestTag(bool isPreRelease) + public TagGraph? GetLatestTag(bool isPreRelease) { var latestTags = GetLatestTags(); + if (!latestTags.Any()) + { + return null; + } if (!isPreRelease) {