Skip to content

Commit ff7a737

Browse files
jamesmontemagnojamesmontemagno
authored
Merge pull request davidfowl#4 from jamesmontemagno/blob-hn-cache
Cache HackerNews in Blob Storage
2 parents daa8f81 + 6135cb7 commit ff7a737

8 files changed

Lines changed: 93 additions & 57 deletions

File tree

‎.github/workflows/build.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@ name: Build
33
on:
44
push:
55
branches:
6-
- main
6+
- '**'
77
pull_request:
88
branches:
9-
- main
9+
- '**'
1010

1111
jobs:
1212
build:

‎.gitignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -492,3 +492,8 @@ $RECYCLE.BIN/
492492
/feedbackfunctions/Properties/serviceDependencies.feedbackfunctions20250414121421.json
493493
/feedbackfunctions/Properties/serviceDependencies.json
494494
/feedbackfunctions/local.settings.json
495+
__azurite_db_blob__.json
496+
__blobstorage__/AzuriteConfig
497+
__blobstorage__/d2f77979-09d4-4fc1-b2b6-dd90abca22bf
498+
__blobstorage__/834d1fd8-4b36-441c-a0a9-a14c1ec20695
499+
__azurite_db_blob_extent__.json

‎README.md‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,29 @@ func start
114114

115115
> Note: Keep your API keys and tokens secure and never commit them to source control.
116116
117+
## Azure Functions local.settings.json
118+
119+
You will need the following:
120+
```json
121+
{
122+
"IsEncrypted": false,
123+
"Values": {
124+
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
125+
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated"
126+
},
127+
"YouTube:ApiKey": "YOUR_API_KEY_HERE",
128+
"Reddit:ClientId": "YOUR_REDDIT_CLIENT_ID",
129+
"Reddit:ClientSecret": "YOUR_REDDIT_CLIENT_SECRET",
130+
"GitHub:AccessToken": "YOUR_ACCESS_TOKEN_HERE",
131+
"Azure:OpenAI:Endpoint": "YOUR_AZURE_OPENAI_ENDPOINT",
132+
"Azure:OpenAI:ApiKey": "YOUR_AZURE_OPENAI_API_KEY",
133+
"Azure:OpenAI:Deployment": "YOUR_DEPLOYMENT_NAME",
134+
"Twitter:BearerToken": "YOUR_TWITTER_BEARER_TOKEN",
135+
"BlueSky:Username": "YOUR_BLUESKY_USERNAME",
136+
"BlueSky:AppPassword": "YOUR_BLUESKY_APP_PASSWORD"
137+
}
138+
```
139+
117140
## License
118141

119142
This project is licensed under the MIT License. See the [LICENSE](./LICENSE) file for details.

‎feedbackfunctions/ContentFeedFunctions.cs‎

Lines changed: 18 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@ public class ContentFeedFunctions
1818
private readonly HackerNewsService _hnService;
1919
private readonly YouTubeService _ytService;
2020
private readonly RedditService _redditService;
21-
private static readonly ConcurrentDictionary<string, (List<HackerNewsItemBasicInfo> Items, DateTime ExpirationTime)> _hnSearchCache = new();
22-
private static readonly TimeSpan _cacheDuration = TimeSpan.FromMinutes(60);
21+
private static readonly TimeSpan _cacheDuration = TimeSpan.FromMinutes(10);
2322

2423
public ContentFeedFunctions(
2524
ILogger<ContentFeedFunctions> logger,
@@ -30,6 +29,7 @@ public ContentFeedFunctions(
3029

3130
_configuration = new ConfigurationBuilder()
3231
.AddUserSecrets<Program>()
32+
.AddJsonFile("local.settings.json")
3333
.Build();
3434
#else
3535

@@ -139,42 +139,27 @@ public async Task<HttpResponseData> GetTrendingRedditThreads(
139139
}
140140
}
141141

142-
private bool TryGetFromCache(string cacheKey, out List<HackerNewsItemBasicInfo>? items)
143-
{
144-
items = null;
145-
if (_hnSearchCache.TryGetValue(cacheKey, out var cacheEntry))
146-
{
147-
if (DateTime.UtcNow <= cacheEntry.ExpirationTime)
148-
{
149-
items = cacheEntry.Items;
150-
return true;
151-
}
152-
_hnSearchCache.TryRemove(cacheKey, out _);
153-
}
154-
return false;
155-
}
156-
157142
[Function("SearchHackerNewsArticles")]
158143
public async Task<HttpResponseData> SearchHackerNewsArticles(
159-
[HttpTrigger(AuthorizationLevel.Function, "get")] HttpRequestData req)
144+
[HttpTrigger(AuthorizationLevel.Function, "get")] HttpRequestData req,
145+
[BlobInput("hackernews-cache/all.json")] string? cachedBlob)
160146
{
161147
_logger.LogInformation("Processing Hacker News search request");
162148

163149
try
164150
{
165-
var cacheKey = "all";
166-
167151
List<HackerNewsItemBasicInfo> matchingItems;
168-
if (!TryGetFromCache(cacheKey, out var cachedItems))
152+
153+
if (!string.IsNullOrEmpty(cachedBlob))
169154
{
170-
matchingItems = await _hnService.SearchByTitleBasicInfo();
171-
_hnSearchCache.TryAdd(cacheKey, (matchingItems, DateTime.UtcNow.Add(_cacheDuration)));
172-
_logger.LogInformation("Cache miss for key: {CacheKey}", cacheKey);
155+
// Use blob cache if available
156+
matchingItems = System.Text.Json.JsonSerializer.Deserialize<List<HackerNewsItemBasicInfo>>(cachedBlob) ?? new List<HackerNewsItemBasicInfo>();
157+
_logger.LogInformation("Blob cache hit");
173158
}
174159
else
175160
{
176-
matchingItems = cachedItems!;
177-
_logger.LogInformation("Cache hit for key: {CacheKey}", cacheKey);
161+
matchingItems = await _hnService.SearchByTitleBasicInfo();
162+
_logger.LogInformation("Blob cache miss, fetched from service");
178163
}
179164

180165
var response = req.CreateResponse(HttpStatusCode.OK);
@@ -191,28 +176,23 @@ public async Task<HttpResponseData> SearchHackerNewsArticles(
191176
}
192177

193178
[Function("CacheHackerNewsArticlesHourly")]
194-
public async Task CacheHackerNewsArticlesHourly(
179+
[BlobOutput("hackernews-cache/all.json")]
180+
public async Task<string> CacheHackerNewsArticlesHourly(
195181
[TimerTrigger("0 0 * * * *")] TimerInfo timerInfo)
196182
{
197183
_logger.LogInformation("Starting hourly Hacker News cache refresh at: {Time}", DateTime.UtcNow);
198184

199185
try
200186
{
201-
var cacheKey = "all";
202-
var items = await _hnService.SearchByTitleBasicInfo();
203-
_hnSearchCache.AddOrUpdate(
204-
cacheKey,
205-
(items, DateTime.UtcNow.Add(_cacheDuration)),
206-
(key, old) => (items, DateTime.UtcNow.Add(_cacheDuration))
207-
);
208-
209-
210-
_logger.LogInformation("Pre-cached Hacker News articles for key: {CacheKey}", cacheKey);
211-
187+
var items = await _hnService.SearchByTitleBasicInfo();
188+
_logger.LogInformation("Pre-cached Hacker News articles");
189+
// Serialize items to JSON for blob output
190+
return System.Text.Json.JsonSerializer.Serialize(items);
212191
}
213192
catch (Exception ex)
214193
{
215194
_logger.LogError(ex, "Error during hourly Hacker News cache refresh");
195+
return string.Empty;
216196
}
217197
}
218198
}

‎feedbackfunctions/FeedbackFunctions.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ public FeedbackFunctions(ILogger<FeedbackFunctions> logger, IConfiguration confi
3737

3838
_configuration = new ConfigurationBuilder()
3939
.AddUserSecrets<Program>()
40+
.AddJsonFile("local.settings.json")
4041
.Build();
4142
#else
4243

‎feedbackfunctions/feedbackfunctions.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
<!-- <PackageReference Include="Microsoft.ApplicationInsights.WorkerService" Version="2.22.0" /> -->
1616
<!-- <PackageReference Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" Version="2.0.0" /> -->
1717
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="2.0.0" />
18+
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs" Version="6.7.0" />
1819
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Timer" Version="4.3.1" />
1920
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.2.0" />
2021
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" Version="2.0.0" />

‎feedbackfunctions/local.settings.json‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,17 @@
11
{
22
"IsEncrypted": false,
33
"Values": {
4-
"AzureWebJobsStorage": "",
4+
"AzureWebJobsStorage": "UseDevelopmentStorage=true",
55
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated"
6-
}
6+
},
7+
"YouTube:ApiKey": "YOUR_API_KEY_HERE",
8+
"Reddit:ClientId": "YOUR_REDDIT_CLIENT_ID",
9+
"Reddit:ClientSecret": "YOUR_REDDIT_CLIENT_SECRET",
10+
"GitHub:AccessToken": "YOUR_ACCESS_TOKEN_HERE",
11+
"Azure:OpenAI:Endpoint": "YOUR_AZURE_OPENAI_ENDPOINT",
12+
"Azure:OpenAI:ApiKey": "YOUR_AZURE_OPENAI_API_KEY",
13+
"Azure:OpenAI:Deployment": "YOUR_DEPLOYMENT_NAME",
14+
"Twitter:BearerToken": "YOUR_TWITTER_BEARER_TOKEN",
15+
"BlueSky:Username": "YOUR_BLUESKY_USERNAME",
16+
"BlueSky:AppPassword": "YOUR_BLUESKY_APP_PASSWORD"
717
}

‎shareddump/Models/HackerNews/HackerNewsService.cs‎

Lines changed: 31 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -87,23 +87,39 @@ public async Task<List<HackerNewsItemBasicInfo>> SearchByTitleBasicInfo()
8787
var topStories = await GetTopStories();
8888
var results = new List<HackerNewsItemBasicInfo>();
8989

90-
foreach (var storyId in topStories)
91-
{
92-
var item = await GetItemData(storyId);
93-
if (!string.IsNullOrWhiteSpace(item?.Title))
90+
var tasks = topStories
91+
.Select((storyId, index) => new { storyId, index })
92+
.Select(async x =>
9493
{
95-
results.Add(new HackerNewsItemBasicInfo
94+
var item = await GetItemData(x.storyId);
95+
if (!string.IsNullOrWhiteSpace(item?.Title))
9696
{
97-
Id = item.Id,
98-
Title = item.Title,
99-
By = item.By ?? string.Empty,
100-
Time = item.Time,
101-
Url = item.Url,
102-
Score = item.Score ?? 0,
103-
Descendants = item.Descendants ?? 0
104-
});
105-
}
106-
}
97+
return new
98+
{
99+
Index = x.index,
100+
Info = new HackerNewsItemBasicInfo
101+
{
102+
Id = item.Id,
103+
Title = item.Title,
104+
By = item.By ?? string.Empty,
105+
Time = item.Time,
106+
Url = item.Url,
107+
Score = item.Score ?? 0,
108+
Descendants = item.Descendants ?? 0
109+
}
110+
};
111+
}
112+
return null;
113+
});
114+
115+
var items = await Task.WhenAll(tasks);
116+
117+
results.AddRange(
118+
items
119+
.Where(i => i?.Info is not null)
120+
.OrderBy(i => i!.Index)
121+
.Select(i => i!.Info!)
122+
);
107123

108124
return results;
109125
}

0 commit comments

Comments
 (0)