Skip to content

Commit c6a4e74

Browse files
vflauxvflaux
committed
fix(cloudflare): reduce dns & custom hostnames list api calls
1 parent 2de3b50 commit c6a4e74

File tree

2 files changed

+74
-49
lines changed

2 files changed

+74
-49
lines changed

‎provider/cloudflare/cloudflare.go

Lines changed: 68 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -57,22 +57,60 @@ var proxyEnabled *bool = boolPtr(true)
5757
// proxyDisabled is a pointer to a bool false showing the record should not be proxied through cloudflare
5858
var proxyDisabled *bool = boolPtr(false)
5959

60-
// for faster getRecordID() lookup
60+
// for faster GetRecordID() lookup
6161
type DNSRecordIndex struct {
6262
Name string
6363
Type string
6464
Content string
6565
}
6666

67+
func newDNSRecordIndex(r cloudflare.DNSRecord) DNSRecordIndex {
68+
return DNSRecordIndex{Name: r.Name, Type: r.Type, Content: r.Content}
69+
}
70+
6771
type DNSRecordsMap map[DNSRecordIndex]cloudflare.DNSRecord
6872

69-
// for faster getCustomHostname() lookup
73+
func (m DNSRecordsMap) GetRecordID(record cloudflare.DNSRecord) string {
74+
return m[newDNSRecordIndex(record)].ID
75+
}
76+
77+
func (m DNSRecordsMap) Set(record cloudflare.DNSRecord) {
78+
m[newDNSRecordIndex(record)] = record
79+
}
80+
81+
func (m DNSRecordsMap) Delete(record cloudflare.DNSRecord) {
82+
delete(m, newDNSRecordIndex(record))
83+
}
84+
85+
// for faster GetCustomHostname() lookup
7086
type CustomHostnameIndex struct {
7187
Hostname string
7288
}
7389

90+
func newCustomHostnameIndex(ch cloudflare.CustomHostname) CustomHostnameIndex {
91+
return CustomHostnameIndex{Hostname: ch.Hostname}
92+
}
93+
7494
type CustomHostnamesMap map[CustomHostnameIndex]cloudflare.CustomHostname
7595

96+
func (m CustomHostnamesMap) Get(hostname string) (cloudflare.CustomHostname, error) {
97+
if hostname == "" {
98+
return cloudflare.CustomHostname{}, fmt.Errorf("failed to get custom hostname: %q is empty", hostname)
99+
}
100+
if ch, ok := m[CustomHostnameIndex{Hostname: hostname}]; ok {
101+
return ch, nil
102+
}
103+
return cloudflare.CustomHostname{}, fmt.Errorf("failed to get custom hostname: %q not found", hostname)
104+
}
105+
106+
func (m CustomHostnamesMap) Set(ch cloudflare.CustomHostname) {
107+
m[newCustomHostnameIndex(ch)] = ch
108+
}
109+
110+
func (m CustomHostnamesMap) Delete(ch cloudflare.CustomHostname) {
111+
delete(m, newCustomHostnameIndex(ch))
112+
}
113+
76114
type DataLocalizationRegionalHostnameChange struct {
77115
Action string
78116
cloudflare.RegionalHostname
@@ -434,7 +472,7 @@ func (p *CloudFlareProvider) submitCustomHostnameChanges(ctx context.Context, zo
434472
add, remove, _ := provider.Difference(change.CustomHostnamesPrev, slices.Collect(maps.Keys(change.CustomHostnames)))
435473

436474
for _, changeCH := range remove {
437-
if prevCh, err := getCustomHostname(chs, changeCH); err == nil {
475+
if prevCh, err := chs.Get(changeCH); err == nil {
438476
prevChID := prevCh.ID
439477
if prevChID != "" {
440478
log.WithFields(logFields).Infof("Removing previous custom hostname %q/%q", prevChID, changeCH)
@@ -443,29 +481,33 @@ func (p *CloudFlareProvider) submitCustomHostnameChanges(ctx context.Context, zo
443481
failedChange = true
444482
log.WithFields(logFields).Errorf("failed to remove previous custom hostname %q/%q: %v", prevChID, changeCH, chErr)
445483
}
484+
chs.Delete(prevCh)
446485
}
447486
}
448487
}
449488
for _, changeCH := range add {
450489
log.WithFields(logFields).Infof("Adding custom hostname %q", changeCH)
451-
_, chErr := p.Client.CreateCustomHostname(ctx, zoneID, change.CustomHostnames[changeCH])
490+
ch := change.CustomHostnames[changeCH]
491+
_, chErr := p.Client.CreateCustomHostname(ctx, zoneID, ch)
452492
if chErr != nil {
453493
failedChange = true
454494
log.WithFields(logFields).Errorf("failed to add custom hostname %q: %v", changeCH, chErr)
455495
}
496+
chs.Set(ch)
456497
}
457498
}
458499
case cloudFlareDelete:
459500
for _, changeCH := range change.CustomHostnames {
460501
if recordTypeCustomHostnameSupported[change.ResourceRecord.Type] && changeCH.Hostname != "" {
461502
log.WithFields(logFields).Infof("Deleting custom hostname %q", changeCH.Hostname)
462-
if ch, err := getCustomHostname(chs, changeCH.Hostname); err == nil {
503+
if ch, err := chs.Get(changeCH.Hostname); err == nil {
463504
chID := ch.ID
464505
chErr := p.Client.DeleteCustomHostname(ctx, zoneID, chID)
465506
if chErr != nil {
466507
failedChange = true
467508
log.WithFields(logFields).Errorf("failed to delete custom hostname %q/%q: %v", chID, changeCH.Hostname, chErr)
468509
}
510+
chs.Delete(changeCH)
469511
} else {
470512
log.WithFields(logFields).Warnf("failed to delete custom hostname %q: %v", changeCH.Hostname, err)
471513
}
@@ -475,7 +517,7 @@ func (p *CloudFlareProvider) submitCustomHostnameChanges(ctx context.Context, zo
475517
for _, changeCH := range change.CustomHostnames {
476518
if recordTypeCustomHostnameSupported[change.ResourceRecord.Type] && changeCH.Hostname != "" {
477519
log.WithFields(logFields).Infof("Creating custom hostname %q", changeCH.Hostname)
478-
if ch, err := getCustomHostname(chs, changeCH.Hostname); err == nil {
520+
if ch, err := chs.Get(changeCH.Hostname); err == nil {
479521
if changeCH.CustomOriginServer == ch.CustomOriginServer {
480522
log.WithFields(logFields).Warnf("custom hostname %q already exists with the same origin %q, continue", changeCH.Hostname, ch.CustomOriginServer)
481523
} else {
@@ -489,6 +531,7 @@ func (p *CloudFlareProvider) submitCustomHostnameChanges(ctx context.Context, zo
489531
log.WithFields(logFields).Errorf("failed to create custom hostname %q: %v", changeCH.Hostname, chErr)
490532
}
491533
}
534+
chs.Set(changeCH)
492535
}
493536
}
494537
}
@@ -636,8 +679,22 @@ func (p *CloudFlareProvider) submitChanges(ctx context.Context, changes []*cloud
636679
var failedZones []string
637680
for zoneID, zoneChanges := range changesByZone {
638681
var failedChange bool
682+
var records DNSRecordsMap
683+
var chs CustomHostnamesMap
639684
resourceContainer := cloudflare.ZoneIdentifier(zoneID)
640685

686+
if !p.DryRun {
687+
records, err = p.listDNSRecordsWithAutoPagination(ctx, zoneID)
688+
if err != nil {
689+
return fmt.Errorf("could not fetch records from zone, %w", err)
690+
}
691+
692+
chs, err = p.listCustomHostnamesWithPagination(ctx, zoneID)
693+
if err != nil {
694+
return fmt.Errorf("could not fetch custom hostnames from zone, %v", err)
695+
}
696+
}
697+
641698
for _, change := range zoneChanges {
642699
logFields := log.Fields{
643700
"record": change.ResourceRecord.Name,
@@ -653,19 +710,11 @@ func (p *CloudFlareProvider) submitChanges(ctx context.Context, changes []*cloud
653710
continue
654711
}
655712

656-
records, err := p.listDNSRecordsWithAutoPagination(ctx, zoneID)
657-
if err != nil {
658-
return fmt.Errorf("could not fetch records from zone, %w", err)
659-
}
660-
chs, chErr := p.listCustomHostnamesWithPagination(ctx, zoneID)
661-
if chErr != nil {
662-
return fmt.Errorf("could not fetch custom hostnames from zone, %v", chErr)
663-
}
664713
if change.Action == cloudFlareUpdate {
665714
if !p.submitCustomHostnameChanges(ctx, zoneID, change, chs, logFields) {
666715
failedChange = true
667716
}
668-
recordID := p.getRecordID(records, change.ResourceRecord)
717+
recordID := records.GetRecordID(change.ResourceRecord)
669718
if recordID == "" {
670719
log.WithFields(logFields).Errorf("failed to find previous record: %v", change.ResourceRecord)
671720
continue
@@ -678,7 +727,7 @@ func (p *CloudFlareProvider) submitChanges(ctx context.Context, changes []*cloud
678727
log.WithFields(logFields).Errorf("failed to update record: %v", err)
679728
}
680729
} else if change.Action == cloudFlareDelete {
681-
recordID := p.getRecordID(records, change.ResourceRecord)
730+
recordID := records.GetRecordID(change.ResourceRecord)
682731
if recordID == "" {
683732
log.WithFields(logFields).Errorf("failed to find previous record: %v", change.ResourceRecord)
684733
continue
@@ -688,16 +737,18 @@ func (p *CloudFlareProvider) submitChanges(ctx context.Context, changes []*cloud
688737
failedChange = true
689738
log.WithFields(logFields).Errorf("failed to delete record: %v", err)
690739
}
740+
records.Delete(change.ResourceRecord)
691741
if !p.submitCustomHostnameChanges(ctx, zoneID, change, chs, logFields) {
692742
failedChange = true
693743
}
694744
} else if change.Action == cloudFlareCreate {
695745
recordParam := getCreateDNSRecordParam(*change)
696-
_, err := p.Client.CreateDNSRecord(ctx, resourceContainer, recordParam)
746+
record, err := p.Client.CreateDNSRecord(ctx, resourceContainer, recordParam)
697747
if err != nil {
698748
failedChange = true
699749
log.WithFields(logFields).Errorf("failed to create record: %v", err)
700750
}
751+
records.Set(record)
701752
if !p.submitCustomHostnameChanges(ctx, zoneID, change, chs, logFields) {
702753
failedChange = true
703754
}
@@ -776,23 +827,6 @@ func (p *CloudFlareProvider) changesByZone(zones []cloudflare.Zone, changeSet []
776827
return changes
777828
}
778829

779-
func (p *CloudFlareProvider) getRecordID(records DNSRecordsMap, record cloudflare.DNSRecord) string {
780-
if zoneRecord, ok := records[DNSRecordIndex{Name: record.Name, Type: record.Type, Content: record.Content}]; ok {
781-
return zoneRecord.ID
782-
}
783-
return ""
784-
}
785-
786-
func getCustomHostname(chs CustomHostnamesMap, chName string) (cloudflare.CustomHostname, error) {
787-
if chName == "" {
788-
return cloudflare.CustomHostname{}, fmt.Errorf("failed to get custom hostname: %q is empty", chName)
789-
}
790-
if ch, ok := chs[CustomHostnameIndex{Hostname: chName}]; ok {
791-
return ch, nil
792-
}
793-
return cloudflare.CustomHostname{}, fmt.Errorf("failed to get custom hostname: %q not found", chName)
794-
}
795-
796830
func (p *CloudFlareProvider) newCustomHostname(customHostname string, origin string) cloudflare.CustomHostname {
797831
return cloudflare.CustomHostname{
798832
Hostname: customHostname,
@@ -843,10 +877,6 @@ func (p *CloudFlareProvider) newCloudFlareChange(action string, ep *endpoint.End
843877
}
844878
}
845879

846-
func newDNSRecordIndex(r cloudflare.DNSRecord) DNSRecordIndex {
847-
return DNSRecordIndex{Name: r.Name, Type: r.Type, Content: r.Content}
848-
}
849-
850880
// listDNSRecordsWithAutoPagination performs automatic pagination of results on requests to cloudflare.ListDNSRecords with custom per_page values
851881
func (p *CloudFlareProvider) listDNSRecordsWithAutoPagination(ctx context.Context, zoneID string) (DNSRecordsMap, error) {
852882
// for faster getRecordID lookup
@@ -877,10 +907,6 @@ func (p *CloudFlareProvider) listDNSRecordsWithAutoPagination(ctx context.Contex
877907
return records, nil
878908
}
879909

880-
func newCustomHostnameIndex(ch cloudflare.CustomHostname) CustomHostnameIndex {
881-
return CustomHostnameIndex{Hostname: ch.Hostname}
882-
}
883-
884910
// listCustomHostnamesWithPagination performs automatic pagination of results on requests to cloudflare.CustomHostnames
885911
func (p *CloudFlareProvider) listCustomHostnamesWithPagination(ctx context.Context, zoneID string) (CustomHostnamesMap, error) {
886912
if !p.CustomHostnamesConfig.Enabled {

‎provider/cloudflare/cloudflare_test.go

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1149,7 +1149,6 @@ func TestCloudflareApplyChangesError(t *testing.T) {
11491149
}
11501150

11511151
func TestCloudflareGetRecordID(t *testing.T) {
1152-
p := &CloudFlareProvider{}
11531152
recordsMap := DNSRecordsMap{
11541153
{Name: "foo.com", Type: endpoint.RecordTypeCNAME, Content: "foobar"}: {
11551154
Name: "foo.com",
@@ -1170,29 +1169,29 @@ func TestCloudflareGetRecordID(t *testing.T) {
11701169
},
11711170
}
11721171

1173-
assert.Equal(t, "", p.getRecordID(recordsMap, cloudflare.DNSRecord{
1172+
assert.Equal(t, "", recordsMap.GetRecordID(cloudflare.DNSRecord{
11741173
Name: "foo.com",
11751174
Type: endpoint.RecordTypeA,
11761175
Content: "foobar",
11771176
}))
11781177

1179-
assert.Equal(t, "", p.getRecordID(recordsMap, cloudflare.DNSRecord{
1178+
assert.Equal(t, "", recordsMap.GetRecordID(cloudflare.DNSRecord{
11801179
Name: "foo.com",
11811180
Type: endpoint.RecordTypeCNAME,
11821181
Content: "fizfuz",
11831182
}))
11841183

1185-
assert.Equal(t, "1", p.getRecordID(recordsMap, cloudflare.DNSRecord{
1184+
assert.Equal(t, "1", recordsMap.GetRecordID(cloudflare.DNSRecord{
11861185
Name: "foo.com",
11871186
Type: endpoint.RecordTypeCNAME,
11881187
Content: "foobar",
11891188
}))
1190-
assert.Equal(t, "", p.getRecordID(recordsMap, cloudflare.DNSRecord{
1189+
assert.Equal(t, "", recordsMap.GetRecordID(cloudflare.DNSRecord{
11911190
Name: "bar.de",
11921191
Type: endpoint.RecordTypeA,
11931192
Content: "2.3.4.5",
11941193
}))
1195-
assert.Equal(t, "2", p.getRecordID(recordsMap, cloudflare.DNSRecord{
1194+
assert.Equal(t, "2", recordsMap.GetRecordID(cloudflare.DNSRecord{
11961195
Name: "bar.de",
11971196
Type: endpoint.RecordTypeA,
11981197
Content: "1.2.3.4",
@@ -2706,7 +2705,7 @@ func TestCloudflareCustomHostnameNotFoundOnRecordDeletion(t *testing.T) {
27062705
t.Error(e)
27072706
}
27082707
if tc.preApplyHook == "corrupt" {
2709-
if ch, err := getCustomHostname(chs, "newerror-getCustomHostnameOrigin.foo.fancybar.com"); errors.Is(err, nil) {
2708+
if ch, err := chs.Get("newerror-getCustomHostnameOrigin.foo.fancybar.com"); errors.Is(err, nil) {
27102709
chID := ch.ID
27112710
t.Logf("corrupting custom hostname %q", chID)
27122711
oldIdx := getCustomHostnameIdxByID(client.customHostnames[zoneID], chID)

0 commit comments

Comments
 (0)