diff --git a/go.mod b/go.mod index 229b9daa..4cb6089e 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.32.35 github.com/tidwall/gjson v1.19.0 github.com/tidwall/sjson v1.2.5 + golang.org/x/text v0.40.0 ) require ( @@ -35,5 +36,4 @@ require ( golang.org/x/crypto v0.54.0 // indirect golang.org/x/net v0.57.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.40.0 // indirect ) diff --git a/packages/ssestream/content_type_case_test.go b/packages/ssestream/content_type_case_test.go new file mode 100644 index 00000000..228ceb3b --- /dev/null +++ b/packages/ssestream/content_type_case_test.go @@ -0,0 +1,328 @@ +package ssestream + +import ( + "io" + "net/http" + "strings" + "testing" +) + +func TestRegisterDecoderPreservesCaseSensitiveParameterValues(t *testing.T) { + const ( + mediaType = "application/x-openai-go-test-registration-case" + profileV1 = mediaType + "; profile=\"https://example.com/V1\"" + profileV1Lo = mediaType + "; profile=\"https://example.com/v1\"" + ) + wantDefault := &testDecoder{} + wantProfile := &testDecoder{} + RegisterDecoder(mediaType, func(io.ReadCloser) Decoder { return wantDefault }) + RegisterDecoder(profileV1, func(io.ReadCloser) Decoder { return wantProfile }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(mediaType)) + delete(decoderTypes, decoderContentTypeKey(profileV1)) + }) + + for name, test := range map[string]struct { + contentType string + want Decoder + }{ + "registered uppercase profile": { + contentType: "Application/X-OpenAI-Go-Test-Registration-Case; profile=\"https://example.com/V1\"", + want: wantProfile, + }, + "distinct lowercase profile": { + contentType: profileV1Lo, + want: wantDefault, + }, + } { + t.Run(name, func(t *testing.T) { + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.contentType}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != test.want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + +func TestRegisterDecoderNormalizesCaseInsensitiveParameterComponents(t *testing.T) { + const registered = "application/x-openai-go-test-registration-components; Profile=\"https://example.com/V1\"; Charset=UTF-8" + want := &testDecoder{} + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(registered)) + }) + + decoder := NewDecoder(&http.Response{ + Header: http.Header{ + "Content-Type": {"Application/X-OpenAI-Go-Test-Registration-Components; profile=\"https://example.com/V1\"; charset=utf-8"}, + }, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } +} + +func TestRegisterDecoderNormalizesExternalBodyCaseInsensitiveValues(t *testing.T) { + for name, test := range map[string]struct { + parameter string + registered string + response string + }{ + "access type": { + parameter: "Access-Type", + registered: "LOCAL-FILE", + response: "local-file", + }, + "permission": { + parameter: "Permission", + registered: "READ-WRITE", + response: "read-write", + }, + } { + t.Run(name, func(t *testing.T) { + registered := "message/external-body; " + test.parameter + "=" + test.registered + want := &testDecoder{} + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(registered)) + }) + + decoder := NewDecoder(&http.Response{ + Header: http.Header{ + "Content-Type": {"Message/External-Body; " + strings.ToLower(test.parameter) + "=" + test.response}, + }, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + +func TestRegisterDecoderNormalizesLogicalCaseInsensitiveParameters(t *testing.T) { + for name, test := range map[string]struct { + registered string + response string + }{ + "text plain format": { + registered: "text/plain; Format=FLOWED", + response: "Text/Plain; format=flowed", + }, + "text plain delsp": { + registered: "text/plain; DelSP=YES", + response: "Text/Plain; delsp=yes", + }, + "encoded charset": { + registered: "text/plain; Charset*=US-ASCII'EN'%55TF-8", + response: "Text/Plain; charset*=us-ascii'en'%75tf-8", + }, + "quoted encoded charset": { + registered: "text/plain; Charset*=\"US-ASCII'EN'UTF%2D8\"", + response: "Text/Plain; charset*=\"us-ascii'en'utf%2d8\"", + }, + "multipart related type": { + registered: "multipart/related; Type=\"Application/X-Test\"", + response: "Multipart/Related; type=\"application/x-test\"", + }, + "multipart signed protocol": { + registered: "multipart/signed; Protocol=\"Application/PGP-Signature\"", + response: "Multipart/Signed; protocol=\"application/pgp-signature\"", + }, + "multipart encrypted protocol": { + registered: "multipart/encrypted; Protocol=\"Application/PGP-Encrypted\"", + response: "Multipart/Encrypted; protocol=\"application/pgp-encrypted\"", + }, + "multipart report type": { + registered: "multipart/report; Report-Type=DELIVERY-STATUS", + response: "Multipart/Report; report-type=delivery-status", + }, + "unencoded format continuation": { + registered: "text/plain; Format*0=FLO; Format*1=WED", + response: "Text/Plain; format*0=flo; format*1=wed", + }, + "encoded access type continuation": { + registered: "message/external-body; Access-Type*0*=US-ASCII''LOCAL-; Access-Type*1*=FILE", + response: "Message/External-Body; access-type*0*=us-ascii''local-; access-type*1*=file", + }, + } { + t.Run(name, func(t *testing.T) { + want := &testDecoder{} + RegisterDecoder(test.registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(test.registered)) + }) + + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.response}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + +func TestDecoderContentTypeKeyKeepsContextualValuesCaseSensitive(t *testing.T) { + for name, test := range map[string]struct { + contentType string + want string + }{ + "format outside text plain": { + contentType: "text/html; Format=FLOWED", + want: "text/html; format=FLOWED", + }, + "report type outside multipart report": { + contentType: "multipart/mixed; Report-Type=DELIVERY-STATUS", + want: "multipart/mixed; report-type=DELIVERY-STATUS", + }, + "header outside text csv": { + contentType: "text/plain; Header=PRESENT", + want: "text/plain; header=PRESENT", + }, + "micalg outside multipart signed": { + contentType: "multipart/encrypted; Micalg=PGP-SHA256", + want: "multipart/encrypted; micalg=PGP-SHA256", + }, + } { + t.Run(name, func(t *testing.T) { + if got := decoderContentTypeKey(test.contentType); got != test.want { + t.Fatalf("decoder content type key = %q, want %q", got, test.want) + } + }) + } +} + +func TestRegisterDecoderExtendedParameterPreservesUnescapedValueCase(t *testing.T) { + const ( + mediaType = "application/x-openai-go-test-registration-extended" + variantV1 = mediaType + "; Variant*=ISO-8859-1'EN'caf%E9V1" + variantv1 = mediaType + "; variant*=iso-8859-1'en'caf%e9v1" + ) + wantDefault := &testDecoder{} + wantVariant := &testDecoder{} + RegisterDecoder(mediaType, func(io.ReadCloser) Decoder { return wantDefault }) + RegisterDecoder(variantV1, func(io.ReadCloser) Decoder { return wantVariant }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(mediaType)) + delete(decoderTypes, decoderContentTypeKey(variantV1)) + }) + + for name, test := range map[string]struct { + contentType string + want Decoder + }{ + "equivalent encoding case": { + contentType: mediaType + "; variant*=iso-8859-1'en'caf%e9V1", + want: wantVariant, + }, + "distinct unescaped value case": { + contentType: variantv1, + want: wantDefault, + }, + } { + t.Run(name, func(t *testing.T) { + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.contentType}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != test.want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + +func TestRegisterDecoderEncodedContinuationPreservesLaterSegmentValueCase(t *testing.T) { + const ( + mediaType = "application/x-openai-go-test-registration-encoded-continuation" + titleV1 = mediaType + "; title*0*=us-ascii''prefix; title*1*=Bob%2D's'V1" + titlev1 = mediaType + "; title*0*=us-ascii''prefix; title*1*=bob%2d's'V1" + ) + wantDefault := &testDecoder{} + wantTitle := &testDecoder{} + RegisterDecoder(mediaType, func(io.ReadCloser) Decoder { return wantDefault }) + RegisterDecoder(titleV1, func(io.ReadCloser) Decoder { return wantTitle }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(mediaType)) + delete(decoderTypes, decoderContentTypeKey(titleV1)) + }) + + for name, test := range map[string]struct { + contentType string + want Decoder + }{ + "equivalent percent encoding case": { + contentType: mediaType + "; title*0*=us-ascii''prefix; title*1*=Bob%2d's'V1", + want: wantTitle, + }, + "distinct later segment data case": { + contentType: titlev1, + want: wantDefault, + }, + } { + t.Run(name, func(t *testing.T) { + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.contentType}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != test.want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + +func TestRegisterDecoderUnencodedContinuationPreservesValueCase(t *testing.T) { + const ( + mediaType = "application/x-openai-go-test-registration-continuation" + upper = mediaType + "; title*0=V%AB" + lower = mediaType + "; title*0=V%ab" + ) + wantDefault := &testDecoder{} + wantUpper := &testDecoder{} + RegisterDecoder(mediaType, func(io.ReadCloser) Decoder { return wantDefault }) + RegisterDecoder(upper, func(io.ReadCloser) Decoder { return wantUpper }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(mediaType)) + delete(decoderTypes, decoderContentTypeKey(upper)) + }) + + for name, test := range map[string]struct { + contentType string + want Decoder + }{ + "registered uppercase escape text": { + contentType: upper, + want: wantUpper, + }, + "distinct lowercase escape text": { + contentType: lower, + want: wantDefault, + }, + } { + t.Run(name, func(t *testing.T) { + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.contentType}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != test.want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + +func TestDecoderContentTypeKeyDoesNotSplitQuotedSemicolons(t *testing.T) { + const contentType = "Application/X-OpenAI-Go-Test-Quoted; Profile=\"https://example.com/a;b?x*=V1\"; Charset=UTF-8" + got := decoderContentTypeKey(contentType) + want := "application/x-openai-go-test-quoted; profile=\"https://example.com/a;b?x*=V1\"; charset=utf-8" + if got != want { + t.Fatalf("decoder content type key = %q, want %q", got, want) + } +} diff --git a/packages/ssestream/content_type_semantics_test.go b/packages/ssestream/content_type_semantics_test.go new file mode 100644 index 00000000..1382c9aa --- /dev/null +++ b/packages/ssestream/content_type_semantics_test.go @@ -0,0 +1,417 @@ +package ssestream + +import ( + "io" + "net/http" + "strings" + "testing" +) + +func TestRegisterDecoderDoesNotFoldProtocolDefinedOrExtensionValues(t *testing.T) { + for name, test := range map[string]struct { + base string + registered string + response string + }{ + "multipart signed protocol-defined micalg": { + base: "multipart/signed", + // RFC 1847 delegates micalg value semantics to the selected protocol. + // This extension protocol intentionally treats V1 and v1 as distinct. + registered: `multipart/signed; protocol="application/x-test-signature"; micalg=V1`, + response: `multipart/signed; protocol="application/x-test-signature"; micalg=v1`, + }, + "text csv header": { + base: "text/csv", + registered: "text/csv; header=PRESENT", + response: "text/csv; header=present", + }, + "external body extension mode": { + base: "message/external-body", + registered: "message/external-body; access-type=X-TEST; mode=V1", + response: "message/external-body; access-type=x-test; mode=v1", + }, + "external body extended extension mode": { + base: "message/external-body", + registered: "message/external-body; access-type*=ISO-8859-1''X-TEST; mode=V1", + response: "message/external-body; access-type*=iso-8859-1''x-test; mode=v1", + }, + "external body utf16 extension mode": { + base: "message/external-body", + registered: "message/external-body; access-type*=UTF-16BE''%00X%00-%00T%00E%00S%00T; mode=V1", + response: "message/external-body; access-type*=utf-16be''%00x%00-%00t%00e%00s%00t; mode=v1", + }, + "external body unsupported utf32 mode": { + base: "message/external-body", + registered: "message/external-body; access-type*=UTF-32BE''%00%00%00F%00%00%00T%00%00%00P; mode=IMAGE", + response: "message/external-body; access-type*=utf-32be''%00%00%00f%00%00%00t%00%00%00p; mode=image", + }, + "smime type": { + base: "application/pkcs7-mime", + registered: "application/pkcs7-mime; smime-type=SIGNED-DATA", + response: "application/pkcs7-mime; smime-type=signed-data", + }, + } { + t.Run(name, func(t *testing.T) { + wantBare := &testDecoder{} + wantSpecific := &testDecoder{} + RegisterDecoder(test.base, func(io.ReadCloser) Decoder { return wantBare }) + RegisterDecoder(test.registered, func(io.ReadCloser) Decoder { return wantSpecific }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(test.base)) + delete(decoderTypes, decoderContentTypeKey(test.registered)) + }) + + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.response}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != wantBare { + t.Fatalf("decoder = %T, want bare decoder for distinct parameter value", decoder) + } + }) + } +} + +func TestRegisterDecoderDoesNotFoldExternalBodyModeForMalformedContinuation(t *testing.T) { + const base = "message/external-body" + for name, accessType := range map[string]string{ + "malformed section zero": "access-type*0*=UTF-8''%ZZ; access-type*1*=FTP", + "malformed later section": "access-type*0*=UTF-8''FTP; access-type*1*=%ZZ", + "missing section": "access-type*0*=UTF-8''FTP; access-type*2*=X", + "missing metadata": "access-type*0*=BROKEN; access-type*1*=FTP", + "invalid charset syntax": `access-type*="BAD CHAR''FTP"`, + "invalid language syntax": `access-type*="UTF-8'BAD LANG'FTP"`, + } { + t.Run(name, func(t *testing.T) { + registered := base + "; " + accessType + "; mode=IMAGE" + response := base + "; " + accessType + "; mode=image" + wantBare := &testDecoder{} + wantSpecific := &testDecoder{} + RegisterDecoder(base, func(io.ReadCloser) Decoder { return wantBare }) + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return wantSpecific }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(base)) + delete(decoderTypes, decoderContentTypeKey(registered)) + }) + + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {response}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != wantBare { + t.Fatalf("decoder = %T, want bare decoder for malformed continuation", decoder) + } + }) + } +} + +func TestRegisterDecoderFoldsExternalBodyModeWithUnsupportedExtendedCharset(t *testing.T) { + for name, test := range map[string]struct { + registered string + response string + }{ + "single extended value": { + registered: "message/external-body; access-type*=ISO-8859-1''FTP; mode=IMAGE", + response: "Message/External-Body; access-type*=iso-8859-1''ftp; mode=image", + }, + "continued extended value": { + registered: "message/external-body; access-type*0*=ISO-8859-1''ANON%2D; access-type*1*=FTP; mode=IMAGE", + response: "Message/External-Body; access-type*0*=iso-8859-1''anon%2d; access-type*1*=ftp; mode=image", + }, + "quoted extended value": { + registered: "message/external-body; access-type*=\"ISO-8859-1''TFTP\"; mode=IMAGE", + response: "Message/External-Body; access-type*=\"iso-8859-1''tftp\"; mode=image", + }, + "mode before access type": { + registered: "message/external-body; mode=IMAGE; access-type*=ISO-8859-1''FTP", + response: "Message/External-Body; mode=image; access-type*=iso-8859-1''ftp", + }, + "empty charset and language": { + registered: "message/external-body; access-type*=''FTP; mode=IMAGE", + response: "Message/External-Body; access-type*=''ftp; mode=image", + }, + "valid language tag": { + registered: "message/external-body; access-type*=UTF-8'en-US'FTP; mode=IMAGE", + response: "Message/External-Body; access-type*=utf-8'EN-us'ftp; mode=image", + }, + "registered charset alias with period": { + registered: "message/external-body; access-type*=ANSI_X3.4-1968''FTP; mode=IMAGE", + response: "Message/External-Body; access-type*=ansi_x3.4-1968''ftp; mode=image", + }, + "utf16be value": { + registered: "message/external-body; access-type*=UTF-16BE''%00F%00T%00P; mode=IMAGE", + response: "Message/External-Body; access-type*=utf-16be''%00f%00t%00p; mode=image", + }, + "utf16be continued value": { + registered: "message/external-body; access-type*0*=UTF-16BE''%00F%00; access-type*1*=T%00P; mode=IMAGE", + response: "Message/External-Body; access-type*0*=utf-16be''%00f%00; access-type*1*=t%00p; mode=image", + }, + "ebcdic value": { + registered: "message/external-body; access-type*=IBM037''%C6%E3%D7; mode=IMAGE", + response: "Message/External-Body; access-type*=ibm037''%86%A3%97; mode=image", + }, + } { + t.Run(name, func(t *testing.T) { + want := &testDecoder{} + RegisterDecoder(test.registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(test.registered)) + }) + + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.response}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + +func TestRegisterDecoderExternalBodyAccessTypeKeysDoNotCollide(t *testing.T) { + tests := map[string]struct { + first string + second string + }{ + "decoded extended delimiter": { + first: "message/external-body;access-type*=UTF-8''FTP%3Bmode%3DIMAGE", + second: "message/external-body;access-type*=UTF-8''FTP;mode=image", + }, + "quoted ordinary delimiter": { + first: `message/external-body;access-type="FTP;mode=IMAGE"`, + second: "message/external-body;access-type=FTP;mode=image", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + firstDecoder := &testDecoder{} + secondDecoder := &testDecoder{} + RegisterDecoder(test.first, func(io.ReadCloser) Decoder { return firstDecoder }) + RegisterDecoder(test.second, func(io.ReadCloser) Decoder { return secondDecoder }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(test.first)) + delete(decoderTypes, decoderContentTypeKey(test.second)) + }) + + if firstKey, secondKey := decoderContentTypeKey(test.first), decoderContentTypeKey(test.second); firstKey == secondKey { + t.Fatalf("distinct Content-Type values share decoder key %q", firstKey) + } + + for label, response := range map[string]struct { + contentType string + want Decoder + }{ + "first": {contentType: test.first, want: firstDecoder}, + "second": {contentType: test.second, want: secondDecoder}, + } { + t.Run(label, func(t *testing.T) { + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {response.contentType}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != response.want { + t.Fatalf("decoder = %T, want independently registered decoder", decoder) + } + }) + } + }) + } +} + +func TestRegisterDecoderDoesNotFoldExternalBodyModeForAmbiguousAccessType(t *testing.T) { + const base = "message/external-body" + for name, accessTypes := range map[string]string{ + "plain then extended": "access-type=X-TEST; access-type*=UTF-8''FTP", + "extended then plain": "access-type*=UTF-8''FTP; access-type=X-TEST", + } { + t.Run(name, func(t *testing.T) { + registered := base + "; " + accessTypes + "; mode=IMAGE" + response := base + "; " + accessTypes + "; mode=image" + wantSpecific := &testDecoder{} + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return wantSpecific }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(registered)) + }) + + if registeredKey, responseKey := decoderContentTypeKey(registered), decoderContentTypeKey(response); registeredKey == responseKey { + t.Fatalf("ambiguous duplicate access-type folded mode into shared key %q", registeredKey) + } + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {response}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder == wantSpecific { + t.Fatal("ambiguous duplicate access-type selected parameter-specific decoder") + } + }) + } +} + +func TestRegisterDecoderFoldsCaseInsensitiveExtendedValuesAcrossCharsets(t *testing.T) { + for name, test := range map[string]struct { + registered string + response string + }{ + "charset": { + registered: `text/plain; charset*=IBM037''%E4%E3%C6%60%F8`, + response: `text/plain; charset*=ibm037''%A4%A3%86%60%F8`, + }, + "external permission": { + registered: `message/external-body; permission*=IBM037''%D9%C5%C1%C4`, + response: `message/external-body; permission*=ibm037''%99%85%81%84`, + }, + "external mode": { + registered: `message/external-body; access-type=FTP; mode*=IBM037''%C9%D4%C1%C7%C5`, + response: `message/external-body; access-type=ftp; mode*=ibm037''%89%94%81%87%85`, + }, + "multipart encrypted protocol": { + registered: `multipart/encrypted; protocol*=IBM037''%C1%D7%D7%D3%C9%C3%C1%E3%C9%D6%D5%61%E3%C5%E2%E3`, + response: `multipart/encrypted; protocol*=ibm037''%81%97%97%93%89%83%81%A3%89%96%95%61%A3%85%A2%A3`, + }, + "multipart signed protocol": { + registered: `multipart/signed; protocol*=IBM037''%C1%D7%D7%D3%C9%C3%C1%E3%C9%D6%D5%61%E3%C5%E2%E3`, + response: `multipart/signed; protocol*=ibm037''%81%97%97%93%89%83%81%A3%89%96%95%61%A3%85%A2%A3`, + }, + "multipart report type": { + registered: `multipart/report; report-type*=IBM037''%C4%C5%D3%C9%E5%C5%D9%E8%60%E2%E3%C1%E3%E4%E2`, + response: `multipart/report; report-type*=ibm037''%84%85%93%89%A5%85%99%A8%60%A2%A3%81%A3%A4%A2`, + }, + "multipart related type": { + registered: `multipart/related; type*=IBM037''%C1%D7%D7%D3%C9%C3%C1%E3%C9%D6%D5%61%E3%C5%E2%E3`, + response: `multipart/related; type*=ibm037''%81%97%97%93%89%83%81%A3%89%96%95%61%A3%85%A2%A3`, + }, + "text plain format": { + registered: `text/plain; format*=IBM037''%C6%D3%D6%E6%C5%C4`, + response: `text/plain; format*=ibm037''%86%93%96%A6%85%84`, + }, + "text plain delsp": { + registered: `text/plain; delsp*=IBM037''%E8%C5%E2`, + response: `text/plain; delsp*=ibm037''%A8%85%A2`, + }, + } { + t.Run(name, func(t *testing.T) { + want := &testDecoder{} + RegisterDecoder(test.registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { delete(decoderTypes, decoderContentTypeKey(test.registered)) }) + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.response}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + +func TestRegisterDecoderFoldsExternalBodyModeForStandardAccessTypes(t *testing.T) { + for _, accessType := range []string{"FTP", "ANON-FTP", "TFTP"} { + t.Run(accessType, func(t *testing.T) { + registered := "message/external-body; access-type=" + accessType + "; mode=IMAGE" + want := &testDecoder{} + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(registered)) + }) + + decoder := NewDecoder(&http.Response{ + Header: http.Header{ + "Content-Type": {"Message/External-Body; access-type=" + strings.ToLower(accessType) + "; mode=image"}, + }, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + +func TestRegisterDecoderRejectsMalformedUnquotedExtendedValueCollision(t *testing.T) { + registered := `multipart/signed; protocol*="UTF-8''application/pgp-signature"` + malformed := `multipart/signed; protocol*=UTF-8''application/pgp-signature` + want := &testDecoder{} + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { delete(decoderTypes, decoderContentTypeKey(registered)) }) + + if registeredKey, malformedKey := decoderContentTypeKey(registered), decoderContentTypeKey(malformed); registeredKey == malformedKey { + t.Fatalf("malformed unquoted extended value shares decoder key %q", registeredKey) + } + decoder := NewDecoder(&http.Response{Header: http.Header{"Content-Type": {malformed}}, Body: io.NopCloser(strings.NewReader(""))}) + if decoder == want { + t.Fatal("malformed unquoted extended value selected parameter-specific decoder") + } +} + +func TestRegisterDecoderFoldsTextCalendarMIMEParameters(t *testing.T) { + for name, test := range map[string]struct{ registered, response string }{ + "component": { + registered: "text/calendar; component=VEVENT", + response: "Text/Calendar; component=vevent", + }, + "method": { + registered: "text/calendar; method=REQUEST", + response: "Text/Calendar; method=request", + }, + } { + t.Run(name, func(t *testing.T) { + want := &testDecoder{} + RegisterDecoder(test.registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { delete(decoderTypes, decoderContentTypeKey(test.registered)) }) + decoder := NewDecoder(&http.Response{Header: http.Header{"Content-Type": {test.response}}, Body: io.NopCloser(strings.NewReader(""))}) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + +func TestRegisterDecoderDecodesMixedRFC2231ContinuationSegments(t *testing.T) { + registered := "text/plain; charset*0*=UTF-16BE''%00U; charset*1=TF-8" + response := "Text/Plain; charset*0*=utf-16be''%00u; charset*1=tf-8" + want := &testDecoder{} + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { delete(decoderTypes, decoderContentTypeKey(registered)) }) + decoder := NewDecoder(&http.Response{Header: http.Header{"Content-Type": {response}}, Body: io.NopCloser(strings.NewReader(""))}) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } +} + +func TestRegisterDecoderPreservesRFC2231LanguageIdentity(t *testing.T) { + en := "text/plain; format*=UTF-8'en'FLOWED" + fr := "text/plain; format*=UTF-8'fr'FLOWED" + enResponse := "Text/Plain; format*=utf-8'EN'flowed" + wantEN := &testDecoder{} + wantFR := &testDecoder{} + RegisterDecoder(en, func(io.ReadCloser) Decoder { return wantEN }) + RegisterDecoder(fr, func(io.ReadCloser) Decoder { return wantFR }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(en)) + delete(decoderTypes, decoderContentTypeKey(fr)) + }) + if enKey, frKey := decoderContentTypeKey(en), decoderContentTypeKey(fr); enKey == frKey { + t.Fatalf("distinct language tags share decoder key %q", enKey) + } + decoder := NewDecoder(&http.Response{Header: http.Header{"Content-Type": {enResponse}}, Body: io.NopCloser(strings.NewReader(""))}) + if decoder != wantEN { + t.Fatalf("decoder = %T, want English registration", decoder) + } +} + +func TestRegisterDecoderDoesNotFoldInvalidUnicodeIntoMIMEProtocol(t *testing.T) { + registered := "multipart/signed; protocol*=UTF-8''application%2FK" + malformed := "multipart/signed; protocol*=UTF-8''application%2F%E2%84%AA" + want := &testDecoder{} + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { delete(decoderTypes, decoderContentTypeKey(registered)) }) + if registeredKey, malformedKey := decoderContentTypeKey(registered), decoderContentTypeKey(malformed); registeredKey == malformedKey { + t.Fatalf("invalid Unicode protocol shares valid MIME decoder key %q", registeredKey) + } + decoder := NewDecoder(&http.Response{Header: http.Header{"Content-Type": {malformed}}, Body: io.NopCloser(strings.NewReader(""))}) + if decoder == want { + t.Fatal("invalid Unicode protocol selected valid parameter-specific decoder") + } +} diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index 1e9e16bd..9c2f1cba 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -3,17 +3,20 @@ package ssestream import ( "bufio" "bytes" + "encoding/hex" "encoding/json" "fmt" "io" "mime" "net/http" + "strconv" "strings" "sync" "sync/atomic" shimjson "github.com/openai/openai-go/v3/internal/encoding/json" "github.com/tidwall/gjson" + "golang.org/x/text/encoding/ianaindex" ) type Decoder interface { @@ -49,12 +52,11 @@ func NewDecoder(res *http.Response) Decoder { var decoderTypes = map[string](func(io.ReadCloser) Decoder){} func RegisterDecoder(contentType string, decoder func(io.ReadCloser) Decoder) { - decoderTypes[strings.ToLower(contentType)] = decoder + decoderTypes[decoderContentTypeKey(contentType)] = decoder } func decoderContentTypes(contentType string) (string, string) { - base, _, _ := strings.Cut(contentType, ";") - exactType := strings.ToLower(base) + contentType[len(base):] + exactType := decoderContentTypeKey(contentType) mediaType, _, err := mime.ParseMediaType(contentType) if err != nil { @@ -63,6 +65,751 @@ func decoderContentTypes(contentType string) (string, string) { return exactType, mediaType } +// decoderContentTypeKey normalizes only MIME components whose case is +// semantically insignificant. Parameter values remain case-sensitive unless +// their parameter defines otherwise. Extended parameter percent-encoding is +// normalized without changing unescaped value bytes. +func decoderContentTypeKey(contentType string) string { + base, params, found := strings.Cut(contentType, ";") + if !found { + return asciiLower(contentType) + } + normalizedBase := asciiLower(base) + externalBodyAccessType := "" + externalBodyAccessLanguage := "" + hasExternalBodyAccessType := false + if strings.EqualFold(strings.TrimSpace(normalizedBase), "message/external-body") { + externalBodyAccessType, externalBodyAccessLanguage, hasExternalBodyAccessType = parseExternalBodyAccessType(contentType, params) + } + return normalizedBase + ";" + normalizeMediaParameterTail(normalizedBase, params, externalBodyAccessType, externalBodyAccessLanguage, hasExternalBodyAccessType) +} + +func parseExternalBodyAccessType(contentType string, params string) (string, string, bool) { + if accessType, language, found, ok := decodeExtendedMediaParameter(params, "access-type"); found { + if !ok || !isMIMEToken(accessType) { + return "", "", false + } + return asciiLower(accessType), asciiLower(language), true + } + + _, parsedParams, err := mime.ParseMediaType(contentType) + if err != nil { + return "", "", false + } + accessType, ok := parsedParams["access-type"] + if !ok || !isMIMEToken(accessType) { + return "", "", false + } + return asciiLower(accessType), "", true +} + +type extendedMediaParameterSegment struct { + encoded bool + value string +} + +func decodeExtendedMediaParameter(params string, logicalName string) (string, string, bool, bool) { + var single extendedMediaParameterSegment + hasSingle := false + sections := map[int]extendedMediaParameterSegment{} + plainFound := false + found := false + valid := true + + forEachMediaParameter(params, func(param string) { + equals := strings.IndexByte(param, '=') + if !valid || equals < 0 { + return + } + nameStart, nameEnd := trimOWSBounds(param[:equals]) + name := param[nameStart:nameEnd] + if !strings.EqualFold(mediaParameterLogicalName(name), logicalName) { + return + } + if strings.EqualFold(name, logicalName) { + if found { + valid = false + } + plainFound = true + return + } + if plainFound { + found = true + valid = false + return + } + found = true + + encoded := strings.HasSuffix(name, "*") + sectionName := strings.TrimSuffix(name, "*") + segment := extendedMediaParameterSegment{encoded: encoded, value: param[equals+1:]} + if strings.EqualFold(sectionName, logicalName) { + if hasSingle || len(sections) != 0 || !encoded { + valid = false + return + } + single = segment + hasSingle = true + return + } + + star := strings.LastIndexByte(sectionName, '*') + if star < 0 { + valid = false + return + } + section, err := strconv.Atoi(sectionName[star+1:]) + if err != nil || hasSingle { + valid = false + return + } + if _, duplicate := sections[section]; duplicate { + valid = false + return + } + sections[section] = segment + }) + + if !found { + return "", "", false, false + } + if !valid { + return "", "", true, false + } + if hasSingle { + core, quoted, ok := mediaParameterValueCore(single.value) + if !ok { + return "", "", true, false + } + charset, language, data, ok := splitExtendedInitialValue(core) + if !ok || (!quoted && !validRFC2231ExtendedData(data)) { + return "", "", true, false + } + raw, ok := decodeExtendedOctets(data) + if !ok { + return "", "", true, false + } + decoded, ok := decodeMIMEParameterValue(charset, raw) + return decoded, language, true, ok + } + if len(sections) == 0 { + return "", "", true, false + } + + var decoded strings.Builder + var encodedRun []byte + charset := "" + language := "" + flushEncoded := func() bool { + if len(encodedRun) == 0 { + return true + } + text, ok := decodeMIMEParameterValue(charset, encodedRun) + if !ok { + return false + } + decoded.WriteString(text) + encodedRun = encodedRun[:0] + return true + } + + for section := 0; section < len(sections); section++ { + segment, ok := sections[section] + if !ok { + return "", "", true, false + } + core, quoted, ok := mediaParameterValueCore(segment.value) + if !ok { + return "", "", true, false + } + data := core + if section == 0 && segment.encoded { + charset, language, data, ok = splitExtendedInitialValue(core) + if !ok { + return "", "", true, false + } + } + + if segment.encoded { + if !quoted && !validRFC2231ExtendedData(data) { + return "", "", true, false + } + octets, ok := decodeExtendedOctets(data) + if !ok { + return "", "", true, false + } + encodedRun = append(encodedRun, octets...) + continue + } + + if !quoted && !isMIMEToken(data) { + return "", "", true, false + } + if !flushEncoded() { + return "", "", true, false + } + decoded.WriteString(data) + } + if !flushEncoded() { + return "", "", true, false + } + return decoded.String(), language, true, true +} + +func mediaParameterValueCore(value string) (string, bool, bool) { + valueStart, valueEnd := trimOWSBounds(value) + core := value[valueStart:valueEnd] + if !strings.HasPrefix(core, "\"") { + return core, false, true + } + contents, ok := quotedMediaParameterContents(core) + if !ok { + return "", false, false + } + + var decoded strings.Builder + for i := 0; i < len(contents); i++ { + if contents[i] == '\\' { + if i+1 >= len(contents) { + return "", false, false + } + i++ + } + decoded.WriteByte(contents[i]) + } + return decoded.String(), true, true +} + +func splitExtendedInitialValue(value string) (string, string, string, bool) { + firstQuote := strings.IndexByte(value, '\'') + if firstQuote < 0 { + return "", "", "", false + } + secondOffset := strings.IndexByte(value[firstQuote+1:], '\'') + if secondOffset < 0 { + return "", "", "", false + } + secondQuote := firstQuote + secondOffset + 1 + charset := value[:firstQuote] + language := value[firstQuote+1 : secondQuote] + if charset != "" { + if charset != strings.TrimSpace(charset) { + return "", "", "", false + } + if _, err := ianaindex.IANA.Encoding(charset); err != nil { + return "", "", "", false + } + } + if language != "" && !isRFC1766LanguageTag(language) { + return "", "", "", false + } + return charset, language, value[secondQuote+1:], true +} + +func validRFC2231ExtendedData(value string) bool { + for i := 0; i < len(value); i++ { + if value[i] == '%' { + if i+2 >= len(value) || !isHexDigit(value[i+1]) || !isHexDigit(value[i+2]) { + return false + } + i += 2 + continue + } + if !isRFC2231AttributeChar(value[i]) { + return false + } + } + return true +} + +func isRFC2231AttributeChar(value byte) bool { + return isMIMETokenChar(value) && value != '*' && value != '\'' && value != '%' +} + +func isMIMEToken(value string) bool { + if value == "" { + return false + } + for i := 0; i < len(value); i++ { + if !isMIMETokenChar(value[i]) { + return false + } + } + return true +} + +func isMIMETokenChar(value byte) bool { + if value <= ' ' || value >= 0x7f { + return false + } + switch value { + case '(', ')', '<', '>', '@', ',', ';', ':', '\\', '"', '/', '[', ']', '?', '=': + return false + default: + return true + } +} + +func validMIMEMediaType(value string) bool { + typePart, subtype, found := strings.Cut(value, "/") + return found && !strings.Contains(subtype, "/") && isMIMEToken(typePart) && isMIMEToken(subtype) +} + +func asciiLower(value string) string { + bytes := []byte(value) + for i := range bytes { + if bytes[i] >= 'A' && bytes[i] <= 'Z' { + bytes[i] += 'a' - 'A' + } + } + return string(bytes) +} + +func decodeExtendedOctets(value string) ([]byte, bool) { + decoded := make([]byte, 0, len(value)) + for i := 0; i < len(value); i++ { + if value[i] != '%' { + decoded = append(decoded, value[i]) + continue + } + if i+2 >= len(value) || !isHexDigit(value[i+1]) || !isHexDigit(value[i+2]) { + return nil, false + } + decoded = append(decoded, hexValue(value[i+1])<<4|hexValue(value[i+2])) + i += 2 + } + return decoded, true +} + +func decodeMIMEParameterValue(charset string, value []byte) (string, bool) { + if charset == "" { + return string(value), true + } + encoding, err := ianaindex.IANA.Encoding(charset) + if err != nil || encoding == nil { + return "", false + } + decoded, err := encoding.NewDecoder().Bytes(value) + if err != nil { + return "", false + } + return string(decoded), true +} + +func isRFC1766LanguageTag(language string) bool { + partLength := 0 + for i := 0; i <= len(language); i++ { + if i == len(language) || language[i] == '-' { + if partLength == 0 || partLength > 8 { + return false + } + partLength = 0 + continue + } + c := language[i] + if c < 'A' || c > 'Z' { + if c < 'a' || c > 'z' { + return false + } + } + partLength++ + } + return true +} + +type decodedExtendedParameterState struct { + value string + language string + found bool + decoded bool +} + +func normalizeMediaParameterTail(mediaType string, params string, externalBodyAccessType string, externalBodyAccessLanguage string, hasExternalBodyAccessType bool) string { + decodedParameters := map[string]decodedExtendedParameterState{} + var normalized strings.Builder + first := true + forEachMediaParameter(params, func(param string) { + if !first { + normalized.WriteByte(';') + } + first = false + normalized.WriteString(normalizeMediaParameter(mediaType, params, param, externalBodyAccessType, externalBodyAccessLanguage, hasExternalBodyAccessType, decodedParameters)) + }) + return normalized.String() +} + +func forEachMediaParameter(params string, visit func(string)) { + segmentStart := 0 + inQuotes := false + escaped := false + + for i := 0; i <= len(params); i++ { + if i == len(params) || (!inQuotes && params[i] == ';') { + visit(params[segmentStart:i]) + segmentStart = i + 1 + continue + } + + switch params[i] { + case '\\': + if inQuotes && !escaped { + escaped = true + continue + } + case '"': + if !escaped { + inQuotes = !inQuotes + } + } + escaped = false + } +} + +func normalizeMediaParameter(mediaType string, params string, param string, externalBodyAccessType string, externalBodyAccessLanguage string, hasExternalBodyAccessType bool, decodedParameters map[string]decodedExtendedParameterState) string { + equals := strings.IndexByte(param, '=') + if equals < 0 { + return param + } + + namePart := param[:equals] + nameStart, nameEnd := trimOWSBounds(namePart) + if nameStart == nameEnd { + return param + } + name := namePart[nameStart:nameEnd] + logicalName := mediaParameterLogicalName(name) + + var normalized strings.Builder + normalized.WriteString(namePart[:nameStart]) + normalized.WriteString(asciiLower(name)) + normalized.WriteString(namePart[nameEnd:]) + normalized.WriteByte('=') + + value := param[equals+1:] + isExternalBodyAccessType := strings.TrimSpace(mediaType) == "message/external-body" && strings.EqualFold(logicalName, "access-type") + switch { + case isExternalBodyAccessType: + normalized.WriteString(normalizeCanonicalMediaParameterValue(name, value, "access-type", externalBodyAccessType, externalBodyAccessLanguage, hasExternalBodyAccessType)) + case isCaseInsensitiveMediaParameterValue(mediaType, logicalName, externalBodyAccessType): + state := decodedCaseInsensitiveParameter(mediaType, params, logicalName, externalBodyAccessType, decodedParameters) + if state.found { + normalized.WriteString(normalizeCanonicalMediaParameterValue(name, value, logicalName, state.value, state.language, state.decoded)) + } else { + core, _, ok := mediaParameterValueCore(value) + if ok && validCaseInsensitiveMediaParameterValue(mediaType, logicalName, core, externalBodyAccessType) { + valueStart, valueEnd := trimOWSBounds(value) + normalized.WriteString(value[:valueStart]) + normalized.WriteString(asciiLower(value[valueStart:valueEnd])) + normalized.WriteString(value[valueEnd:]) + } else { + normalized.WriteString(value) + } + } + case strings.HasSuffix(name, "*"): + normalized.WriteString(normalizeExtendedParameterValue(value, extendedMediaParameterHasMetadata(name))) + default: + normalized.WriteString(value) + } + + return normalized.String() +} + +func decodedCaseInsensitiveParameter(mediaType string, params string, logicalName string, externalBodyAccessType string, cache map[string]decodedExtendedParameterState) decodedExtendedParameterState { + key := asciiLower(logicalName) + if state, ok := cache[key]; ok { + return state + } + value, language, found, decoded := decodeExtendedMediaParameter(params, logicalName) + if decoded { + if validCaseInsensitiveMediaParameterValue(mediaType, logicalName, value, externalBodyAccessType) { + value = asciiLower(value) + language = asciiLower(language) + } else { + decoded = false + } + } + state := decodedExtendedParameterState{value: value, language: language, found: found, decoded: decoded} + cache[key] = state + return state +} + +func normalizeCanonicalMediaParameterValue(name string, value string, logicalName string, decodedValue string, decodedLanguage string, decoded bool) string { + if decoded { + canonical := "d" + if isInitialMediaParameterSegment(name, logicalName) { + canonical = encodeDecodedDecoderKeyValue(decodedLanguage, decodedValue) + } + valueStart, valueEnd := trimOWSBounds(value) + return value[:valueStart] + canonical + value[valueEnd:] + } + + normalizedValue := value + if strings.HasSuffix(name, "*") { + normalizedValue = normalizeExtendedParameterValue(value, extendedMediaParameterHasMetadata(name)) + } + valueStart, valueEnd := trimOWSBounds(normalizedValue) + return normalizedValue[:valueStart] + encodeDecoderKeyValue('r', normalizedValue[valueStart:valueEnd]) + normalizedValue[valueEnd:] +} + +func encodeDecodedDecoderKeyValue(language string, value string) string { + languageHex := make([]byte, hex.EncodedLen(len(language))) + hex.Encode(languageHex, []byte(language)) + valueHex := make([]byte, hex.EncodedLen(len(value))) + hex.Encode(valueHex, []byte(value)) + return "d" + string(languageHex) + "g" + string(valueHex) +} + +func encodeDecoderKeyValue(prefix byte, value string) string { + encoded := make([]byte, 1+hex.EncodedLen(len(value))) + encoded[0] = prefix + hex.Encode(encoded[1:], []byte(value)) + return string(encoded) +} + +func isInitialMediaParameterSegment(name string, logicalName string) bool { + if strings.EqualFold(name, logicalName) { + return true + } + sectionName := strings.TrimSuffix(name, "*") + if strings.EqualFold(sectionName, logicalName) { + return true + } + star := strings.LastIndexByte(sectionName, '*') + return star >= 0 && strings.EqualFold(sectionName[:star], logicalName) && sectionName[star+1:] == "0" +} + +func mediaParameterLogicalName(name string) string { + logicalName := strings.TrimSuffix(name, "*") + section := strings.LastIndexByte(logicalName, '*') + if section < 0 || !isRFC2231Section(logicalName[section+1:]) { + return logicalName + } + return logicalName[:section] +} + +func extendedMediaParameterHasMetadata(name string) bool { + if !strings.HasSuffix(name, "*") { + return false + } + + encodedName := strings.TrimSuffix(name, "*") + section := strings.LastIndexByte(encodedName, '*') + if section < 0 { + return true + } + return encodedName[section+1:] == "0" +} + +func isRFC2231Section(section string) bool { + if section == "0" { + return true + } + if len(section) == 0 || section[0] < '1' || section[0] > '9' { + return false + } + for i := 1; i < len(section); i++ { + if section[i] < '0' || section[i] > '9' { + return false + } + } + return true +} + +func isCaseInsensitiveMediaParameterValue(mediaType string, name string, externalBodyAccessType string) bool { + if strings.EqualFold(name, "charset") { + return true + } + + switch strings.TrimSpace(mediaType) { + case "message/external-body": + switch asciiLower(name) { + case "access-type", "permission": + return true + case "mode": + switch externalBodyAccessType { + case "ftp", "anon-ftp", "tftp": + return true + } + } + case "multipart/encrypted": + return strings.EqualFold(name, "protocol") + case "multipart/signed": + // RFC 1847 makes micalg value syntax and semantics protocol-defined. + // RFC 2045 therefore leaves micalg case-sensitive unless that selected + // protocol explicitly defines otherwise; do not fold it generically. + return strings.EqualFold(name, "protocol") + case "multipart/report": + return strings.EqualFold(name, "report-type") + case "multipart/related": + return strings.EqualFold(name, "type") + case "text/plain": + switch asciiLower(name) { + case "format", "delsp": + return true + } + case "text/calendar": + switch asciiLower(name) { + case "method", "component": + return true + } + } + return false +} + +func validCaseInsensitiveMediaParameterValue(mediaType string, name string, value string, externalBodyAccessType string) bool { + if strings.EqualFold(name, "charset") { + if value == "" { + return false + } + _, err := ianaindex.IANA.Encoding(value) + return err == nil + } + + switch strings.TrimSpace(mediaType) { + case "message/external-body": + switch asciiLower(name) { + case "access-type", "permission", "mode": + return isMIMEToken(value) + } + case "multipart/encrypted", "multipart/signed": + if strings.EqualFold(name, "protocol") { + return validMIMEMediaType(value) + } + case "multipart/report": + if strings.EqualFold(name, "report-type") { + return isMIMEToken(value) + } + case "multipart/related": + if strings.EqualFold(name, "type") { + return validMIMEMediaType(value) + } + case "text/plain": + switch asciiLower(name) { + case "format", "delsp": + return isMIMEToken(value) + } + case "text/calendar": + switch asciiLower(name) { + case "method", "component": + return isMIMEToken(value) + } + } + return false +} + +func normalizeExtendedParameterValue(value string, hasMetadata bool) string { + valueStart, valueEnd := trimOWSBounds(value) + core := value[valueStart:valueEnd] + quoted := false + if strings.HasPrefix(core, "\"") { + var ok bool + core, ok = quotedMediaParameterContents(core) + if !ok { + return value + } + quoted = true + } + + firstQuote := -1 + secondQuote := -1 + if hasMetadata { + firstQuote = strings.IndexByte(core, '\'') + if firstQuote >= 0 { + if offset := strings.IndexByte(core[firstQuote+1:], '\''); offset >= 0 { + secondQuote = firstQuote + 1 + offset + } + } + } + + var normalized string + if firstQuote >= 0 && secondQuote >= 0 { + normalized = asciiLower(core[:firstQuote]) + "'" + + asciiLower(core[firstQuote+1:secondQuote]) + "'" + + normalizePercentEncoding(core[secondQuote+1:]) + } else { + normalized = normalizePercentEncoding(core) + } + if quoted { + normalized = "\"" + normalized + "\"" + } + + return value[:valueStart] + normalized + value[valueEnd:] +} + +func quotedMediaParameterContents(value string) (string, bool) { + if len(value) < 2 || value[0] != '"' || value[len(value)-1] != '"' { + return "", false + } + + escaped := false + for i := 1; i < len(value)-1; i++ { + switch value[i] { + case '\\': + escaped = !escaped + case '"': + if !escaped { + return "", false + } + escaped = false + default: + escaped = false + } + } + if escaped { + return "", false + } + return value[1 : len(value)-1], true +} + +func hexValue(value byte) byte { + switch { + case value >= '0' && value <= '9': + return value - '0' + case value >= 'a' && value <= 'f': + return value - 'a' + 10 + default: + return value - 'A' + 10 + } +} + +func normalizePercentEncoding(value string) string { + bytes := []byte(value) + for i := 0; i+2 < len(bytes); i++ { + if bytes[i] != '%' || !isHexDigit(bytes[i+1]) || !isHexDigit(bytes[i+2]) { + continue + } + bytes[i+1] = lowerHexDigit(bytes[i+1]) + bytes[i+2] = lowerHexDigit(bytes[i+2]) + i += 2 + } + return string(bytes) +} + +func isHexDigit(value byte) bool { + return value >= '0' && value <= '9' || + value >= 'a' && value <= 'f' || + value >= 'A' && value <= 'F' +} + +func lowerHexDigit(value byte) byte { + if value >= 'A' && value <= 'F' { + return value + ('a' - 'A') + } + return value +} + +func trimOWSBounds(value string) (int, int) { + start := 0 + end := len(value) + for start < end && (value[start] == ' ' || value[start] == '\t') { + start++ + } + for end > start && (value[end-1] == ' ' || value[end-1] == '\t') { + end-- + } + return start, end +} + type Event struct { Type string Data []byte