From 36d776358123a684d197bc1a1a417b64eef48b00 Mon Sep 17 00:00:00 2001 From: Carlos Acosta <93443910+charle-z@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:44:30 -0500 Subject: [PATCH 01/20] fix(ssestream): preserve parameter value case in decoder keys --- packages/ssestream/ssestream.go | 150 +++++++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 3 deletions(-) diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index 1e9e16bd..d79b99f0 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -49,12 +49,11 @@ func NewDecoder(res *http.Response) Decoder { var decoderTypes = map[string](func(io.ReadCloser) Decoder){} func RegisterDecoder(contentType string, decoder func(io.ReadCloser) Decoder) { - decoderTypes[strings.ToLower(contentType)] = decoder + decoderTypes[decoderContentTypeKey(contentType)] = decoder } func decoderContentTypes(contentType string) (string, string) { - base, _, _ := strings.Cut(contentType, ";") - exactType := strings.ToLower(base) + contentType[len(base):] + exactType := decoderContentTypeKey(contentType) mediaType, _, err := mime.ParseMediaType(contentType) if err != nil { @@ -63,6 +62,151 @@ func decoderContentTypes(contentType string) (string, string) { return exactType, mediaType } +// decoderContentTypeKey normalizes only MIME components whose case is +// semantically insignificant. Parameter values remain case-sensitive unless +// their parameter defines otherwise, such as charset. Extended parameter +// percent-encoding is normalized without changing unescaped value bytes. +func decoderContentTypeKey(contentType string) string { + base, params, found := strings.Cut(contentType, ";") + if !found { + return strings.ToLower(contentType) + } + return strings.ToLower(base) + ";" + normalizeMediaParameterTail(params) +} + +func normalizeMediaParameterTail(params string) string { + var normalized strings.Builder + segmentStart := 0 + inQuotes := false + escaped := false + + for i := 0; i <= len(params); i++ { + if i == len(params) || (!inQuotes && params[i] == ';') { + normalized.WriteString(normalizeMediaParameter(params[segmentStart:i])) + if i < len(params) { + normalized.WriteByte(';') + } + segmentStart = i + 1 + continue + } + + switch params[i] { + case '\\': + if inQuotes && !escaped { + escaped = true + continue + } + case '"': + if !escaped { + inQuotes = !inQuotes + } + } + escaped = false + } + + return normalized.String() +} + +func normalizeMediaParameter(param string) string { + equals := strings.IndexByte(param, '=') + if equals < 0 { + return param + } + + namePart := param[:equals] + nameStart, nameEnd := trimOWSBounds(namePart) + if nameStart == nameEnd { + return param + } + name := namePart[nameStart:nameEnd] + + var normalized strings.Builder + normalized.WriteString(namePart[:nameStart]) + normalized.WriteString(strings.ToLower(name)) + normalized.WriteString(namePart[nameEnd:]) + normalized.WriteByte('=') + + value := param[equals+1:] + switch { + case strings.EqualFold(name, "charset"): + valueStart, valueEnd := trimOWSBounds(value) + normalized.WriteString(value[:valueStart]) + normalized.WriteString(strings.ToLower(value[valueStart:valueEnd])) + normalized.WriteString(value[valueEnd:]) + case strings.Contains(name, "*"): + normalized.WriteString(normalizeExtendedParameterValue(value)) + default: + normalized.WriteString(value) + } + + return normalized.String() +} + +func normalizeExtendedParameterValue(value string) string { + valueStart, valueEnd := trimOWSBounds(value) + core := value[valueStart:valueEnd] + if strings.HasPrefix(core, "\"") { + return value + } + + firstQuote := strings.IndexByte(core, '\'') + secondQuote := -1 + if firstQuote >= 0 { + if offset := strings.IndexByte(core[firstQuote+1:], '\''); offset >= 0 { + secondQuote = firstQuote + 1 + offset + } + } + + var normalized string + if firstQuote >= 0 && secondQuote >= 0 { + normalized = strings.ToLower(core[:firstQuote]) + "'" + + strings.ToLower(core[firstQuote+1:secondQuote]) + "'" + + normalizePercentEncoding(core[secondQuote+1:]) + } else { + normalized = normalizePercentEncoding(core) + } + + return value[:valueStart] + normalized + value[valueEnd:] +} + +func normalizePercentEncoding(value string) string { + bytes := []byte(value) + for i := 0; i+2 < len(bytes); i++ { + if bytes[i] != '%' || !isHexDigit(bytes[i+1]) || !isHexDigit(bytes[i+2]) { + continue + } + bytes[i+1] = lowerHexDigit(bytes[i+1]) + bytes[i+2] = lowerHexDigit(bytes[i+2]) + i += 2 + } + return string(bytes) +} + +func isHexDigit(value byte) bool { + return value >= '0' && value <= '9' || + value >= 'a' && value <= 'f' || + value >= 'A' && value <= 'F' +} + +func lowerHexDigit(value byte) byte { + if value >= 'A' && value <= 'F' { + return value + ('a' - 'A') + } + return value +} + +func trimOWSBounds(value string) (int, int) { + start := 0 + end := len(value) + for start < end && (value[start] == ' ' || value[start] == '\t') { + start++ + } + for end > start && (value[end-1] == ' ' || value[end-1] == '\t') { + end-- + } + return start, end +} + type Event struct { Type string Data []byte From 593ecdc12cca7c536e70b00e4755bc173feb0b11 Mon Sep 17 00:00:00 2001 From: Carlos Acosta <93443910+charle-z@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:44:49 -0500 Subject: [PATCH 02/20] test(ssestream): cover decoder registration parameter case --- packages/ssestream/content_type_case_test.go | 116 +++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 packages/ssestream/content_type_case_test.go diff --git a/packages/ssestream/content_type_case_test.go b/packages/ssestream/content_type_case_test.go new file mode 100644 index 00000000..94db12ef --- /dev/null +++ b/packages/ssestream/content_type_case_test.go @@ -0,0 +1,116 @@ +package ssestream + +import ( + "io" + "net/http" + "strings" + "testing" +) + +func TestRegisterDecoderPreservesCaseSensitiveParameterValues(t *testing.T) { + const ( + mediaType = "application/x-openai-go-test-registration-case" + profileV1 = mediaType + "; profile=\"https://example.com/V1\"" + profileV1Lo = mediaType + "; profile=\"https://example.com/v1\"" + ) + wantDefault := &testDecoder{} + wantProfile := &testDecoder{} + RegisterDecoder(mediaType, func(io.ReadCloser) Decoder { return wantDefault }) + RegisterDecoder(profileV1, func(io.ReadCloser) Decoder { return wantProfile }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(mediaType)) + delete(decoderTypes, decoderContentTypeKey(profileV1)) + }) + + for name, test := range map[string]struct { + contentType string + want Decoder + }{ + "registered uppercase profile": { + contentType: "Application/X-OpenAI-Go-Test-Registration-Case; profile=\"https://example.com/V1\"", + want: wantProfile, + }, + "distinct lowercase profile": { + contentType: profileV1Lo, + want: wantDefault, + }, + } { + t.Run(name, func(t *testing.T) { + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.contentType}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != test.want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + +func TestRegisterDecoderNormalizesCaseInsensitiveParameterComponents(t *testing.T) { + const registered = "application/x-openai-go-test-registration-components; Profile=\"https://example.com/V1\"; Charset=UTF-8" + want := &testDecoder{} + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(registered)) + }) + + decoder := NewDecoder(&http.Response{ + Header: http.Header{ + "Content-Type": {"Application/X-OpenAI-Go-Test-Registration-Components; profile=\"https://example.com/V1\"; charset=utf-8"}, + }, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } +} + +func TestRegisterDecoderExtendedParameterPreservesUnescapedValueCase(t *testing.T) { + const ( + mediaType = "application/x-openai-go-test-registration-extended" + variantV1 = mediaType + "; Variant*=ISO-8859-1'EN'caf%E9V1" + variantv1 = mediaType + "; variant*=iso-8859-1'en'caf%e9v1" + ) + wantDefault := &testDecoder{} + wantVariant := &testDecoder{} + RegisterDecoder(mediaType, func(io.ReadCloser) Decoder { return wantDefault }) + RegisterDecoder(variantV1, func(io.ReadCloser) Decoder { return wantVariant }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(mediaType)) + delete(decoderTypes, decoderContentTypeKey(variantV1)) + }) + + for name, test := range map[string]struct { + contentType string + want Decoder + }{ + "equivalent encoding case": { + contentType: mediaType + "; variant*=iso-8859-1'en'caf%e9V1", + want: wantVariant, + }, + "distinct unescaped value case": { + contentType: variantv1, + want: wantDefault, + }, + } { + t.Run(name, func(t *testing.T) { + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.contentType}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != test.want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + +func TestDecoderContentTypeKeyDoesNotSplitQuotedSemicolons(t *testing.T) { + const contentType = "Application/X-OpenAI-Go-Test-Quoted; Profile=\"https://example.com/a;b?x*=V1\"; Charset=UTF-8" + got := decoderContentTypeKey(contentType) + want := "application/x-openai-go-test-quoted; profile=\"https://example.com/a;b?x*=V1\"; charset=utf-8" + if got != want { + t.Fatalf("decoder content type key = %q, want %q", got, want) + } +} From ef8282a20435833630ec94111ba940e7a63bfc3d Mon Sep 17 00:00:00 2001 From: Carlos Acosta <93443910+charle-z@users.noreply.github.com> Date: Sun, 23 Aug 2026 12:45:21 -0500 Subject: [PATCH 03/20] fix(ssestream): preserve MIME parameter semantics --- packages/ssestream/ssestream.go | 39 +++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index d79b99f0..5200401b 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -64,17 +64,18 @@ func decoderContentTypes(contentType string) (string, string) { // decoderContentTypeKey normalizes only MIME components whose case is // semantically insignificant. Parameter values remain case-sensitive unless -// their parameter defines otherwise, such as charset. Extended parameter -// percent-encoding is normalized without changing unescaped value bytes. +// their parameter defines otherwise. Extended parameter percent-encoding is +// normalized without changing unescaped value bytes. func decoderContentTypeKey(contentType string) string { base, params, found := strings.Cut(contentType, ";") if !found { return strings.ToLower(contentType) } - return strings.ToLower(base) + ";" + normalizeMediaParameterTail(params) + normalizedBase := strings.ToLower(base) + return normalizedBase + ";" + normalizeMediaParameterTail(normalizedBase, params) } -func normalizeMediaParameterTail(params string) string { +func normalizeMediaParameterTail(mediaType string, params string) string { var normalized strings.Builder segmentStart := 0 inQuotes := false @@ -82,7 +83,7 @@ func normalizeMediaParameterTail(params string) string { for i := 0; i <= len(params); i++ { if i == len(params) || (!inQuotes && params[i] == ';') { - normalized.WriteString(normalizeMediaParameter(params[segmentStart:i])) + normalized.WriteString(normalizeMediaParameter(mediaType, params[segmentStart:i])) if i < len(params) { normalized.WriteByte(';') } @@ -107,7 +108,7 @@ func normalizeMediaParameterTail(params string) string { return normalized.String() } -func normalizeMediaParameter(param string) string { +func normalizeMediaParameter(mediaType string, param string) string { equals := strings.IndexByte(param, '=') if equals < 0 { return param @@ -128,12 +129,12 @@ func normalizeMediaParameter(param string) string { value := param[equals+1:] switch { - case strings.EqualFold(name, "charset"): + case isCaseInsensitiveMediaParameterValue(mediaType, name): valueStart, valueEnd := trimOWSBounds(value) normalized.WriteString(value[:valueStart]) normalized.WriteString(strings.ToLower(value[valueStart:valueEnd])) normalized.WriteString(value[valueEnd:]) - case strings.Contains(name, "*"): + case strings.HasSuffix(name, "*"): normalized.WriteString(normalizeExtendedParameterValue(value)) default: normalized.WriteString(value) @@ -142,6 +143,22 @@ func normalizeMediaParameter(param string) string { return normalized.String() } +func isCaseInsensitiveMediaParameterValue(mediaType string, name string) bool { + if strings.EqualFold(name, "charset") { + return true + } + if !strings.EqualFold(strings.TrimSpace(mediaType), "message/external-body") { + return false + } + + switch strings.ToLower(name) { + case "access-type", "permission", "mode": + return true + default: + return false + } +} + func normalizeExtendedParameterValue(value string) string { valueStart, valueEnd := trimOWSBounds(value) core := value[valueStart:valueEnd] @@ -240,7 +257,7 @@ func (s *eventStreamDecoder) Next() bool { var data []byte for s.scn.Scan() { - txt := s.scn.Bytes() + txt := scnBytes(s.scn) // Dispatch event on an empty line if len(txt) == 0 { @@ -282,6 +299,10 @@ func (s *eventStreamDecoder) Next() bool { return false } +func scnBytes(scn *bufio.Scanner) []byte { + return scn.Bytes() +} + func (s *eventStreamDecoder) Event() Event { return s.evt } From dd8f9ddcca841628b3b75c1bf111dcf74769d4de Mon Sep 17 00:00:00 2001 From: Carlos Acosta <93443910+charle-z@users.noreply.github.com> Date: Sun, 23 Aug 2026 12:46:18 -0500 Subject: [PATCH 04/20] chore(ssestream): remove unrelated scanner wrapper --- packages/ssestream/ssestream.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index 5200401b..f660d666 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -257,7 +257,7 @@ func (s *eventStreamDecoder) Next() bool { var data []byte for s.scn.Scan() { - txt := scnBytes(s.scn) + txt := s.scn.Bytes() // Dispatch event on an empty line if len(txt) == 0 { @@ -299,10 +299,6 @@ func (s *eventStreamDecoder) Next() bool { return false } -func scnBytes(scn *bufio.Scanner) []byte { - return scn.Bytes() -} - func (s *eventStreamDecoder) Event() Event { return s.evt } From 82aeabfca7184dea3a5fac7384f9eea0b12f1587 Mon Sep 17 00:00:00 2001 From: Carlos Acosta <93443910+charle-z@users.noreply.github.com> Date: Sun, 23 Aug 2026 12:46:44 -0500 Subject: [PATCH 05/20] test(ssestream): cover MIME case exceptions --- packages/ssestream/content_type_case_test.go | 83 ++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/packages/ssestream/content_type_case_test.go b/packages/ssestream/content_type_case_test.go index 94db12ef..376782a6 100644 --- a/packages/ssestream/content_type_case_test.go +++ b/packages/ssestream/content_type_case_test.go @@ -66,6 +66,49 @@ func TestRegisterDecoderNormalizesCaseInsensitiveParameterComponents(t *testing. } } +func TestRegisterDecoderNormalizesExternalBodyCaseInsensitiveValues(t *testing.T) { + for name, test := range map[string]struct { + parameter string + registered string + response string + }{ + "access type": { + parameter: "Access-Type", + registered: "LOCAL-FILE", + response: "local-file", + }, + "permission": { + parameter: "Permission", + registered: "READ-WRITE", + response: "read-write", + }, + "mode": { + parameter: "Mode", + registered: "IMAGE", + response: "image", + }, + } { + t.Run(name, func(t *testing.T) { + registered := "message/external-body; " + test.parameter + "=" + test.registered + want := &testDecoder{} + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(registered)) + }) + + decoder := NewDecoder(&http.Response{ + Header: http.Header{ + "Content-Type": {"Message/External-Body; " + strings.ToLower(test.parameter) + "=" + test.response}, + }, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + func TestRegisterDecoderExtendedParameterPreservesUnescapedValueCase(t *testing.T) { const ( mediaType = "application/x-openai-go-test-registration-extended" @@ -106,6 +149,46 @@ func TestRegisterDecoderExtendedParameterPreservesUnescapedValueCase(t *testing. } } +func TestRegisterDecoderUnencodedContinuationPreservesValueCase(t *testing.T) { + const ( + mediaType = "application/x-openai-go-test-registration-continuation" + upper = mediaType + "; title*0=V%AB" + lower = mediaType + "; title*0=V%ab" + ) + wantDefault := &testDecoder{} + wantUpper := &testDecoder{} + RegisterDecoder(mediaType, func(io.ReadCloser) Decoder { return wantDefault }) + RegisterDecoder(upper, func(io.ReadCloser) Decoder { return wantUpper }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(mediaType)) + delete(decoderTypes, decoderContentTypeKey(upper)) + }) + + for name, test := range map[string]struct { + contentType string + want Decoder + }{ + "registered uppercase escape text": { + contentType: upper, + want: wantUpper, + }, + "distinct lowercase escape text": { + contentType: lower, + want: wantDefault, + }, + } { + t.Run(name, func(t *testing.T) { + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.contentType}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != test.want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + func TestDecoderContentTypeKeyDoesNotSplitQuotedSemicolons(t *testing.T) { const contentType = "Application/X-OpenAI-Go-Test-Quoted; Profile=\"https://example.com/a;b?x*=V1\"; Charset=UTF-8" got := decoderContentTypeKey(contentType) From 443d6ce6d8e2f259e43457ac17ddbb7027e7a72a Mon Sep 17 00:00:00 2001 From: Carlos Acosta <93443910+charle-z@users.noreply.github.com> Date: Sun, 23 Aug 2026 12:59:38 -0500 Subject: [PATCH 06/20] fix(ssestream): normalize logical MIME parameter semantics --- packages/ssestream/ssestream.go | 120 +++++++++++++++++++++++++++++--- 1 file changed, 109 insertions(+), 11 deletions(-) diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index f660d666..869b4ab0 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -120,6 +120,7 @@ func normalizeMediaParameter(mediaType string, param string) string { return param } name := namePart[nameStart:nameEnd] + logicalName := mediaParameterLogicalName(name) var normalized strings.Builder normalized.WriteString(namePart[:nameStart]) @@ -129,11 +130,15 @@ func normalizeMediaParameter(mediaType string, param string) string { value := param[equals+1:] switch { - case isCaseInsensitiveMediaParameterValue(mediaType, name): - valueStart, valueEnd := trimOWSBounds(value) - normalized.WriteString(value[:valueStart]) - normalized.WriteString(strings.ToLower(value[valueStart:valueEnd])) - normalized.WriteString(value[valueEnd:]) + case isCaseInsensitiveMediaParameterValue(mediaType, logicalName): + if strings.HasSuffix(name, "*") { + normalized.WriteString(normalizeCaseInsensitiveExtendedParameterValue(value)) + } else { + valueStart, valueEnd := trimOWSBounds(value) + normalized.WriteString(value[:valueStart]) + normalized.WriteString(strings.ToLower(value[valueStart:valueEnd])) + normalized.WriteString(value[valueEnd:]) + } case strings.HasSuffix(name, "*"): normalized.WriteString(normalizeExtendedParameterValue(value)) default: @@ -143,20 +148,113 @@ func normalizeMediaParameter(mediaType string, param string) string { return normalized.String() } -func isCaseInsensitiveMediaParameterValue(mediaType string, name string) bool { - if strings.EqualFold(name, "charset") { +func mediaParameterLogicalName(name string) string { + logicalName := strings.TrimSuffix(name, "*") + section := strings.LastIndexByte(logicalName, '*') + if section < 0 || !isRFC2231Section(logicalName[section+1:]) { + return logicalName + } + return logicalName[:section] +} + +func isRFC2231Section(section string) bool { + if section == "0" { return true } - if !strings.EqualFold(strings.TrimSpace(mediaType), "message/external-body") { + if len(section) == 0 || section[0] < '1' || section[0] > '9' { return false } + for i := 1; i < len(section); i++ { + if section[i] < '0' || section[i] > '9' { + return false + } + } + return true +} - switch strings.ToLower(name) { - case "access-type", "permission", "mode": +func isCaseInsensitiveMediaParameterValue(mediaType string, name string) bool { + if strings.EqualFold(name, "charset") { return true + } + + switch strings.ToLower(strings.TrimSpace(mediaType)) { + case "message/external-body": + switch strings.ToLower(name) { + case "access-type", "permission", "mode": + return true + } + case "text/plain": + switch strings.ToLower(name) { + case "format", "delsp": + return true + } + } + return false +} + +func normalizeCaseInsensitiveExtendedParameterValue(value string) string { + valueStart, valueEnd := trimOWSBounds(value) + core := value[valueStart:valueEnd] + if strings.HasPrefix(core, "\"") { + return value + } + + firstQuote := strings.IndexByte(core, '\'') + secondQuote := -1 + if firstQuote >= 0 { + if offset := strings.IndexByte(core[firstQuote+1:], '\''); offset >= 0 { + secondQuote = firstQuote + 1 + offset + } + } + + var normalized string + if firstQuote >= 0 && secondQuote >= 0 { + normalized = strings.ToLower(core[:firstQuote]) + "'" + + strings.ToLower(core[firstQuote+1:secondQuote]) + "'" + + normalizeCaseInsensitiveExtendedData(core[secondQuote+1:]) + } else { + normalized = normalizeCaseInsensitiveExtendedData(core) + } + + return value[:valueStart] + normalized + value[valueEnd:] +} + +func normalizeCaseInsensitiveExtendedData(value string) string { + bytes := []byte(value) + for i := 0; i < len(bytes); i++ { + if bytes[i] == '%' && i+2 < len(bytes) && isHexDigit(bytes[i+1]) && isHexDigit(bytes[i+2]) { + decoded := hexValue(bytes[i+1])<<4 | hexValue(bytes[i+2]) + if decoded >= 'A' && decoded <= 'Z' { + decoded += 'a' - 'A' + } + bytes[i+1] = hexDigit(decoded >> 4) + bytes[i+2] = hexDigit(decoded & 0x0f) + i += 2 + continue + } + if bytes[i] >= 'A' && bytes[i] <= 'Z' { + bytes[i] += 'a' - 'A' + } + } + return string(bytes) +} + +func hexValue(value byte) byte { + switch { + case value >= '0' && value <= '9': + return value - '0' + case value >= 'a' && value <= 'f': + return value - 'a' + 10 default: - return false + return value - 'A' + 10 + } +} + +func hexDigit(value byte) byte { + if value < 10 { + return '0' + value } + return 'a' + value - 10 } func normalizeExtendedParameterValue(value string) string { From dcb689026e46d7a8d0c510215062e3426ea1173e Mon Sep 17 00:00:00 2001 From: Carlos Acosta <93443910+charle-z@users.noreply.github.com> Date: Sun, 23 Aug 2026 13:01:16 -0500 Subject: [PATCH 07/20] test(ssestream): cover logical MIME parameter semantics --- packages/ssestream/content_type_case_test.go | 53 +++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/packages/ssestream/content_type_case_test.go b/packages/ssestream/content_type_case_test.go index 376782a6..5a9ef8c4 100644 --- a/packages/ssestream/content_type_case_test.go +++ b/packages/ssestream/content_type_case_test.go @@ -68,7 +68,7 @@ func TestRegisterDecoderNormalizesCaseInsensitiveParameterComponents(t *testing. func TestRegisterDecoderNormalizesExternalBodyCaseInsensitiveValues(t *testing.T) { for name, test := range map[string]struct { - parameter string + parameter string registered string response string }{ @@ -109,6 +109,57 @@ func TestRegisterDecoderNormalizesExternalBodyCaseInsensitiveValues(t *testing.T } } +func TestRegisterDecoderNormalizesLogicalCaseInsensitiveParameters(t *testing.T) { + for name, test := range map[string]struct { + registered string + response string + }{ + "text plain format": { + registered: "text/plain; Format=FLOWED", + response: "Text/Plain; format=flowed", + }, + "text plain delsp": { + registered: "text/plain; DelSP=YES", + response: "Text/Plain; delsp=yes", + }, + "encoded charset": { + registered: "text/plain; Charset*=US-ASCII'EN'%55TF-8", + response: "Text/Plain; charset*=us-ascii'en'%75tf-8", + }, + "unencoded format continuation": { + registered: "text/plain; Format*0=FLO; Format*1=WED", + response: "Text/Plain; format*0=flo; format*1=wed", + }, + "encoded access type continuation": { + registered: "message/external-body; Access-Type*0*=US-ASCII''LOCAL-; Access-Type*1*=FILE", + response: "Message/External-Body; access-type*0*=us-ascii''local-; access-type*1*=file", + }, + } { + t.Run(name, func(t *testing.T) { + want := &testDecoder{} + RegisterDecoder(test.registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(test.registered)) + }) + + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.response}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + +func TestDecoderContentTypeKeyKeepsContextualValuesCaseSensitive(t *testing.T) { + const contentType = "text/html; Format=FLOWED" + if got := decoderContentTypeKey(contentType); got != "text/html; format=FLOWED" { + t.Fatalf("decoder content type key = %q, want context-specific value case preserved", got) + } +} + func TestRegisterDecoderExtendedParameterPreservesUnescapedValueCase(t *testing.T) { const ( mediaType = "application/x-openai-go-test-registration-extended" From 136594ca3989e6e7fa73dfe065c93283727c342a Mon Sep 17 00:00:00 2001 From: mcp-devbox Date: Thu, 27 Aug 2026 02:03:44 +0000 Subject: [PATCH 08/20] fix(ssestream): normalize remaining MIME parameter cases --- packages/ssestream/content_type_case_test.go | 8 ++ packages/ssestream/ssestream.go | 77 ++++++++++++-------- 2 files changed, 55 insertions(+), 30 deletions(-) diff --git a/packages/ssestream/content_type_case_test.go b/packages/ssestream/content_type_case_test.go index 5a9ef8c4..8b792023 100644 --- a/packages/ssestream/content_type_case_test.go +++ b/packages/ssestream/content_type_case_test.go @@ -126,6 +126,14 @@ func TestRegisterDecoderNormalizesLogicalCaseInsensitiveParameters(t *testing.T) registered: "text/plain; Charset*=US-ASCII'EN'%55TF-8", response: "Text/Plain; charset*=us-ascii'en'%75tf-8", }, + "quoted encoded charset": { + registered: "text/plain; Charset*=\"US-ASCII'EN'UTF%2D8\"", + response: "Text/Plain; charset*=\"us-ascii'en'utf%2d8\"", + }, + "multipart related type": { + registered: "multipart/related; Type=\"Application/X-Test\"", + response: "Multipart/Related; type=\"application/x-test\"", + }, "unencoded format continuation": { registered: "text/plain; Format*0=FLO; Format*1=WED", response: "Text/Plain; format*0=flo; format*1=wed", diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index 869b4ab0..6f9c5f2b 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -183,6 +183,8 @@ func isCaseInsensitiveMediaParameterValue(mediaType string, name string) bool { case "access-type", "permission", "mode": return true } + case "multipart/related": + return strings.EqualFold(name, "type") case "text/plain": switch strings.ToLower(name) { case "format", "delsp": @@ -193,10 +195,24 @@ func isCaseInsensitiveMediaParameterValue(mediaType string, name string) bool { } func normalizeCaseInsensitiveExtendedParameterValue(value string) string { + return normalizeExtendedParameterValueWithData(value, normalizeCaseInsensitiveExtendedData) +} + +func normalizeExtendedParameterValue(value string) string { + return normalizeExtendedParameterValueWithData(value, normalizePercentEncoding) +} + +func normalizeExtendedParameterValueWithData(value string, normalizeData func(string) string) string { valueStart, valueEnd := trimOWSBounds(value) core := value[valueStart:valueEnd] + quoted := false if strings.HasPrefix(core, "\"") { - return value + var ok bool + core, ok = quotedMediaParameterContents(core) + if !ok { + return value + } + quoted = true } firstQuote := strings.IndexByte(core, '\'') @@ -211,14 +227,42 @@ func normalizeCaseInsensitiveExtendedParameterValue(value string) string { if firstQuote >= 0 && secondQuote >= 0 { normalized = strings.ToLower(core[:firstQuote]) + "'" + strings.ToLower(core[firstQuote+1:secondQuote]) + "'" + - normalizeCaseInsensitiveExtendedData(core[secondQuote+1:]) + normalizeData(core[secondQuote+1:]) } else { - normalized = normalizeCaseInsensitiveExtendedData(core) + normalized = normalizeData(core) + } + if quoted { + normalized = "\"" + normalized + "\"" } return value[:valueStart] + normalized + value[valueEnd:] } +func quotedMediaParameterContents(value string) (string, bool) { + if len(value) < 2 || value[0] != '"' || value[len(value)-1] != '"' { + return "", false + } + + escaped := false + for i := 1; i < len(value)-1; i++ { + switch value[i] { + case '\\': + escaped = !escaped + case '"': + if !escaped { + return "", false + } + escaped = false + default: + escaped = false + } + } + if escaped { + return "", false + } + return value[1 : len(value)-1], true +} + func normalizeCaseInsensitiveExtendedData(value string) string { bytes := []byte(value) for i := 0; i < len(bytes); i++ { @@ -257,33 +301,6 @@ func hexDigit(value byte) byte { return 'a' + value - 10 } -func normalizeExtendedParameterValue(value string) string { - valueStart, valueEnd := trimOWSBounds(value) - core := value[valueStart:valueEnd] - if strings.HasPrefix(core, "\"") { - return value - } - - firstQuote := strings.IndexByte(core, '\'') - secondQuote := -1 - if firstQuote >= 0 { - if offset := strings.IndexByte(core[firstQuote+1:], '\''); offset >= 0 { - secondQuote = firstQuote + 1 + offset - } - } - - var normalized string - if firstQuote >= 0 && secondQuote >= 0 { - normalized = strings.ToLower(core[:firstQuote]) + "'" + - strings.ToLower(core[firstQuote+1:secondQuote]) + "'" + - normalizePercentEncoding(core[secondQuote+1:]) - } else { - normalized = normalizePercentEncoding(core) - } - - return value[:valueStart] + normalized + value[valueEnd:] -} - func normalizePercentEncoding(value string) string { bytes := []byte(value) for i := 0; i+2 < len(bytes); i++ { From 4cbccd1a71edcec774deac50935f52bb209c28af Mon Sep 17 00:00:00 2001 From: mcp-devbox Date: Thu, 27 Aug 2026 02:40:58 +0000 Subject: [PATCH 09/20] fix(ssestream): preserve continuation value case --- packages/ssestream/content_type_case_test.go | 48 ++++++++++++++++++++ packages/ssestream/ssestream.go | 40 +++++++++++----- 2 files changed, 77 insertions(+), 11 deletions(-) diff --git a/packages/ssestream/content_type_case_test.go b/packages/ssestream/content_type_case_test.go index 8b792023..41371a89 100644 --- a/packages/ssestream/content_type_case_test.go +++ b/packages/ssestream/content_type_case_test.go @@ -134,6 +134,14 @@ func TestRegisterDecoderNormalizesLogicalCaseInsensitiveParameters(t *testing.T) registered: "multipart/related; Type=\"Application/X-Test\"", response: "Multipart/Related; type=\"application/x-test\"", }, + "multipart signed protocol": { + registered: "multipart/signed; Protocol=\"Application/PGP-Signature\"", + response: "Multipart/Signed; protocol=\"application/pgp-signature\"", + }, + "multipart encrypted protocol": { + registered: "multipart/encrypted; Protocol=\"Application/PGP-Encrypted\"", + response: "Multipart/Encrypted; protocol=\"application/pgp-encrypted\"", + }, "unencoded format continuation": { registered: "text/plain; Format*0=FLO; Format*1=WED", response: "Text/Plain; format*0=flo; format*1=wed", @@ -208,6 +216,46 @@ func TestRegisterDecoderExtendedParameterPreservesUnescapedValueCase(t *testing. } } +func TestRegisterDecoderEncodedContinuationPreservesLaterSegmentValueCase(t *testing.T) { + const ( + mediaType = "application/x-openai-go-test-registration-encoded-continuation" + titleV1 = mediaType + "; title*0*=us-ascii''prefix; title*1*=Bob%2D's'V1" + titlev1 = mediaType + "; title*0*=us-ascii''prefix; title*1*=bob%2d's'V1" + ) + wantDefault := &testDecoder{} + wantTitle := &testDecoder{} + RegisterDecoder(mediaType, func(io.ReadCloser) Decoder { return wantDefault }) + RegisterDecoder(titleV1, func(io.ReadCloser) Decoder { return wantTitle }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(mediaType)) + delete(decoderTypes, decoderContentTypeKey(titleV1)) + }) + + for name, test := range map[string]struct { + contentType string + want Decoder + }{ + "equivalent percent encoding case": { + contentType: mediaType + "; title*0*=us-ascii''prefix; title*1*=Bob%2d's'V1", + want: wantTitle, + }, + "distinct later segment data case": { + contentType: titlev1, + want: wantDefault, + }, + } { + t.Run(name, func(t *testing.T) { + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.contentType}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != test.want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + func TestRegisterDecoderUnencodedContinuationPreservesValueCase(t *testing.T) { const ( mediaType = "application/x-openai-go-test-registration-continuation" diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index 6f9c5f2b..c9d548c5 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -132,7 +132,7 @@ func normalizeMediaParameter(mediaType string, param string) string { switch { case isCaseInsensitiveMediaParameterValue(mediaType, logicalName): if strings.HasSuffix(name, "*") { - normalized.WriteString(normalizeCaseInsensitiveExtendedParameterValue(value)) + normalized.WriteString(normalizeCaseInsensitiveExtendedParameterValue(value, extendedMediaParameterHasMetadata(name))) } else { valueStart, valueEnd := trimOWSBounds(value) normalized.WriteString(value[:valueStart]) @@ -140,7 +140,7 @@ func normalizeMediaParameter(mediaType string, param string) string { normalized.WriteString(value[valueEnd:]) } case strings.HasSuffix(name, "*"): - normalized.WriteString(normalizeExtendedParameterValue(value)) + normalized.WriteString(normalizeExtendedParameterValue(value, extendedMediaParameterHasMetadata(name))) default: normalized.WriteString(value) } @@ -157,6 +157,19 @@ func mediaParameterLogicalName(name string) string { return logicalName[:section] } +func extendedMediaParameterHasMetadata(name string) bool { + if !strings.HasSuffix(name, "*") { + return false + } + + encodedName := strings.TrimSuffix(name, "*") + section := strings.LastIndexByte(encodedName, '*') + if section < 0 { + return true + } + return encodedName[section+1:] == "0" +} + func isRFC2231Section(section string) bool { if section == "0" { return true @@ -183,6 +196,8 @@ func isCaseInsensitiveMediaParameterValue(mediaType string, name string) bool { case "access-type", "permission", "mode": return true } + case "multipart/encrypted", "multipart/signed": + return strings.EqualFold(name, "protocol") case "multipart/related": return strings.EqualFold(name, "type") case "text/plain": @@ -194,15 +209,15 @@ func isCaseInsensitiveMediaParameterValue(mediaType string, name string) bool { return false } -func normalizeCaseInsensitiveExtendedParameterValue(value string) string { - return normalizeExtendedParameterValueWithData(value, normalizeCaseInsensitiveExtendedData) +func normalizeCaseInsensitiveExtendedParameterValue(value string, hasMetadata bool) string { + return normalizeExtendedParameterValueWithData(value, hasMetadata, normalizeCaseInsensitiveExtendedData) } -func normalizeExtendedParameterValue(value string) string { - return normalizeExtendedParameterValueWithData(value, normalizePercentEncoding) +func normalizeExtendedParameterValue(value string, hasMetadata bool) string { + return normalizeExtendedParameterValueWithData(value, hasMetadata, normalizePercentEncoding) } -func normalizeExtendedParameterValueWithData(value string, normalizeData func(string) string) string { +func normalizeExtendedParameterValueWithData(value string, hasMetadata bool, normalizeData func(string) string) string { valueStart, valueEnd := trimOWSBounds(value) core := value[valueStart:valueEnd] quoted := false @@ -215,11 +230,14 @@ func normalizeExtendedParameterValueWithData(value string, normalizeData func(st quoted = true } - firstQuote := strings.IndexByte(core, '\'') + firstQuote := -1 secondQuote := -1 - if firstQuote >= 0 { - if offset := strings.IndexByte(core[firstQuote+1:], '\''); offset >= 0 { - secondQuote = firstQuote + 1 + offset + if hasMetadata { + firstQuote = strings.IndexByte(core, '\'') + if firstQuote >= 0 { + if offset := strings.IndexByte(core[firstQuote+1:], '\''); offset >= 0 { + secondQuote = firstQuote + 1 + offset + } } } From d57e93c9289af6210aea10b3a1888718ccbca09c Mon Sep 17 00:00:00 2001 From: mcp-devbox Date: Thu, 27 Aug 2026 03:00:05 +0000 Subject: [PATCH 10/20] fix(ssestream): cover remaining case-insensitive MIME values --- packages/ssestream/content_type_case_test.go | 33 ++++++++++++++++++-- packages/ssestream/ssestream.go | 4 +++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/packages/ssestream/content_type_case_test.go b/packages/ssestream/content_type_case_test.go index 41371a89..2151a057 100644 --- a/packages/ssestream/content_type_case_test.go +++ b/packages/ssestream/content_type_case_test.go @@ -142,6 +142,14 @@ func TestRegisterDecoderNormalizesLogicalCaseInsensitiveParameters(t *testing.T) registered: "multipart/encrypted; Protocol=\"Application/PGP-Encrypted\"", response: "Multipart/Encrypted; protocol=\"application/pgp-encrypted\"", }, + "multipart report type": { + registered: "multipart/report; Report-Type=DELIVERY-STATUS", + response: "Multipart/Report; report-type=delivery-status", + }, + "text csv header": { + registered: "text/csv; Header=PRESENT", + response: "Text/CSV; header=present", + }, "unencoded format continuation": { registered: "text/plain; Format*0=FLO; Format*1=WED", response: "Text/Plain; format*0=flo; format*1=wed", @@ -170,9 +178,28 @@ func TestRegisterDecoderNormalizesLogicalCaseInsensitiveParameters(t *testing.T) } func TestDecoderContentTypeKeyKeepsContextualValuesCaseSensitive(t *testing.T) { - const contentType = "text/html; Format=FLOWED" - if got := decoderContentTypeKey(contentType); got != "text/html; format=FLOWED" { - t.Fatalf("decoder content type key = %q, want context-specific value case preserved", got) + for name, test := range map[string]struct { + contentType string + want string + }{ + "format outside text plain": { + contentType: "text/html; Format=FLOWED", + want: "text/html; format=FLOWED", + }, + "report type outside multipart report": { + contentType: "multipart/mixed; Report-Type=DELIVERY-STATUS", + want: "multipart/mixed; report-type=DELIVERY-STATUS", + }, + "header outside text csv": { + contentType: "text/plain; Header=PRESENT", + want: "text/plain; header=PRESENT", + }, + } { + t.Run(name, func(t *testing.T) { + if got := decoderContentTypeKey(test.contentType); got != test.want { + t.Fatalf("decoder content type key = %q, want %q", got, test.want) + } + }) } } diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index c9d548c5..badb31f3 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -198,8 +198,12 @@ func isCaseInsensitiveMediaParameterValue(mediaType string, name string) bool { } case "multipart/encrypted", "multipart/signed": return strings.EqualFold(name, "protocol") + case "multipart/report": + return strings.EqualFold(name, "report-type") case "multipart/related": return strings.EqualFold(name, "type") + case "text/csv": + return strings.EqualFold(name, "header") case "text/plain": switch strings.ToLower(name) { case "format", "delsp": From e8b7a35379956aa60dae137bceacaccfb9f63feb Mon Sep 17 00:00:00 2001 From: mcp-devbox Date: Thu, 27 Aug 2026 03:11:25 +0000 Subject: [PATCH 11/20] fix(ssestream): normalize multipart signed micalg --- packages/ssestream/content_type_case_test.go | 8 ++++++++ packages/ssestream/ssestream.go | 7 ++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/ssestream/content_type_case_test.go b/packages/ssestream/content_type_case_test.go index 2151a057..eeccce56 100644 --- a/packages/ssestream/content_type_case_test.go +++ b/packages/ssestream/content_type_case_test.go @@ -138,6 +138,10 @@ func TestRegisterDecoderNormalizesLogicalCaseInsensitiveParameters(t *testing.T) registered: "multipart/signed; Protocol=\"Application/PGP-Signature\"", response: "Multipart/Signed; protocol=\"application/pgp-signature\"", }, + "multipart signed micalg": { + registered: "multipart/signed; Micalg=PGP-SHA256", + response: "Multipart/Signed; micalg=pgp-sha256", + }, "multipart encrypted protocol": { registered: "multipart/encrypted; Protocol=\"Application/PGP-Encrypted\"", response: "Multipart/Encrypted; protocol=\"application/pgp-encrypted\"", @@ -194,6 +198,10 @@ func TestDecoderContentTypeKeyKeepsContextualValuesCaseSensitive(t *testing.T) { contentType: "text/plain; Header=PRESENT", want: "text/plain; header=PRESENT", }, + "micalg outside multipart signed": { + contentType: "multipart/encrypted; Micalg=PGP-SHA256", + want: "multipart/encrypted; micalg=PGP-SHA256", + }, } { t.Run(name, func(t *testing.T) { if got := decoderContentTypeKey(test.contentType); got != test.want { diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index badb31f3..1236f2bc 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -196,8 +196,13 @@ func isCaseInsensitiveMediaParameterValue(mediaType string, name string) bool { case "access-type", "permission", "mode": return true } - case "multipart/encrypted", "multipart/signed": + case "multipart/encrypted": return strings.EqualFold(name, "protocol") + case "multipart/signed": + switch strings.ToLower(name) { + case "protocol", "micalg": + return true + } case "multipart/report": return strings.EqualFold(name, "report-type") case "multipart/related": From 8cfe9c5488d52677df5c6ce1823eb69a4ed7701a Mon Sep 17 00:00:00 2001 From: mcp-devbox Date: Sat, 12 Sep 2026 18:39:55 +0000 Subject: [PATCH 12/20] fix(ssestream): preserve protocol-defined parameter value case --- packages/ssestream/content_type_case_test.go | 13 ---- .../ssestream/content_type_semantics_test.go | 74 +++++++++++++++++++ packages/ssestream/ssestream.go | 34 +++++---- 3 files changed, 94 insertions(+), 27 deletions(-) create mode 100644 packages/ssestream/content_type_semantics_test.go diff --git a/packages/ssestream/content_type_case_test.go b/packages/ssestream/content_type_case_test.go index eeccce56..228ceb3b 100644 --- a/packages/ssestream/content_type_case_test.go +++ b/packages/ssestream/content_type_case_test.go @@ -82,11 +82,6 @@ func TestRegisterDecoderNormalizesExternalBodyCaseInsensitiveValues(t *testing.T registered: "READ-WRITE", response: "read-write", }, - "mode": { - parameter: "Mode", - registered: "IMAGE", - response: "image", - }, } { t.Run(name, func(t *testing.T) { registered := "message/external-body; " + test.parameter + "=" + test.registered @@ -138,10 +133,6 @@ func TestRegisterDecoderNormalizesLogicalCaseInsensitiveParameters(t *testing.T) registered: "multipart/signed; Protocol=\"Application/PGP-Signature\"", response: "Multipart/Signed; protocol=\"application/pgp-signature\"", }, - "multipart signed micalg": { - registered: "multipart/signed; Micalg=PGP-SHA256", - response: "Multipart/Signed; micalg=pgp-sha256", - }, "multipart encrypted protocol": { registered: "multipart/encrypted; Protocol=\"Application/PGP-Encrypted\"", response: "Multipart/Encrypted; protocol=\"application/pgp-encrypted\"", @@ -150,10 +141,6 @@ func TestRegisterDecoderNormalizesLogicalCaseInsensitiveParameters(t *testing.T) registered: "multipart/report; Report-Type=DELIVERY-STATUS", response: "Multipart/Report; report-type=delivery-status", }, - "text csv header": { - registered: "text/csv; Header=PRESENT", - response: "Text/CSV; header=present", - }, "unencoded format continuation": { registered: "text/plain; Format*0=FLO; Format*1=WED", response: "Text/Plain; format*0=flo; format*1=wed", diff --git a/packages/ssestream/content_type_semantics_test.go b/packages/ssestream/content_type_semantics_test.go new file mode 100644 index 00000000..819d645a --- /dev/null +++ b/packages/ssestream/content_type_semantics_test.go @@ -0,0 +1,74 @@ +package ssestream + +import ( + "io" + "net/http" + "strings" + "testing" +) + +func TestRegisterDecoderDoesNotFoldProtocolDefinedOrExtensionValues(t *testing.T) { + for name, test := range map[string]struct { + base string + registered string + response string + }{ + "multipart signed micalg": { + base: "multipart/signed", + registered: `multipart/signed; protocol="application/x-test-signature"; micalg=V1`, + response: `multipart/signed; protocol="application/x-test-signature"; micalg=v1`, + }, + "text csv header": { + base: "text/csv", + registered: "text/csv; header=PRESENT", + response: "text/csv; header=present", + }, + "external body extension mode": { + base: "message/external-body", + registered: "message/external-body; access-type=X-TEST; mode=V1", + response: "message/external-body; access-type=x-test; mode=v1", + }, + } { + t.Run(name, func(t *testing.T) { + wantBare := &testDecoder{} + wantSpecific := &testDecoder{} + RegisterDecoder(test.base, func(io.ReadCloser) Decoder { return wantBare }) + RegisterDecoder(test.registered, func(io.ReadCloser) Decoder { return wantSpecific }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(test.base)) + delete(decoderTypes, decoderContentTypeKey(test.registered)) + }) + + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.response}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != wantBare { + t.Fatalf("decoder = %T, want bare decoder for distinct parameter value", decoder) + } + }) + } +} + +func TestRegisterDecoderFoldsExternalBodyModeForStandardAccessTypes(t *testing.T) { + for _, accessType := range []string{"FTP", "ANON-FTP", "TFTP"} { + t.Run(accessType, func(t *testing.T) { + registered := "message/external-body; access-type=" + accessType + "; mode=IMAGE" + want := &testDecoder{} + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(registered)) + }) + + decoder := NewDecoder(&http.Response{ + Header: http.Header{ + "Content-Type": {"Message/External-Body; access-type=" + strings.ToLower(accessType) + "; mode=image"}, + }, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index 1236f2bc..7f74a0cd 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -72,10 +72,16 @@ func decoderContentTypeKey(contentType string) string { return strings.ToLower(contentType) } normalizedBase := strings.ToLower(base) - return normalizedBase + ";" + normalizeMediaParameterTail(normalizedBase, params) + externalBodyAccessType := "" + if strings.EqualFold(strings.TrimSpace(normalizedBase), "message/external-body") { + if _, parsedParams, err := mime.ParseMediaType(contentType); err == nil { + externalBodyAccessType = strings.ToLower(parsedParams["access-type"]) + } + } + return normalizedBase + ";" + normalizeMediaParameterTail(normalizedBase, params, externalBodyAccessType) } -func normalizeMediaParameterTail(mediaType string, params string) string { +func normalizeMediaParameterTail(mediaType string, params string, externalBodyAccessType string) string { var normalized strings.Builder segmentStart := 0 inQuotes := false @@ -83,7 +89,7 @@ func normalizeMediaParameterTail(mediaType string, params string) string { for i := 0; i <= len(params); i++ { if i == len(params) || (!inQuotes && params[i] == ';') { - normalized.WriteString(normalizeMediaParameter(mediaType, params[segmentStart:i])) + normalized.WriteString(normalizeMediaParameter(mediaType, params[segmentStart:i], externalBodyAccessType)) if i < len(params) { normalized.WriteByte(';') } @@ -108,7 +114,7 @@ func normalizeMediaParameterTail(mediaType string, params string) string { return normalized.String() } -func normalizeMediaParameter(mediaType string, param string) string { +func normalizeMediaParameter(mediaType string, param string, externalBodyAccessType string) string { equals := strings.IndexByte(param, '=') if equals < 0 { return param @@ -130,7 +136,7 @@ func normalizeMediaParameter(mediaType string, param string) string { value := param[equals+1:] switch { - case isCaseInsensitiveMediaParameterValue(mediaType, logicalName): + case isCaseInsensitiveMediaParameterValue(mediaType, logicalName, externalBodyAccessType): if strings.HasSuffix(name, "*") { normalized.WriteString(normalizeCaseInsensitiveExtendedParameterValue(value, extendedMediaParameterHasMetadata(name))) } else { @@ -185,30 +191,30 @@ func isRFC2231Section(section string) bool { return true } -func isCaseInsensitiveMediaParameterValue(mediaType string, name string) bool { +func isCaseInsensitiveMediaParameterValue(mediaType string, name string, externalBodyAccessType string) bool { if strings.EqualFold(name, "charset") { return true } - switch strings.ToLower(strings.TrimSpace(mediaType)) { + switch strings.TrimSpace(mediaType) { case "message/external-body": switch strings.ToLower(name) { - case "access-type", "permission", "mode": + case "access-type", "permission": return true + case "mode": + switch externalBodyAccessType { + case "ftp", "anon-ftp", "tftp": + return true + } } case "multipart/encrypted": return strings.EqualFold(name, "protocol") case "multipart/signed": - switch strings.ToLower(name) { - case "protocol", "micalg": - return true - } + return strings.EqualFold(name, "protocol") case "multipart/report": return strings.EqualFold(name, "report-type") case "multipart/related": return strings.EqualFold(name, "type") - case "text/csv": - return strings.EqualFold(name, "header") case "text/plain": switch strings.ToLower(name) { case "format", "delsp": From 940739a852b63f3b9e648ec35762e37b616f2334 Mon Sep 17 00:00:00 2001 From: mcp-devbox Date: Sat, 12 Sep 2026 19:28:46 +0000 Subject: [PATCH 13/20] fix(ssestream): handle extended external-body access types --- .../ssestream/content_type_semantics_test.go | 45 +++++++ packages/ssestream/ssestream.go | 114 ++++++++++++++++-- 2 files changed, 150 insertions(+), 9 deletions(-) diff --git a/packages/ssestream/content_type_semantics_test.go b/packages/ssestream/content_type_semantics_test.go index 819d645a..7f1ab46f 100644 --- a/packages/ssestream/content_type_semantics_test.go +++ b/packages/ssestream/content_type_semantics_test.go @@ -28,6 +28,11 @@ func TestRegisterDecoderDoesNotFoldProtocolDefinedOrExtensionValues(t *testing.T registered: "message/external-body; access-type=X-TEST; mode=V1", response: "message/external-body; access-type=x-test; mode=v1", }, + "external body extended extension mode": { + base: "message/external-body", + registered: "message/external-body; access-type*=ISO-8859-1''X-TEST; mode=V1", + response: "message/external-body; access-type*=iso-8859-1''x-test; mode=v1", + }, } { t.Run(name, func(t *testing.T) { wantBare := &testDecoder{} @@ -50,6 +55,46 @@ func TestRegisterDecoderDoesNotFoldProtocolDefinedOrExtensionValues(t *testing.T } } +func TestRegisterDecoderFoldsExternalBodyModeWithUnsupportedExtendedCharset(t *testing.T) { + for name, test := range map[string]struct { + registered string + response string + }{ + "single extended value": { + registered: "message/external-body; access-type*=ISO-8859-1''FTP; mode=IMAGE", + response: "Message/External-Body; access-type*=iso-8859-1''ftp; mode=image", + }, + "continued extended value": { + registered: "message/external-body; access-type*0*=ISO-8859-1''ANON%2D; access-type*1*=FTP; mode=IMAGE", + response: "Message/External-Body; access-type*0*=iso-8859-1''anon%2d; access-type*1*=ftp; mode=image", + }, + "quoted extended value": { + registered: "message/external-body; access-type*=\"ISO-8859-1''TFTP\"; mode=IMAGE", + response: "Message/External-Body; access-type*=\"iso-8859-1''tftp\"; mode=image", + }, + "mode before access type": { + registered: "message/external-body; mode=IMAGE; access-type*=ISO-8859-1''FTP", + response: "Message/External-Body; mode=image; access-type*=iso-8859-1''ftp", + }, + } { + t.Run(name, func(t *testing.T) { + want := &testDecoder{} + RegisterDecoder(test.registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(test.registered)) + }) + + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.response}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + func TestRegisterDecoderFoldsExternalBodyModeForStandardAccessTypes(t *testing.T) { for _, accessType := range []string{"FTP", "ANON-FTP", "TFTP"} { t.Run(accessType, func(t *testing.T) { diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index 7f74a0cd..d22986d6 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -74,25 +74,123 @@ func decoderContentTypeKey(contentType string) string { normalizedBase := strings.ToLower(base) externalBodyAccessType := "" if strings.EqualFold(strings.TrimSpace(normalizedBase), "message/external-body") { - if _, parsedParams, err := mime.ParseMediaType(contentType); err == nil { - externalBodyAccessType = strings.ToLower(parsedParams["access-type"]) - } + externalBodyAccessType = parseExternalBodyAccessType(contentType, params) } return normalizedBase + ";" + normalizeMediaParameterTail(normalizedBase, params, externalBodyAccessType) } +func parseExternalBodyAccessType(contentType string, params string) string { + _, parsedParams, err := mime.ParseMediaType(contentType) + if err != nil { + return "" + } + if accessType := standardExternalBodyAccessType(parsedParams["access-type"]); accessType != "" { + return accessType + } + + asciiParams, changed := replaceExtendedParameterCharset(params, "access-type") + if !changed { + return "" + } + _, parsedParams, err = mime.ParseMediaType("message/external-body;" + asciiParams) + if err != nil { + return "" + } + return standardExternalBodyAccessType(parsedParams["access-type"]) +} + +func standardExternalBodyAccessType(accessType string) string { + switch strings.ToLower(accessType) { + case "ftp", "anon-ftp", "tftp": + return strings.ToLower(accessType) + default: + return "" + } +} + +// replaceExtendedParameterCharset lets the standard library assemble RFC 2231 +// continuations even when it does not recognize the declared charset. The +// external-body access mechanisms handled here are ASCII tokens, so only the +// metadata charset is substituted; encoded value bytes remain unchanged. +func replaceExtendedParameterCharset(params string, logicalName string) (string, bool) { + var rewritten strings.Builder + changed := false + first := true + forEachMediaParameter(params, func(param string) { + if !first { + rewritten.WriteByte(';') + } + first = false + + equals := strings.IndexByte(param, '=') + if equals < 0 { + rewritten.WriteString(param) + return + } + nameStart, nameEnd := trimOWSBounds(param[:equals]) + name := param[nameStart:nameEnd] + if !strings.EqualFold(mediaParameterLogicalName(name), logicalName) || + !strings.HasSuffix(name, "*") || !extendedMediaParameterHasMetadata(name) { + rewritten.WriteString(param) + return + } + + value := param[equals+1:] + valueStart, valueEnd := trimOWSBounds(value) + core := value[valueStart:valueEnd] + quoted := false + if strings.HasPrefix(core, "\"") { + var ok bool + core, ok = quotedMediaParameterContents(core) + if !ok { + rewritten.WriteString(param) + return + } + quoted = true + } + firstQuote := strings.IndexByte(core, '\'') + if firstQuote < 0 || strings.IndexByte(core[firstQuote+1:], '\'') < 0 { + rewritten.WriteString(param) + return + } + + rewritten.WriteString(param[:equals+1]) + rewritten.WriteString(value[:valueStart]) + if quoted { + rewritten.WriteByte('"') + } + rewritten.WriteString("US-ASCII") + rewritten.WriteString(core[firstQuote:]) + if quoted { + rewritten.WriteByte('"') + } + rewritten.WriteString(value[valueEnd:]) + changed = true + }) + return rewritten.String(), changed +} + func normalizeMediaParameterTail(mediaType string, params string, externalBodyAccessType string) string { var normalized strings.Builder + first := true + forEachMediaParameter(params, func(param string) { + if !first { + normalized.WriteByte(';') + } + first = false + normalized.WriteString(normalizeMediaParameter(mediaType, param, externalBodyAccessType)) + }) + return normalized.String() +} + +func forEachMediaParameter(params string, visit func(string)) { segmentStart := 0 inQuotes := false escaped := false for i := 0; i <= len(params); i++ { if i == len(params) || (!inQuotes && params[i] == ';') { - normalized.WriteString(normalizeMediaParameter(mediaType, params[segmentStart:i], externalBodyAccessType)) - if i < len(params) { - normalized.WriteByte(';') - } + visit(params[segmentStart:i]) segmentStart = i + 1 continue } @@ -110,8 +208,6 @@ func normalizeMediaParameterTail(mediaType string, params string, externalBodyAc } escaped = false } - - return normalized.String() } func normalizeMediaParameter(mediaType string, param string, externalBodyAccessType string) string { From 71e5e79f8d69ffdc1e52c4e9a94af55145c3ea6d Mon Sep 17 00:00:00 2001 From: mcp-devbox Date: Sat, 12 Sep 2026 19:52:12 +0000 Subject: [PATCH 14/20] fix(ssestream): reject partial extended access types --- .../ssestream/content_type_semantics_test.go | 30 +++++++ packages/ssestream/ssestream.go | 84 +++++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/packages/ssestream/content_type_semantics_test.go b/packages/ssestream/content_type_semantics_test.go index 7f1ab46f..c928bb40 100644 --- a/packages/ssestream/content_type_semantics_test.go +++ b/packages/ssestream/content_type_semantics_test.go @@ -55,6 +55,36 @@ func TestRegisterDecoderDoesNotFoldProtocolDefinedOrExtensionValues(t *testing.T } } +func TestRegisterDecoderDoesNotFoldExternalBodyModeForMalformedContinuation(t *testing.T) { + const base = "message/external-body" + for name, accessType := range map[string]string{ + "malformed section zero": "access-type*0*=UTF-8''%ZZ; access-type*1*=FTP", + "malformed later section": "access-type*0*=UTF-8''FTP; access-type*1*=%ZZ", + "missing section": "access-type*0*=UTF-8''FTP; access-type*2*=X", + } { + t.Run(name, func(t *testing.T) { + registered := base + "; " + accessType + "; mode=IMAGE" + response := base + "; " + accessType + "; mode=image" + wantBare := &testDecoder{} + wantSpecific := &testDecoder{} + RegisterDecoder(base, func(io.ReadCloser) Decoder { return wantBare }) + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return wantSpecific }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(base)) + delete(decoderTypes, decoderContentTypeKey(registered)) + }) + + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {response}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != wantBare { + t.Fatalf("decoder = %T, want bare decoder for malformed continuation", decoder) + } + }) + } +} + func TestRegisterDecoderFoldsExternalBodyModeWithUnsupportedExtendedCharset(t *testing.T) { for name, test := range map[string]struct { registered string diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index d22986d6..e80ddeae 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -8,6 +8,7 @@ import ( "io" "mime" "net/http" + "strconv" "strings" "sync" "sync/atomic" @@ -80,6 +81,10 @@ func decoderContentTypeKey(contentType string) string { } func parseExternalBodyAccessType(contentType string, params string) string { + if !validExtendedParameterContinuations(params, "access-type") { + return "" + } + _, parsedParams, err := mime.ParseMediaType(contentType) if err != nil { return "" @@ -99,6 +104,85 @@ func parseExternalBodyAccessType(contentType string, params string) string { return standardExternalBodyAccessType(parsedParams["access-type"]) } +// validExtendedParameterContinuations rejects partial RFC 2231 values before +// mime.ParseMediaType can silently assemble a later valid segment on its own. +func validExtendedParameterContinuations(params string, logicalName string) bool { + sections := map[int]struct{}{} + maxSection := -1 + sawSingle := false + valid := true + + forEachMediaParameter(params, func(param string) { + equals := strings.IndexByte(param, '=') + if !valid || equals < 0 { + return + } + nameStart, nameEnd := trimOWSBounds(param[:equals]) + name := param[nameStart:nameEnd] + if !strings.EqualFold(mediaParameterLogicalName(name), logicalName) || strings.EqualFold(name, logicalName) { + return + } + + encoded := strings.HasSuffix(name, "*") + sectionName := strings.TrimSuffix(name, "*") + if strings.EqualFold(sectionName, logicalName) { + valid = encoded && !sawSingle && len(sections) == 0 && validEncodedParameterValue(param[equals+1:], true) + sawSingle = valid + return + } + + star := strings.LastIndexByte(sectionName, '*') + section, err := strconv.Atoi(sectionName[star+1:]) + if err != nil || section > len(params) || sawSingle { + valid = false + return + } + if _, duplicate := sections[section]; duplicate { + valid = false + return + } + if encoded && !validEncodedParameterValue(param[equals+1:], section == 0) { + valid = false + return + } + sections[section] = struct{}{} + maxSection = max(maxSection, section) + }) + + return valid && (maxSection < 0 || len(sections) == maxSection+1) +} + +func validEncodedParameterValue(value string, hasMetadata bool) bool { + valueStart, valueEnd := trimOWSBounds(value) + core := value[valueStart:valueEnd] + if strings.HasPrefix(core, "\"") { + var ok bool + core, ok = quotedMediaParameterContents(core) + if !ok { + return false + } + } + if hasMetadata { + if firstQuote := strings.IndexByte(core, '\''); firstQuote >= 0 { + secondQuote := strings.IndexByte(core[firstQuote+1:], '\'') + if secondQuote < 0 { + return false + } + core = core[firstQuote+secondQuote+2:] + } + } + for i := 0; i < len(core); i++ { + if core[i] != '%' { + continue + } + if i+2 >= len(core) || !isHexDigit(core[i+1]) || !isHexDigit(core[i+2]) { + return false + } + i += 2 + } + return true +} + func standardExternalBodyAccessType(accessType string) string { switch strings.ToLower(accessType) { case "ftp", "anon-ftp", "tftp": From 187200c061409ba76a35bd181bc4c6337a2c55b2 Mon Sep 17 00:00:00 2001 From: mcp-devbox Date: Sat, 12 Sep 2026 20:21:08 +0000 Subject: [PATCH 15/20] fix(ssestream): require extended access-type metadata --- packages/ssestream/content_type_semantics_test.go | 7 +++++++ packages/ssestream/ssestream.go | 14 ++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/ssestream/content_type_semantics_test.go b/packages/ssestream/content_type_semantics_test.go index c928bb40..4207419c 100644 --- a/packages/ssestream/content_type_semantics_test.go +++ b/packages/ssestream/content_type_semantics_test.go @@ -33,6 +33,11 @@ func TestRegisterDecoderDoesNotFoldProtocolDefinedOrExtensionValues(t *testing.T registered: "message/external-body; access-type*=ISO-8859-1''X-TEST; mode=V1", response: "message/external-body; access-type*=iso-8859-1''x-test; mode=v1", }, + "smime type": { + base: "application/pkcs7-mime", + registered: "application/pkcs7-mime; smime-type=SIGNED-DATA", + response: "application/pkcs7-mime; smime-type=signed-data", + }, } { t.Run(name, func(t *testing.T) { wantBare := &testDecoder{} @@ -61,6 +66,8 @@ func TestRegisterDecoderDoesNotFoldExternalBodyModeForMalformedContinuation(t *t "malformed section zero": "access-type*0*=UTF-8''%ZZ; access-type*1*=FTP", "malformed later section": "access-type*0*=UTF-8''FTP; access-type*1*=%ZZ", "missing section": "access-type*0*=UTF-8''FTP; access-type*2*=X", + "missing metadata": "access-type*0*=BROKEN; access-type*1*=FTP", + "empty charset": "access-type*0*=''FTP; access-type*1*=X", } { t.Run(name, func(t *testing.T) { registered := base + "; " + accessType + "; mode=IMAGE" diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index e80ddeae..5799c0dc 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -163,13 +163,15 @@ func validEncodedParameterValue(value string, hasMetadata bool) bool { } } if hasMetadata { - if firstQuote := strings.IndexByte(core, '\''); firstQuote >= 0 { - secondQuote := strings.IndexByte(core[firstQuote+1:], '\'') - if secondQuote < 0 { - return false - } - core = core[firstQuote+secondQuote+2:] + firstQuote := strings.IndexByte(core, '\'') + if firstQuote <= 0 { + return false + } + secondQuote := strings.IndexByte(core[firstQuote+1:], '\'') + if secondQuote < 0 { + return false } + core = core[firstQuote+secondQuote+2:] } for i := 0; i < len(core); i++ { if core[i] != '%' { From a4b362333ac7d4f225f7cf187ba0abb4e725bae5 Mon Sep 17 00:00:00 2001 From: mcp-devbox Date: Sat, 12 Sep 2026 20:50:29 +0000 Subject: [PATCH 16/20] fix(ssestream): validate extended access-type metadata --- .../ssestream/content_type_semantics_test.go | 11 +++- packages/ssestream/ssestream.go | 54 +++++++++++++++++-- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/packages/ssestream/content_type_semantics_test.go b/packages/ssestream/content_type_semantics_test.go index 4207419c..91454510 100644 --- a/packages/ssestream/content_type_semantics_test.go +++ b/packages/ssestream/content_type_semantics_test.go @@ -67,7 +67,8 @@ func TestRegisterDecoderDoesNotFoldExternalBodyModeForMalformedContinuation(t *t "malformed later section": "access-type*0*=UTF-8''FTP; access-type*1*=%ZZ", "missing section": "access-type*0*=UTF-8''FTP; access-type*2*=X", "missing metadata": "access-type*0*=BROKEN; access-type*1*=FTP", - "empty charset": "access-type*0*=''FTP; access-type*1*=X", + "invalid charset syntax": `access-type*="BAD CHAR''FTP"`, + "invalid language syntax": `access-type*="UTF-8'BAD LANG'FTP"`, } { t.Run(name, func(t *testing.T) { registered := base + "; " + accessType + "; mode=IMAGE" @@ -113,6 +114,14 @@ func TestRegisterDecoderFoldsExternalBodyModeWithUnsupportedExtendedCharset(t *t registered: "message/external-body; mode=IMAGE; access-type*=ISO-8859-1''FTP", response: "Message/External-Body; mode=image; access-type*=iso-8859-1''ftp", }, + "empty charset and language": { + registered: "message/external-body; access-type*=''FTP; mode=IMAGE", + response: "Message/External-Body; access-type*=''ftp; mode=image", + }, + "valid language tag": { + registered: "message/external-body; access-type*=UTF-8'en-US'FTP; mode=IMAGE", + response: "Message/External-Body; access-type*=utf-8'EN-us'ftp; mode=image", + }, } { t.Run(name, func(t *testing.T) { want := &testDecoder{} diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index 5799c0dc..c8b0ba17 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -164,14 +164,23 @@ func validEncodedParameterValue(value string, hasMetadata bool) bool { } if hasMetadata { firstQuote := strings.IndexByte(core, '\'') - if firstQuote <= 0 { + if firstQuote < 0 { return false } - secondQuote := strings.IndexByte(core[firstQuote+1:], '\'') - if secondQuote < 0 { + secondOffset := strings.IndexByte(core[firstQuote+1:], '\'') + if secondOffset < 0 { return false } - core = core[firstQuote+secondQuote+2:] + secondQuote := firstQuote + secondOffset + 1 + charset := core[:firstQuote] + language := core[firstQuote+1 : secondQuote] + if charset != "" && !isMIMECharset(charset) { + return false + } + if language != "" && !isRFC1766LanguageTag(language) { + return false + } + core = core[secondQuote+1:] } for i := 0; i < len(core); i++ { if core[i] != '%' { @@ -185,6 +194,43 @@ func validEncodedParameterValue(value string, hasMetadata bool) bool { return true } +func isMIMECharset(charset string) bool { + for i := 0; i < len(charset); i++ { + c := charset[i] + if c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' { + continue + } + switch c { + case '!', '#', '$', '%', '&', '+', '-', '^', '_', '`', '~': + continue + default: + return false + } + } + return len(charset) > 0 +} + +func isRFC1766LanguageTag(language string) bool { + partLength := 0 + for i := 0; i <= len(language); i++ { + if i == len(language) || language[i] == '-' { + if partLength == 0 || partLength > 8 { + return false + } + partLength = 0 + continue + } + c := language[i] + if c < 'A' || c > 'Z' { + if c < 'a' || c > 'z' { + return false + } + } + partLength++ + } + return true +} + func standardExternalBodyAccessType(accessType string) string { switch strings.ToLower(accessType) { case "ftp", "anon-ftp", "tftp": From dd8129cd5c8e5d1e7da9f5aceec03e0da47c887a Mon Sep 17 00:00:00 2001 From: mcp-devbox Date: Sat, 12 Sep 2026 21:38:46 +0000 Subject: [PATCH 17/20] fix(ssestream): decode extended access-type charsets --- go.mod | 2 +- .../ssestream/content_type_semantics_test.go | 22 ++ packages/ssestream/ssestream.go | 311 ++++++++++-------- 3 files changed, 199 insertions(+), 136 deletions(-) diff --git a/go.mod b/go.mod index 229b9daa..4cb6089e 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.32.35 github.com/tidwall/gjson v1.19.0 github.com/tidwall/sjson v1.2.5 + golang.org/x/text v0.40.0 ) require ( @@ -35,5 +36,4 @@ require ( golang.org/x/crypto v0.54.0 // indirect golang.org/x/net v0.57.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.40.0 // indirect ) diff --git a/packages/ssestream/content_type_semantics_test.go b/packages/ssestream/content_type_semantics_test.go index 91454510..003a63f2 100644 --- a/packages/ssestream/content_type_semantics_test.go +++ b/packages/ssestream/content_type_semantics_test.go @@ -33,6 +33,16 @@ func TestRegisterDecoderDoesNotFoldProtocolDefinedOrExtensionValues(t *testing.T registered: "message/external-body; access-type*=ISO-8859-1''X-TEST; mode=V1", response: "message/external-body; access-type*=iso-8859-1''x-test; mode=v1", }, + "external body utf16 extension mode": { + base: "message/external-body", + registered: "message/external-body; access-type*=UTF-16BE''%00X%00-%00T%00E%00S%00T; mode=V1", + response: "message/external-body; access-type*=utf-16be''%00x%00-%00t%00e%00s%00t; mode=v1", + }, + "external body unsupported utf32 mode": { + base: "message/external-body", + registered: "message/external-body; access-type*=UTF-32BE''%00%00%00F%00%00%00T%00%00%00P; mode=IMAGE", + response: "message/external-body; access-type*=utf-32be''%00%00%00f%00%00%00t%00%00%00p; mode=image", + }, "smime type": { base: "application/pkcs7-mime", registered: "application/pkcs7-mime; smime-type=SIGNED-DATA", @@ -122,6 +132,18 @@ func TestRegisterDecoderFoldsExternalBodyModeWithUnsupportedExtendedCharset(t *t registered: "message/external-body; access-type*=UTF-8'en-US'FTP; mode=IMAGE", response: "Message/External-Body; access-type*=utf-8'EN-us'ftp; mode=image", }, + "utf16be value": { + registered: "message/external-body; access-type*=UTF-16BE''%00F%00T%00P; mode=IMAGE", + response: "Message/External-Body; access-type*=utf-16be''%00f%00t%00p; mode=image", + }, + "utf16be continued value": { + registered: "message/external-body; access-type*0*=UTF-16BE''%00F%00; access-type*1*=T%00P; mode=IMAGE", + response: "Message/External-Body; access-type*0*=utf-16be''%00f%00; access-type*1*=t%00p; mode=image", + }, + "ebcdic value": { + registered: "message/external-body; access-type*=IBM037''%C6%E3%D7; mode=IMAGE", + response: "Message/External-Body; access-type*=ibm037''%86%A3%97; mode=image", + }, } { t.Run(name, func(t *testing.T) { want := &testDecoder{} diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index c8b0ba17..2e5b8b35 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -15,6 +15,7 @@ import ( shimjson "github.com/openai/openai-go/v3/internal/encoding/json" "github.com/tidwall/gjson" + "golang.org/x/text/encoding/ianaindex" ) type Decoder interface { @@ -74,42 +75,42 @@ func decoderContentTypeKey(contentType string) string { } normalizedBase := strings.ToLower(base) externalBodyAccessType := "" + hasExternalBodyAccessType := false if strings.EqualFold(strings.TrimSpace(normalizedBase), "message/external-body") { - externalBodyAccessType = parseExternalBodyAccessType(contentType, params) + externalBodyAccessType, hasExternalBodyAccessType = parseExternalBodyAccessType(contentType, params) } - return normalizedBase + ";" + normalizeMediaParameterTail(normalizedBase, params, externalBodyAccessType) + return normalizedBase + ";" + normalizeMediaParameterTail(normalizedBase, params, externalBodyAccessType, hasExternalBodyAccessType) } -func parseExternalBodyAccessType(contentType string, params string) string { - if !validExtendedParameterContinuations(params, "access-type") { - return "" +func parseExternalBodyAccessType(contentType string, params string) (string, bool) { + if accessType, found, ok := decodeExtendedMediaParameter(params, "access-type"); found { + if !ok { + return "", false + } + return strings.ToLower(accessType), true } _, parsedParams, err := mime.ParseMediaType(contentType) if err != nil { - return "" + return "", false } - if accessType := standardExternalBodyAccessType(parsedParams["access-type"]); accessType != "" { - return accessType + accessType, ok := parsedParams["access-type"] + if !ok { + return "", false } + return strings.ToLower(accessType), true +} - asciiParams, changed := replaceExtendedParameterCharset(params, "access-type") - if !changed { - return "" - } - _, parsedParams, err = mime.ParseMediaType("message/external-body;" + asciiParams) - if err != nil { - return "" - } - return standardExternalBodyAccessType(parsedParams["access-type"]) +type extendedMediaParameterSegment struct { + encoded bool + value string } -// validExtendedParameterContinuations rejects partial RFC 2231 values before -// mime.ParseMediaType can silently assemble a later valid segment on its own. -func validExtendedParameterContinuations(params string, logicalName string) bool { - sections := map[int]struct{}{} - maxSection := -1 - sawSingle := false +func decodeExtendedMediaParameter(params string, logicalName string) (string, bool, bool) { + var single extendedMediaParameterSegment + hasSingle := false + sections := map[int]extendedMediaParameterSegment{} + found := false valid := true forEachMediaParameter(params, func(param string) { @@ -122,76 +123,173 @@ func validExtendedParameterContinuations(params string, logicalName string) bool if !strings.EqualFold(mediaParameterLogicalName(name), logicalName) || strings.EqualFold(name, logicalName) { return } + found = true encoded := strings.HasSuffix(name, "*") sectionName := strings.TrimSuffix(name, "*") + segment := extendedMediaParameterSegment{encoded: encoded, value: param[equals+1:]} if strings.EqualFold(sectionName, logicalName) { - valid = encoded && !sawSingle && len(sections) == 0 && validEncodedParameterValue(param[equals+1:], true) - sawSingle = valid + if hasSingle || len(sections) != 0 || !encoded { + valid = false + return + } + single = segment + hasSingle = true return } star := strings.LastIndexByte(sectionName, '*') - section, err := strconv.Atoi(sectionName[star+1:]) - if err != nil || section > len(params) || sawSingle { + if star < 0 { valid = false return } - if _, duplicate := sections[section]; duplicate { + section, err := strconv.Atoi(sectionName[star+1:]) + if err != nil || hasSingle { valid = false return } - if encoded && !validEncodedParameterValue(param[equals+1:], section == 0) { + if _, duplicate := sections[section]; duplicate { valid = false return } - sections[section] = struct{}{} - maxSection = max(maxSection, section) + sections[section] = segment }) - return valid && (maxSection < 0 || len(sections) == maxSection+1) -} - -func validEncodedParameterValue(value string, hasMetadata bool) bool { - valueStart, valueEnd := trimOWSBounds(value) - core := value[valueStart:valueEnd] - if strings.HasPrefix(core, "\"") { - var ok bool - core, ok = quotedMediaParameterContents(core) + if !found { + return "", false, false + } + if !valid { + return "", true, false + } + if hasSingle { + core, ok := decodedMediaParameterCore(single.value) if !ok { - return false + return "", true, false } + charset, data, ok := splitExtendedInitialValue(core) + if !ok { + return "", true, false + } + raw, ok := decodeExtendedOctets(data) + if !ok { + return "", true, false + } + decoded, ok := decodeMIMEParameterValue(charset, raw) + return decoded, true, ok } - if hasMetadata { - firstQuote := strings.IndexByte(core, '\'') - if firstQuote < 0 { - return false + if len(sections) == 0 { + return "", true, false + } + + var raw []byte + charset := "" + for section := 0; section < len(sections); section++ { + segment, ok := sections[section] + if !ok { + return "", true, false } - secondOffset := strings.IndexByte(core[firstQuote+1:], '\'') - if secondOffset < 0 { - return false + core, ok := decodedMediaParameterCore(segment.value) + if !ok { + return "", true, false } - secondQuote := firstQuote + secondOffset + 1 - charset := core[:firstQuote] - language := core[firstQuote+1 : secondQuote] - if charset != "" && !isMIMECharset(charset) { - return false + data := core + if section == 0 && segment.encoded { + charset, data, ok = splitExtendedInitialValue(core) + if !ok { + return "", true, false + } } - if language != "" && !isRFC1766LanguageTag(language) { - return false + + var octets []byte + if segment.encoded { + octets, ok = decodeExtendedOctets(data) + if !ok { + return "", true, false + } + } else { + octets = []byte(data) + } + raw = append(raw, octets...) + } + + decoded, ok := decodeMIMEParameterValue(charset, raw) + return decoded, true, ok +} + +func decodedMediaParameterCore(value string) (string, bool) { + valueStart, valueEnd := trimOWSBounds(value) + core := value[valueStart:valueEnd] + if !strings.HasPrefix(core, "\"") { + return core, true + } + contents, ok := quotedMediaParameterContents(core) + if !ok { + return "", false + } + + var decoded strings.Builder + for i := 0; i < len(contents); i++ { + if contents[i] == '\\' { + if i+1 >= len(contents) { + return "", false + } + i++ } - core = core[secondQuote+1:] + decoded.WriteByte(contents[i]) + } + return decoded.String(), true +} + +func splitExtendedInitialValue(value string) (string, string, bool) { + firstQuote := strings.IndexByte(value, '\'') + if firstQuote < 0 { + return "", "", false + } + secondOffset := strings.IndexByte(value[firstQuote+1:], '\'') + if secondOffset < 0 { + return "", "", false + } + secondQuote := firstQuote + secondOffset + 1 + charset := value[:firstQuote] + language := value[firstQuote+1 : secondQuote] + if charset != "" && !isMIMECharset(charset) { + return "", "", false } - for i := 0; i < len(core); i++ { - if core[i] != '%' { + if language != "" && !isRFC1766LanguageTag(language) { + return "", "", false + } + return charset, value[secondQuote+1:], true +} + +func decodeExtendedOctets(value string) ([]byte, bool) { + decoded := make([]byte, 0, len(value)) + for i := 0; i < len(value); i++ { + if value[i] != '%' { + decoded = append(decoded, value[i]) continue } - if i+2 >= len(core) || !isHexDigit(core[i+1]) || !isHexDigit(core[i+2]) { - return false + if i+2 >= len(value) || !isHexDigit(value[i+1]) || !isHexDigit(value[i+2]) { + return nil, false } + decoded = append(decoded, hexValue(value[i+1])<<4|hexValue(value[i+2])) i += 2 } - return true + return decoded, true +} + +func decodeMIMEParameterValue(charset string, value []byte) (string, bool) { + if charset == "" { + return string(value), true + } + encoding, err := ianaindex.MIME.Encoding(charset) + if err != nil || encoding == nil { + return "", false + } + decoded, err := encoding.NewDecoder().Bytes(value) + if err != nil { + return "", false + } + return string(decoded), true } func isMIMECharset(charset string) bool { @@ -231,78 +329,7 @@ func isRFC1766LanguageTag(language string) bool { return true } -func standardExternalBodyAccessType(accessType string) string { - switch strings.ToLower(accessType) { - case "ftp", "anon-ftp", "tftp": - return strings.ToLower(accessType) - default: - return "" - } -} - -// replaceExtendedParameterCharset lets the standard library assemble RFC 2231 -// continuations even when it does not recognize the declared charset. The -// external-body access mechanisms handled here are ASCII tokens, so only the -// metadata charset is substituted; encoded value bytes remain unchanged. -func replaceExtendedParameterCharset(params string, logicalName string) (string, bool) { - var rewritten strings.Builder - changed := false - first := true - forEachMediaParameter(params, func(param string) { - if !first { - rewritten.WriteByte(';') - } - first = false - - equals := strings.IndexByte(param, '=') - if equals < 0 { - rewritten.WriteString(param) - return - } - nameStart, nameEnd := trimOWSBounds(param[:equals]) - name := param[nameStart:nameEnd] - if !strings.EqualFold(mediaParameterLogicalName(name), logicalName) || - !strings.HasSuffix(name, "*") || !extendedMediaParameterHasMetadata(name) { - rewritten.WriteString(param) - return - } - - value := param[equals+1:] - valueStart, valueEnd := trimOWSBounds(value) - core := value[valueStart:valueEnd] - quoted := false - if strings.HasPrefix(core, "\"") { - var ok bool - core, ok = quotedMediaParameterContents(core) - if !ok { - rewritten.WriteString(param) - return - } - quoted = true - } - firstQuote := strings.IndexByte(core, '\'') - if firstQuote < 0 || strings.IndexByte(core[firstQuote+1:], '\'') < 0 { - rewritten.WriteString(param) - return - } - - rewritten.WriteString(param[:equals+1]) - rewritten.WriteString(value[:valueStart]) - if quoted { - rewritten.WriteByte('"') - } - rewritten.WriteString("US-ASCII") - rewritten.WriteString(core[firstQuote:]) - if quoted { - rewritten.WriteByte('"') - } - rewritten.WriteString(value[valueEnd:]) - changed = true - }) - return rewritten.String(), changed -} - -func normalizeMediaParameterTail(mediaType string, params string, externalBodyAccessType string) string { +func normalizeMediaParameterTail(mediaType string, params string, externalBodyAccessType string, hasExternalBodyAccessType bool) string { var normalized strings.Builder first := true forEachMediaParameter(params, func(param string) { @@ -310,7 +337,7 @@ func normalizeMediaParameterTail(mediaType string, params string, externalBodyAc normalized.WriteByte(';') } first = false - normalized.WriteString(normalizeMediaParameter(mediaType, param, externalBodyAccessType)) + normalized.WriteString(normalizeMediaParameter(mediaType, param, externalBodyAccessType, hasExternalBodyAccessType)) }) return normalized.String() } @@ -342,7 +369,7 @@ func forEachMediaParameter(params string, visit func(string)) { } } -func normalizeMediaParameter(mediaType string, param string, externalBodyAccessType string) string { +func normalizeMediaParameter(mediaType string, param string, externalBodyAccessType string, hasExternalBodyAccessType bool) string { equals := strings.IndexByte(param, '=') if equals < 0 { return param @@ -363,7 +390,21 @@ func normalizeMediaParameter(mediaType string, param string, externalBodyAccessT normalized.WriteByte('=') value := param[equals+1:] + isExternalBodyAccessType := strings.TrimSpace(mediaType) == "message/external-body" && strings.EqualFold(logicalName, "access-type") switch { + case isExternalBodyAccessType && hasExternalBodyAccessType: + valueStart, valueEnd := trimOWSBounds(value) + normalized.WriteString(value[:valueStart]) + normalized.WriteString(externalBodyAccessType) + normalized.WriteString(value[valueEnd:]) + case isExternalBodyAccessType && !strings.EqualFold(name, logicalName): + // If an extended access type uses an unsupported charset, preserve its + // value semantics instead of case-folding encoded bytes as ASCII. + if strings.HasSuffix(name, "*") { + normalized.WriteString(normalizeExtendedParameterValue(value, extendedMediaParameterHasMetadata(name))) + } else { + normalized.WriteString(value) + } case isCaseInsensitiveMediaParameterValue(mediaType, logicalName, externalBodyAccessType): if strings.HasSuffix(name, "*") { normalized.WriteString(normalizeCaseInsensitiveExtendedParameterValue(value, extendedMediaParameterHasMetadata(name))) From 08f6e9203ab19f2e030d1f66296ac12c805220b3 Mon Sep 17 00:00:00 2001 From: mcp-devbox Date: Sun, 13 Sep 2026 00:53:01 +0000 Subject: [PATCH 18/20] fix(ssestream): isolate decoded access-type keys --- .../ssestream/content_type_semantics_test.go | 141 ++++++++++++++++ packages/ssestream/ssestream.go | 156 ++++++++++-------- 2 files changed, 229 insertions(+), 68 deletions(-) diff --git a/packages/ssestream/content_type_semantics_test.go b/packages/ssestream/content_type_semantics_test.go index 003a63f2..8e62c305 100644 --- a/packages/ssestream/content_type_semantics_test.go +++ b/packages/ssestream/content_type_semantics_test.go @@ -132,6 +132,10 @@ func TestRegisterDecoderFoldsExternalBodyModeWithUnsupportedExtendedCharset(t *t registered: "message/external-body; access-type*=UTF-8'en-US'FTP; mode=IMAGE", response: "Message/External-Body; access-type*=utf-8'EN-us'ftp; mode=image", }, + "registered charset alias with period": { + registered: "message/external-body; access-type*=ANSI_X3.4-1968''FTP; mode=IMAGE", + response: "Message/External-Body; access-type*=ansi_x3.4-1968''ftp; mode=image", + }, "utf16be value": { registered: "message/external-body; access-type*=UTF-16BE''%00F%00T%00P; mode=IMAGE", response: "Message/External-Body; access-type*=utf-16be''%00f%00t%00p; mode=image", @@ -163,6 +167,143 @@ func TestRegisterDecoderFoldsExternalBodyModeWithUnsupportedExtendedCharset(t *t } } +func TestRegisterDecoderExternalBodyAccessTypeKeysDoNotCollide(t *testing.T) { + tests := map[string]struct { + first string + second string + }{ + "decoded extended delimiter": { + first: "message/external-body;access-type*=UTF-8''FTP%3Bmode%3DIMAGE", + second: "message/external-body;access-type*=UTF-8''FTP;mode=image", + }, + "quoted ordinary delimiter": { + first: `message/external-body;access-type="FTP;mode=IMAGE"`, + second: "message/external-body;access-type=FTP;mode=image", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + firstDecoder := &testDecoder{} + secondDecoder := &testDecoder{} + RegisterDecoder(test.first, func(io.ReadCloser) Decoder { return firstDecoder }) + RegisterDecoder(test.second, func(io.ReadCloser) Decoder { return secondDecoder }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(test.first)) + delete(decoderTypes, decoderContentTypeKey(test.second)) + }) + + if firstKey, secondKey := decoderContentTypeKey(test.first), decoderContentTypeKey(test.second); firstKey == secondKey { + t.Fatalf("distinct Content-Type values share decoder key %q", firstKey) + } + + for label, response := range map[string]struct { + contentType string + want Decoder + }{ + "first": {contentType: test.first, want: firstDecoder}, + "second": {contentType: test.second, want: secondDecoder}, + } { + t.Run(label, func(t *testing.T) { + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {response.contentType}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != response.want { + t.Fatalf("decoder = %T, want independently registered decoder", decoder) + } + }) + } + }) + } +} + +func TestRegisterDecoderDoesNotFoldExternalBodyModeForAmbiguousAccessType(t *testing.T) { + const base = "message/external-body" + for name, accessTypes := range map[string]string{ + "plain then extended": "access-type=X-TEST; access-type*=UTF-8''FTP", + "extended then plain": "access-type*=UTF-8''FTP; access-type=X-TEST", + } { + t.Run(name, func(t *testing.T) { + registered := base + "; " + accessTypes + "; mode=IMAGE" + response := base + "; " + accessTypes + "; mode=image" + wantSpecific := &testDecoder{} + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return wantSpecific }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(registered)) + }) + + if registeredKey, responseKey := decoderContentTypeKey(registered), decoderContentTypeKey(response); registeredKey == responseKey { + t.Fatalf("ambiguous duplicate access-type folded mode into shared key %q", registeredKey) + } + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {response}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder == wantSpecific { + t.Fatal("ambiguous duplicate access-type selected parameter-specific decoder") + } + }) + } +} + +func TestRegisterDecoderFoldsCaseInsensitiveExtendedValuesAcrossCharsets(t *testing.T) { + for name, test := range map[string]struct { + registered string + response string + }{ + "charset": { + registered: `text/plain; charset*=IBM037''%E4%E3%C6%60%F8`, + response: `text/plain; charset*=ibm037''%A4%A3%86%60%F8`, + }, + "external permission": { + registered: `message/external-body; permission*=IBM037''%D9%C5%C1%C4`, + response: `message/external-body; permission*=ibm037''%99%85%81%84`, + }, + "external mode": { + registered: `message/external-body; access-type=FTP; mode*=IBM037''%C9%D4%C1%C7%C5`, + response: `message/external-body; access-type=ftp; mode*=ibm037''%89%94%81%87%85`, + }, + "multipart encrypted protocol": { + registered: `multipart/encrypted; protocol*=IBM037''%C1%D7%D7%D3%C9%C3%C1%E3%C9%D6%D5%61%E3%C5%E2%E3`, + response: `multipart/encrypted; protocol*=ibm037''%81%97%97%93%89%83%81%A3%89%96%95%61%A3%85%A2%A3`, + }, + "multipart signed protocol": { + registered: `multipart/signed; protocol*=IBM037''%C1%D7%D7%D3%C9%C3%C1%E3%C9%D6%D5%61%E3%C5%E2%E3`, + response: `multipart/signed; protocol*=ibm037''%81%97%97%93%89%83%81%A3%89%96%95%61%A3%85%A2%A3`, + }, + "multipart report type": { + registered: `multipart/report; report-type*=IBM037''%C4%C5%D3%C9%E5%C5%D9%E8%60%E2%E3%C1%E3%E4%E2`, + response: `multipart/report; report-type*=ibm037''%84%85%93%89%A5%85%99%A8%60%A2%A3%81%A3%A4%A2`, + }, + "multipart related type": { + registered: `multipart/related; type*=IBM037''%C1%D7%D7%D3%C9%C3%C1%E3%C9%D6%D5%61%E3%C5%E2%E3`, + response: `multipart/related; type*=ibm037''%81%97%97%93%89%83%81%A3%89%96%95%61%A3%85%A2%A3`, + }, + "text plain format": { + registered: `text/plain; format*=IBM037''%C6%D3%D6%E6%C5%C4`, + response: `text/plain; format*=ibm037''%86%93%96%A6%85%84`, + }, + "text plain delsp": { + registered: `text/plain; delsp*=IBM037''%E8%C5%E2`, + response: `text/plain; delsp*=ibm037''%A8%85%A2`, + }, + } { + t.Run(name, func(t *testing.T) { + want := &testDecoder{} + RegisterDecoder(test.registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { delete(decoderTypes, decoderContentTypeKey(test.registered)) }) + decoder := NewDecoder(&http.Response{ + Header: http.Header{"Content-Type": {test.response}}, + Body: io.NopCloser(strings.NewReader("")), + }) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + func TestRegisterDecoderFoldsExternalBodyModeForStandardAccessTypes(t *testing.T) { for _, accessType := range []string{"FTP", "ANON-FTP", "TFTP"} { t.Run(accessType, func(t *testing.T) { diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index 2e5b8b35..3268f407 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -3,6 +3,7 @@ package ssestream import ( "bufio" "bytes" + "encoding/hex" "encoding/json" "fmt" "io" @@ -110,6 +111,7 @@ func decodeExtendedMediaParameter(params string, logicalName string) (string, bo var single extendedMediaParameterSegment hasSingle := false sections := map[int]extendedMediaParameterSegment{} + plainFound := false found := false valid := true @@ -120,7 +122,19 @@ func decodeExtendedMediaParameter(params string, logicalName string) (string, bo } nameStart, nameEnd := trimOWSBounds(param[:equals]) name := param[nameStart:nameEnd] - if !strings.EqualFold(mediaParameterLogicalName(name), logicalName) || strings.EqualFold(name, logicalName) { + if !strings.EqualFold(mediaParameterLogicalName(name), logicalName) { + return + } + if strings.EqualFold(name, logicalName) { + if found { + valid = false + } + plainFound = true + return + } + if plainFound { + found = true + valid = false return } found = true @@ -252,8 +266,13 @@ func splitExtendedInitialValue(value string) (string, string, bool) { secondQuote := firstQuote + secondOffset + 1 charset := value[:firstQuote] language := value[firstQuote+1 : secondQuote] - if charset != "" && !isMIMECharset(charset) { - return "", "", false + if charset != "" { + if charset != strings.TrimSpace(charset) { + return "", "", false + } + if _, err := ianaindex.IANA.Encoding(charset); err != nil { + return "", "", false + } } if language != "" && !isRFC1766LanguageTag(language) { return "", "", false @@ -281,7 +300,7 @@ func decodeMIMEParameterValue(charset string, value []byte) (string, bool) { if charset == "" { return string(value), true } - encoding, err := ianaindex.MIME.Encoding(charset) + encoding, err := ianaindex.IANA.Encoding(charset) if err != nil || encoding == nil { return "", false } @@ -292,22 +311,6 @@ func decodeMIMEParameterValue(charset string, value []byte) (string, bool) { return string(decoded), true } -func isMIMECharset(charset string) bool { - for i := 0; i < len(charset); i++ { - c := charset[i] - if c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' { - continue - } - switch c { - case '!', '#', '$', '%', '&', '+', '-', '^', '_', '`', '~': - continue - default: - return false - } - } - return len(charset) > 0 -} - func isRFC1766LanguageTag(language string) bool { partLength := 0 for i := 0; i <= len(language); i++ { @@ -329,7 +332,14 @@ func isRFC1766LanguageTag(language string) bool { return true } +type decodedExtendedParameterState struct { + value string + found bool + decoded bool +} + func normalizeMediaParameterTail(mediaType string, params string, externalBodyAccessType string, hasExternalBodyAccessType bool) string { + decodedParameters := map[string]decodedExtendedParameterState{} var normalized strings.Builder first := true forEachMediaParameter(params, func(param string) { @@ -337,7 +347,7 @@ func normalizeMediaParameterTail(mediaType string, params string, externalBodyAc normalized.WriteByte(';') } first = false - normalized.WriteString(normalizeMediaParameter(mediaType, param, externalBodyAccessType, hasExternalBodyAccessType)) + normalized.WriteString(normalizeMediaParameter(mediaType, params, param, externalBodyAccessType, hasExternalBodyAccessType, decodedParameters)) }) return normalized.String() } @@ -369,7 +379,7 @@ func forEachMediaParameter(params string, visit func(string)) { } } -func normalizeMediaParameter(mediaType string, param string, externalBodyAccessType string, hasExternalBodyAccessType bool) string { +func normalizeMediaParameter(mediaType string, params string, param string, externalBodyAccessType string, hasExternalBodyAccessType bool, decodedParameters map[string]decodedExtendedParameterState) string { equals := strings.IndexByte(param, '=') if equals < 0 { return param @@ -392,22 +402,12 @@ func normalizeMediaParameter(mediaType string, param string, externalBodyAccessT value := param[equals+1:] isExternalBodyAccessType := strings.TrimSpace(mediaType) == "message/external-body" && strings.EqualFold(logicalName, "access-type") switch { - case isExternalBodyAccessType && hasExternalBodyAccessType: - valueStart, valueEnd := trimOWSBounds(value) - normalized.WriteString(value[:valueStart]) - normalized.WriteString(externalBodyAccessType) - normalized.WriteString(value[valueEnd:]) - case isExternalBodyAccessType && !strings.EqualFold(name, logicalName): - // If an extended access type uses an unsupported charset, preserve its - // value semantics instead of case-folding encoded bytes as ASCII. - if strings.HasSuffix(name, "*") { - normalized.WriteString(normalizeExtendedParameterValue(value, extendedMediaParameterHasMetadata(name))) - } else { - normalized.WriteString(value) - } + case isExternalBodyAccessType: + normalized.WriteString(normalizeCanonicalMediaParameterValue(name, value, "access-type", externalBodyAccessType, hasExternalBodyAccessType)) case isCaseInsensitiveMediaParameterValue(mediaType, logicalName, externalBodyAccessType): - if strings.HasSuffix(name, "*") { - normalized.WriteString(normalizeCaseInsensitiveExtendedParameterValue(value, extendedMediaParameterHasMetadata(name))) + state := decodedCaseInsensitiveParameter(params, logicalName, decodedParameters) + if state.found { + normalized.WriteString(normalizeCanonicalMediaParameterValue(name, value, logicalName, state.value, state.decoded)) } else { valueStart, valueEnd := trimOWSBounds(value) normalized.WriteString(value[:valueStart]) @@ -423,6 +423,57 @@ func normalizeMediaParameter(mediaType string, param string, externalBodyAccessT return normalized.String() } +func decodedCaseInsensitiveParameter(params string, logicalName string, cache map[string]decodedExtendedParameterState) decodedExtendedParameterState { + key := strings.ToLower(logicalName) + if state, ok := cache[key]; ok { + return state + } + value, found, decoded := decodeExtendedMediaParameter(params, logicalName) + if decoded { + value = strings.ToLower(value) + } + state := decodedExtendedParameterState{value: value, found: found, decoded: decoded} + cache[key] = state + return state +} + +func normalizeCanonicalMediaParameterValue(name string, value string, logicalName string, decodedValue string, decoded bool) string { + if decoded { + canonical := "d" + if isInitialMediaParameterSegment(name, logicalName) { + canonical = encodeDecoderKeyValue('d', decodedValue) + } + valueStart, valueEnd := trimOWSBounds(value) + return value[:valueStart] + canonical + value[valueEnd:] + } + + normalizedValue := value + if strings.HasSuffix(name, "*") { + normalizedValue = normalizeExtendedParameterValue(value, extendedMediaParameterHasMetadata(name)) + } + valueStart, valueEnd := trimOWSBounds(normalizedValue) + return normalizedValue[:valueStart] + encodeDecoderKeyValue('r', normalizedValue[valueStart:valueEnd]) + normalizedValue[valueEnd:] +} + +func encodeDecoderKeyValue(prefix byte, value string) string { + encoded := make([]byte, 1+hex.EncodedLen(len(value))) + encoded[0] = prefix + hex.Encode(encoded[1:], []byte(value)) + return string(encoded) +} + +func isInitialMediaParameterSegment(name string, logicalName string) bool { + if strings.EqualFold(name, logicalName) { + return true + } + sectionName := strings.TrimSuffix(name, "*") + if strings.EqualFold(sectionName, logicalName) { + return true + } + star := strings.LastIndexByte(sectionName, '*') + return star >= 0 && strings.EqualFold(sectionName[:star], logicalName) && sectionName[star+1:] == "0" +} + func mediaParameterLogicalName(name string) string { logicalName := strings.TrimSuffix(name, "*") section := strings.LastIndexByte(logicalName, '*') @@ -493,10 +544,6 @@ func isCaseInsensitiveMediaParameterValue(mediaType string, name string, externa return false } -func normalizeCaseInsensitiveExtendedParameterValue(value string, hasMetadata bool) string { - return normalizeExtendedParameterValueWithData(value, hasMetadata, normalizeCaseInsensitiveExtendedData) -} - func normalizeExtendedParameterValue(value string, hasMetadata bool) string { return normalizeExtendedParameterValueWithData(value, hasMetadata, normalizePercentEncoding) } @@ -565,26 +612,6 @@ func quotedMediaParameterContents(value string) (string, bool) { return value[1 : len(value)-1], true } -func normalizeCaseInsensitiveExtendedData(value string) string { - bytes := []byte(value) - for i := 0; i < len(bytes); i++ { - if bytes[i] == '%' && i+2 < len(bytes) && isHexDigit(bytes[i+1]) && isHexDigit(bytes[i+2]) { - decoded := hexValue(bytes[i+1])<<4 | hexValue(bytes[i+2]) - if decoded >= 'A' && decoded <= 'Z' { - decoded += 'a' - 'A' - } - bytes[i+1] = hexDigit(decoded >> 4) - bytes[i+2] = hexDigit(decoded & 0x0f) - i += 2 - continue - } - if bytes[i] >= 'A' && bytes[i] <= 'Z' { - bytes[i] += 'a' - 'A' - } - } - return string(bytes) -} - func hexValue(value byte) byte { switch { case value >= '0' && value <= '9': @@ -596,13 +623,6 @@ func hexValue(value byte) byte { } } -func hexDigit(value byte) byte { - if value < 10 { - return '0' + value - } - return 'a' + value - 10 -} - func normalizePercentEncoding(value string) string { bytes := []byte(value) for i := 0; i+2 < len(bytes); i++ { From 0dc13ee1d788091e83067b959afd22f74eb0de33 Mon Sep 17 00:00:00 2001 From: mcp-devbox Date: Sun, 13 Sep 2026 02:37:24 +0000 Subject: [PATCH 19/20] test(ssestream): document protocol-defined micalg case --- packages/ssestream/content_type_semantics_test.go | 6 ++++-- packages/ssestream/ssestream.go | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/ssestream/content_type_semantics_test.go b/packages/ssestream/content_type_semantics_test.go index 8e62c305..544d8858 100644 --- a/packages/ssestream/content_type_semantics_test.go +++ b/packages/ssestream/content_type_semantics_test.go @@ -13,8 +13,10 @@ func TestRegisterDecoderDoesNotFoldProtocolDefinedOrExtensionValues(t *testing.T registered string response string }{ - "multipart signed micalg": { - base: "multipart/signed", + "multipart signed protocol-defined micalg": { + base: "multipart/signed", + // RFC 1847 delegates micalg value semantics to the selected protocol. + // This extension protocol intentionally treats V1 and v1 as distinct. registered: `multipart/signed; protocol="application/x-test-signature"; micalg=V1`, response: `multipart/signed; protocol="application/x-test-signature"; micalg=v1`, }, diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index 3268f407..65bbad63 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -530,6 +530,9 @@ func isCaseInsensitiveMediaParameterValue(mediaType string, name string, externa case "multipart/encrypted": return strings.EqualFold(name, "protocol") case "multipart/signed": + // RFC 1847 makes micalg value syntax and semantics protocol-defined. + // RFC 2045 therefore leaves micalg case-sensitive unless that selected + // protocol explicitly defines otherwise; do not fold it generically. return strings.EqualFold(name, "protocol") case "multipart/report": return strings.EqualFold(name, "report-type") From b6c4c5e2232d76fa80a9cbaf3973009312c29c5c Mon Sep 17 00:00:00 2001 From: mcp-devbox Date: Sun, 13 Sep 2026 03:17:20 +0000 Subject: [PATCH 20/20] fix(ssestream): validate extended MIME values --- .../ssestream/content_type_semantics_test.go | 87 +++++ packages/ssestream/ssestream.go | 308 +++++++++++++----- 2 files changed, 314 insertions(+), 81 deletions(-) diff --git a/packages/ssestream/content_type_semantics_test.go b/packages/ssestream/content_type_semantics_test.go index 544d8858..1382c9aa 100644 --- a/packages/ssestream/content_type_semantics_test.go +++ b/packages/ssestream/content_type_semantics_test.go @@ -328,3 +328,90 @@ func TestRegisterDecoderFoldsExternalBodyModeForStandardAccessTypes(t *testing.T }) } } + +func TestRegisterDecoderRejectsMalformedUnquotedExtendedValueCollision(t *testing.T) { + registered := `multipart/signed; protocol*="UTF-8''application/pgp-signature"` + malformed := `multipart/signed; protocol*=UTF-8''application/pgp-signature` + want := &testDecoder{} + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { delete(decoderTypes, decoderContentTypeKey(registered)) }) + + if registeredKey, malformedKey := decoderContentTypeKey(registered), decoderContentTypeKey(malformed); registeredKey == malformedKey { + t.Fatalf("malformed unquoted extended value shares decoder key %q", registeredKey) + } + decoder := NewDecoder(&http.Response{Header: http.Header{"Content-Type": {malformed}}, Body: io.NopCloser(strings.NewReader(""))}) + if decoder == want { + t.Fatal("malformed unquoted extended value selected parameter-specific decoder") + } +} + +func TestRegisterDecoderFoldsTextCalendarMIMEParameters(t *testing.T) { + for name, test := range map[string]struct{ registered, response string }{ + "component": { + registered: "text/calendar; component=VEVENT", + response: "Text/Calendar; component=vevent", + }, + "method": { + registered: "text/calendar; method=REQUEST", + response: "Text/Calendar; method=request", + }, + } { + t.Run(name, func(t *testing.T) { + want := &testDecoder{} + RegisterDecoder(test.registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { delete(decoderTypes, decoderContentTypeKey(test.registered)) }) + decoder := NewDecoder(&http.Response{Header: http.Header{"Content-Type": {test.response}}, Body: io.NopCloser(strings.NewReader(""))}) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } + }) + } +} + +func TestRegisterDecoderDecodesMixedRFC2231ContinuationSegments(t *testing.T) { + registered := "text/plain; charset*0*=UTF-16BE''%00U; charset*1=TF-8" + response := "Text/Plain; charset*0*=utf-16be''%00u; charset*1=tf-8" + want := &testDecoder{} + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { delete(decoderTypes, decoderContentTypeKey(registered)) }) + decoder := NewDecoder(&http.Response{Header: http.Header{"Content-Type": {response}}, Body: io.NopCloser(strings.NewReader(""))}) + if decoder != want { + t.Fatalf("decoder = %T, want registered decoder", decoder) + } +} + +func TestRegisterDecoderPreservesRFC2231LanguageIdentity(t *testing.T) { + en := "text/plain; format*=UTF-8'en'FLOWED" + fr := "text/plain; format*=UTF-8'fr'FLOWED" + enResponse := "Text/Plain; format*=utf-8'EN'flowed" + wantEN := &testDecoder{} + wantFR := &testDecoder{} + RegisterDecoder(en, func(io.ReadCloser) Decoder { return wantEN }) + RegisterDecoder(fr, func(io.ReadCloser) Decoder { return wantFR }) + t.Cleanup(func() { + delete(decoderTypes, decoderContentTypeKey(en)) + delete(decoderTypes, decoderContentTypeKey(fr)) + }) + if enKey, frKey := decoderContentTypeKey(en), decoderContentTypeKey(fr); enKey == frKey { + t.Fatalf("distinct language tags share decoder key %q", enKey) + } + decoder := NewDecoder(&http.Response{Header: http.Header{"Content-Type": {enResponse}}, Body: io.NopCloser(strings.NewReader(""))}) + if decoder != wantEN { + t.Fatalf("decoder = %T, want English registration", decoder) + } +} + +func TestRegisterDecoderDoesNotFoldInvalidUnicodeIntoMIMEProtocol(t *testing.T) { + registered := "multipart/signed; protocol*=UTF-8''application%2FK" + malformed := "multipart/signed; protocol*=UTF-8''application%2F%E2%84%AA" + want := &testDecoder{} + RegisterDecoder(registered, func(io.ReadCloser) Decoder { return want }) + t.Cleanup(func() { delete(decoderTypes, decoderContentTypeKey(registered)) }) + if registeredKey, malformedKey := decoderContentTypeKey(registered), decoderContentTypeKey(malformed); registeredKey == malformedKey { + t.Fatalf("invalid Unicode protocol shares valid MIME decoder key %q", registeredKey) + } + decoder := NewDecoder(&http.Response{Header: http.Header{"Content-Type": {malformed}}, Body: io.NopCloser(strings.NewReader(""))}) + if decoder == want { + t.Fatal("invalid Unicode protocol selected valid parameter-specific decoder") + } +} diff --git a/packages/ssestream/ssestream.go b/packages/ssestream/ssestream.go index 65bbad63..9c2f1cba 100644 --- a/packages/ssestream/ssestream.go +++ b/packages/ssestream/ssestream.go @@ -72,34 +72,35 @@ func decoderContentTypes(contentType string) (string, string) { func decoderContentTypeKey(contentType string) string { base, params, found := strings.Cut(contentType, ";") if !found { - return strings.ToLower(contentType) + return asciiLower(contentType) } - normalizedBase := strings.ToLower(base) + normalizedBase := asciiLower(base) externalBodyAccessType := "" + externalBodyAccessLanguage := "" hasExternalBodyAccessType := false if strings.EqualFold(strings.TrimSpace(normalizedBase), "message/external-body") { - externalBodyAccessType, hasExternalBodyAccessType = parseExternalBodyAccessType(contentType, params) + externalBodyAccessType, externalBodyAccessLanguage, hasExternalBodyAccessType = parseExternalBodyAccessType(contentType, params) } - return normalizedBase + ";" + normalizeMediaParameterTail(normalizedBase, params, externalBodyAccessType, hasExternalBodyAccessType) + return normalizedBase + ";" + normalizeMediaParameterTail(normalizedBase, params, externalBodyAccessType, externalBodyAccessLanguage, hasExternalBodyAccessType) } -func parseExternalBodyAccessType(contentType string, params string) (string, bool) { - if accessType, found, ok := decodeExtendedMediaParameter(params, "access-type"); found { - if !ok { - return "", false +func parseExternalBodyAccessType(contentType string, params string) (string, string, bool) { + if accessType, language, found, ok := decodeExtendedMediaParameter(params, "access-type"); found { + if !ok || !isMIMEToken(accessType) { + return "", "", false } - return strings.ToLower(accessType), true + return asciiLower(accessType), asciiLower(language), true } _, parsedParams, err := mime.ParseMediaType(contentType) if err != nil { - return "", false + return "", "", false } accessType, ok := parsedParams["access-type"] - if !ok { - return "", false + if !ok || !isMIMEToken(accessType) { + return "", "", false } - return strings.ToLower(accessType), true + return asciiLower(accessType), "", true } type extendedMediaParameterSegment struct { @@ -107,7 +108,7 @@ type extendedMediaParameterSegment struct { value string } -func decodeExtendedMediaParameter(params string, logicalName string) (string, bool, bool) { +func decodeExtendedMediaParameter(params string, logicalName string) (string, string, bool, bool) { var single extendedMediaParameterSegment hasSingle := false sections := map[int]extendedMediaParameterSegment{} @@ -170,114 +171,198 @@ func decodeExtendedMediaParameter(params string, logicalName string) (string, bo }) if !found { - return "", false, false + return "", "", false, false } if !valid { - return "", true, false + return "", "", true, false } if hasSingle { - core, ok := decodedMediaParameterCore(single.value) + core, quoted, ok := mediaParameterValueCore(single.value) if !ok { - return "", true, false + return "", "", true, false } - charset, data, ok := splitExtendedInitialValue(core) - if !ok { - return "", true, false + charset, language, data, ok := splitExtendedInitialValue(core) + if !ok || (!quoted && !validRFC2231ExtendedData(data)) { + return "", "", true, false } raw, ok := decodeExtendedOctets(data) if !ok { - return "", true, false + return "", "", true, false } decoded, ok := decodeMIMEParameterValue(charset, raw) - return decoded, true, ok + return decoded, language, true, ok } if len(sections) == 0 { - return "", true, false + return "", "", true, false } - var raw []byte + var decoded strings.Builder + var encodedRun []byte charset := "" + language := "" + flushEncoded := func() bool { + if len(encodedRun) == 0 { + return true + } + text, ok := decodeMIMEParameterValue(charset, encodedRun) + if !ok { + return false + } + decoded.WriteString(text) + encodedRun = encodedRun[:0] + return true + } + for section := 0; section < len(sections); section++ { segment, ok := sections[section] if !ok { - return "", true, false + return "", "", true, false } - core, ok := decodedMediaParameterCore(segment.value) + core, quoted, ok := mediaParameterValueCore(segment.value) if !ok { - return "", true, false + return "", "", true, false } data := core if section == 0 && segment.encoded { - charset, data, ok = splitExtendedInitialValue(core) + charset, language, data, ok = splitExtendedInitialValue(core) if !ok { - return "", true, false + return "", "", true, false } } - var octets []byte if segment.encoded { - octets, ok = decodeExtendedOctets(data) + if !quoted && !validRFC2231ExtendedData(data) { + return "", "", true, false + } + octets, ok := decodeExtendedOctets(data) if !ok { - return "", true, false + return "", "", true, false } - } else { - octets = []byte(data) + encodedRun = append(encodedRun, octets...) + continue } - raw = append(raw, octets...) - } - decoded, ok := decodeMIMEParameterValue(charset, raw) - return decoded, true, ok + if !quoted && !isMIMEToken(data) { + return "", "", true, false + } + if !flushEncoded() { + return "", "", true, false + } + decoded.WriteString(data) + } + if !flushEncoded() { + return "", "", true, false + } + return decoded.String(), language, true, true } -func decodedMediaParameterCore(value string) (string, bool) { +func mediaParameterValueCore(value string) (string, bool, bool) { valueStart, valueEnd := trimOWSBounds(value) core := value[valueStart:valueEnd] if !strings.HasPrefix(core, "\"") { - return core, true + return core, false, true } contents, ok := quotedMediaParameterContents(core) if !ok { - return "", false + return "", false, false } var decoded strings.Builder for i := 0; i < len(contents); i++ { if contents[i] == '\\' { if i+1 >= len(contents) { - return "", false + return "", false, false } i++ } decoded.WriteByte(contents[i]) } - return decoded.String(), true + return decoded.String(), true, true } -func splitExtendedInitialValue(value string) (string, string, bool) { +func splitExtendedInitialValue(value string) (string, string, string, bool) { firstQuote := strings.IndexByte(value, '\'') if firstQuote < 0 { - return "", "", false + return "", "", "", false } secondOffset := strings.IndexByte(value[firstQuote+1:], '\'') if secondOffset < 0 { - return "", "", false + return "", "", "", false } secondQuote := firstQuote + secondOffset + 1 charset := value[:firstQuote] language := value[firstQuote+1 : secondQuote] if charset != "" { if charset != strings.TrimSpace(charset) { - return "", "", false + return "", "", "", false } if _, err := ianaindex.IANA.Encoding(charset); err != nil { - return "", "", false + return "", "", "", false } } if language != "" && !isRFC1766LanguageTag(language) { - return "", "", false + return "", "", "", false } - return charset, value[secondQuote+1:], true + return charset, language, value[secondQuote+1:], true +} + +func validRFC2231ExtendedData(value string) bool { + for i := 0; i < len(value); i++ { + if value[i] == '%' { + if i+2 >= len(value) || !isHexDigit(value[i+1]) || !isHexDigit(value[i+2]) { + return false + } + i += 2 + continue + } + if !isRFC2231AttributeChar(value[i]) { + return false + } + } + return true +} + +func isRFC2231AttributeChar(value byte) bool { + return isMIMETokenChar(value) && value != '*' && value != '\'' && value != '%' +} + +func isMIMEToken(value string) bool { + if value == "" { + return false + } + for i := 0; i < len(value); i++ { + if !isMIMETokenChar(value[i]) { + return false + } + } + return true +} + +func isMIMETokenChar(value byte) bool { + if value <= ' ' || value >= 0x7f { + return false + } + switch value { + case '(', ')', '<', '>', '@', ',', ';', ':', '\\', '"', '/', '[', ']', '?', '=': + return false + default: + return true + } +} + +func validMIMEMediaType(value string) bool { + typePart, subtype, found := strings.Cut(value, "/") + return found && !strings.Contains(subtype, "/") && isMIMEToken(typePart) && isMIMEToken(subtype) +} + +func asciiLower(value string) string { + bytes := []byte(value) + for i := range bytes { + if bytes[i] >= 'A' && bytes[i] <= 'Z' { + bytes[i] += 'a' - 'A' + } + } + return string(bytes) } func decodeExtendedOctets(value string) ([]byte, bool) { @@ -333,12 +418,13 @@ func isRFC1766LanguageTag(language string) bool { } type decodedExtendedParameterState struct { - value string - found bool - decoded bool + value string + language string + found bool + decoded bool } -func normalizeMediaParameterTail(mediaType string, params string, externalBodyAccessType string, hasExternalBodyAccessType bool) string { +func normalizeMediaParameterTail(mediaType string, params string, externalBodyAccessType string, externalBodyAccessLanguage string, hasExternalBodyAccessType bool) string { decodedParameters := map[string]decodedExtendedParameterState{} var normalized strings.Builder first := true @@ -347,7 +433,7 @@ func normalizeMediaParameterTail(mediaType string, params string, externalBodyAc normalized.WriteByte(';') } first = false - normalized.WriteString(normalizeMediaParameter(mediaType, params, param, externalBodyAccessType, hasExternalBodyAccessType, decodedParameters)) + normalized.WriteString(normalizeMediaParameter(mediaType, params, param, externalBodyAccessType, externalBodyAccessLanguage, hasExternalBodyAccessType, decodedParameters)) }) return normalized.String() } @@ -379,7 +465,7 @@ func forEachMediaParameter(params string, visit func(string)) { } } -func normalizeMediaParameter(mediaType string, params string, param string, externalBodyAccessType string, hasExternalBodyAccessType bool, decodedParameters map[string]decodedExtendedParameterState) string { +func normalizeMediaParameter(mediaType string, params string, param string, externalBodyAccessType string, externalBodyAccessLanguage string, hasExternalBodyAccessType bool, decodedParameters map[string]decodedExtendedParameterState) string { equals := strings.IndexByte(param, '=') if equals < 0 { return param @@ -395,7 +481,7 @@ func normalizeMediaParameter(mediaType string, params string, param string, exte var normalized strings.Builder normalized.WriteString(namePart[:nameStart]) - normalized.WriteString(strings.ToLower(name)) + normalized.WriteString(asciiLower(name)) normalized.WriteString(namePart[nameEnd:]) normalized.WriteByte('=') @@ -403,16 +489,21 @@ func normalizeMediaParameter(mediaType string, params string, param string, exte isExternalBodyAccessType := strings.TrimSpace(mediaType) == "message/external-body" && strings.EqualFold(logicalName, "access-type") switch { case isExternalBodyAccessType: - normalized.WriteString(normalizeCanonicalMediaParameterValue(name, value, "access-type", externalBodyAccessType, hasExternalBodyAccessType)) + normalized.WriteString(normalizeCanonicalMediaParameterValue(name, value, "access-type", externalBodyAccessType, externalBodyAccessLanguage, hasExternalBodyAccessType)) case isCaseInsensitiveMediaParameterValue(mediaType, logicalName, externalBodyAccessType): - state := decodedCaseInsensitiveParameter(params, logicalName, decodedParameters) + state := decodedCaseInsensitiveParameter(mediaType, params, logicalName, externalBodyAccessType, decodedParameters) if state.found { - normalized.WriteString(normalizeCanonicalMediaParameterValue(name, value, logicalName, state.value, state.decoded)) + normalized.WriteString(normalizeCanonicalMediaParameterValue(name, value, logicalName, state.value, state.language, state.decoded)) } else { - valueStart, valueEnd := trimOWSBounds(value) - normalized.WriteString(value[:valueStart]) - normalized.WriteString(strings.ToLower(value[valueStart:valueEnd])) - normalized.WriteString(value[valueEnd:]) + core, _, ok := mediaParameterValueCore(value) + if ok && validCaseInsensitiveMediaParameterValue(mediaType, logicalName, core, externalBodyAccessType) { + valueStart, valueEnd := trimOWSBounds(value) + normalized.WriteString(value[:valueStart]) + normalized.WriteString(asciiLower(value[valueStart:valueEnd])) + normalized.WriteString(value[valueEnd:]) + } else { + normalized.WriteString(value) + } } case strings.HasSuffix(name, "*"): normalized.WriteString(normalizeExtendedParameterValue(value, extendedMediaParameterHasMetadata(name))) @@ -423,25 +514,30 @@ func normalizeMediaParameter(mediaType string, params string, param string, exte return normalized.String() } -func decodedCaseInsensitiveParameter(params string, logicalName string, cache map[string]decodedExtendedParameterState) decodedExtendedParameterState { - key := strings.ToLower(logicalName) +func decodedCaseInsensitiveParameter(mediaType string, params string, logicalName string, externalBodyAccessType string, cache map[string]decodedExtendedParameterState) decodedExtendedParameterState { + key := asciiLower(logicalName) if state, ok := cache[key]; ok { return state } - value, found, decoded := decodeExtendedMediaParameter(params, logicalName) + value, language, found, decoded := decodeExtendedMediaParameter(params, logicalName) if decoded { - value = strings.ToLower(value) + if validCaseInsensitiveMediaParameterValue(mediaType, logicalName, value, externalBodyAccessType) { + value = asciiLower(value) + language = asciiLower(language) + } else { + decoded = false + } } - state := decodedExtendedParameterState{value: value, found: found, decoded: decoded} + state := decodedExtendedParameterState{value: value, language: language, found: found, decoded: decoded} cache[key] = state return state } -func normalizeCanonicalMediaParameterValue(name string, value string, logicalName string, decodedValue string, decoded bool) string { +func normalizeCanonicalMediaParameterValue(name string, value string, logicalName string, decodedValue string, decodedLanguage string, decoded bool) string { if decoded { canonical := "d" if isInitialMediaParameterSegment(name, logicalName) { - canonical = encodeDecoderKeyValue('d', decodedValue) + canonical = encodeDecodedDecoderKeyValue(decodedLanguage, decodedValue) } valueStart, valueEnd := trimOWSBounds(value) return value[:valueStart] + canonical + value[valueEnd:] @@ -455,6 +551,14 @@ func normalizeCanonicalMediaParameterValue(name string, value string, logicalNam return normalizedValue[:valueStart] + encodeDecoderKeyValue('r', normalizedValue[valueStart:valueEnd]) + normalizedValue[valueEnd:] } +func encodeDecodedDecoderKeyValue(language string, value string) string { + languageHex := make([]byte, hex.EncodedLen(len(language))) + hex.Encode(languageHex, []byte(language)) + valueHex := make([]byte, hex.EncodedLen(len(value))) + hex.Encode(valueHex, []byte(value)) + return "d" + string(languageHex) + "g" + string(valueHex) +} + func encodeDecoderKeyValue(prefix byte, value string) string { encoded := make([]byte, 1+hex.EncodedLen(len(value))) encoded[0] = prefix @@ -518,7 +622,7 @@ func isCaseInsensitiveMediaParameterValue(mediaType string, name string, externa switch strings.TrimSpace(mediaType) { case "message/external-body": - switch strings.ToLower(name) { + switch asciiLower(name) { case "access-type", "permission": return true case "mode": @@ -539,19 +643,61 @@ func isCaseInsensitiveMediaParameterValue(mediaType string, name string, externa case "multipart/related": return strings.EqualFold(name, "type") case "text/plain": - switch strings.ToLower(name) { + switch asciiLower(name) { case "format", "delsp": return true } + case "text/calendar": + switch asciiLower(name) { + case "method", "component": + return true + } } return false } -func normalizeExtendedParameterValue(value string, hasMetadata bool) string { - return normalizeExtendedParameterValueWithData(value, hasMetadata, normalizePercentEncoding) +func validCaseInsensitiveMediaParameterValue(mediaType string, name string, value string, externalBodyAccessType string) bool { + if strings.EqualFold(name, "charset") { + if value == "" { + return false + } + _, err := ianaindex.IANA.Encoding(value) + return err == nil + } + + switch strings.TrimSpace(mediaType) { + case "message/external-body": + switch asciiLower(name) { + case "access-type", "permission", "mode": + return isMIMEToken(value) + } + case "multipart/encrypted", "multipart/signed": + if strings.EqualFold(name, "protocol") { + return validMIMEMediaType(value) + } + case "multipart/report": + if strings.EqualFold(name, "report-type") { + return isMIMEToken(value) + } + case "multipart/related": + if strings.EqualFold(name, "type") { + return validMIMEMediaType(value) + } + case "text/plain": + switch asciiLower(name) { + case "format", "delsp": + return isMIMEToken(value) + } + case "text/calendar": + switch asciiLower(name) { + case "method", "component": + return isMIMEToken(value) + } + } + return false } -func normalizeExtendedParameterValueWithData(value string, hasMetadata bool, normalizeData func(string) string) string { +func normalizeExtendedParameterValue(value string, hasMetadata bool) string { valueStart, valueEnd := trimOWSBounds(value) core := value[valueStart:valueEnd] quoted := false @@ -577,11 +723,11 @@ func normalizeExtendedParameterValueWithData(value string, hasMetadata bool, nor var normalized string if firstQuote >= 0 && secondQuote >= 0 { - normalized = strings.ToLower(core[:firstQuote]) + "'" + - strings.ToLower(core[firstQuote+1:secondQuote]) + "'" + - normalizeData(core[secondQuote+1:]) + normalized = asciiLower(core[:firstQuote]) + "'" + + asciiLower(core[firstQuote+1:secondQuote]) + "'" + + normalizePercentEncoding(core[secondQuote+1:]) } else { - normalized = normalizeData(core) + normalized = normalizePercentEncoding(core) } if quoted { normalized = "\"" + normalized + "\""