From 603f0de2e47b9a0cbcddb523b9a915c0b9981a0e Mon Sep 17 00:00:00 2001 From: Matthias Dreher Date: Mon, 24 Jan 2022 09:11:18 +0100 Subject: [PATCH 1/5] Support concatenation of headers (as in https://github.com/esp8266/Arduino/commit/1de0c341b55ba8c0993fd3d2e0c5696935578751#diff-977435a9cc4619fa0b8b995085f6ae683485cf563722756bab57108b362da316 for ESP8266, fixes https://github.com/espressif/arduino-esp32/issues/4069) --- libraries/HTTPClient/src/HTTPClient.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/libraries/HTTPClient/src/HTTPClient.cpp b/libraries/HTTPClient/src/HTTPClient.cpp index a7bf13e89f5..9750bb8850f 100644 --- a/libraries/HTTPClient/src/HTTPClient.cpp +++ b/libraries/HTTPClient/src/HTTPClient.cpp @@ -1263,12 +1263,19 @@ int HTTPClient::handleHeaderResponse() _location = headerValue; } - for(size_t i = 0; i < _headerKeysCount; i++) { - if(_currentHeaders[i].key.equalsIgnoreCase(headerName)) { - _currentHeaders[i].value = headerValue; - break; + for (size_t i = 0; i < _headerKeysCount; i++) { + if (_currentHeaders[i].key.equalsIgnoreCase(headerName)) { + if (!_currentHeaders[i].value.isEmpty()) { + // Existing value, append this one with a comma + _currentHeaders[i].value += ','; + _currentHeaders[i].value += headerValue; + } else { + _currentHeaders[i].value = headerValue; + } + break; // We found a match, stop looking } } + } if(headerLine == "") { From 899ad9a28d8b68e32e45f6d0b667d78bc258418c Mon Sep 17 00:00:00 2001 From: Matthias Dreher Date: Sat, 29 Jan 2022 06:12:14 +0100 Subject: [PATCH 2/5] Add support for receiving, storing and sending cookies (cookie jar) --- libraries/HTTPClient/src/HTTPClient.cpp | 187 +++++++++++++++++++++++- libraries/HTTPClient/src/HTTPClient.h | 37 +++++ 2 files changed, 223 insertions(+), 1 deletion(-) diff --git a/libraries/HTTPClient/src/HTTPClient.cpp b/libraries/HTTPClient/src/HTTPClient.cpp index 9750bb8850f..3e9c91e34f7 100644 --- a/libraries/HTTPClient/src/HTTPClient.cpp +++ b/libraries/HTTPClient/src/HTTPClient.cpp @@ -39,6 +39,9 @@ #include "HTTPClient.h" +/// Cookie jar support +#include + #ifdef HTTPCLIENT_1_1_COMPATIBLE class TransportTraits { @@ -603,6 +606,12 @@ int HTTPClient::sendRequest(const char * type, uint8_t * payload, size_t size) addHeader(F("Content-Length"), String(size)); } + // add cookies to header, if present + String cookie_string; + if(generateCookieString(&cookie_string)) { + addHeader("Cookie", cookie_string); + } + // send Header if(!sendHeader(type)) { return returnError(HTTPC_ERROR_SEND_HEADER_FAILED); @@ -706,6 +715,12 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) addHeader("Content-Length", String(size)); } + // add cookies to header, if present + String cookie_string; + if(generateCookieString(&cookie_string)) { + addHeader("Cookie", cookie_string); + } + // send Header if(!sendHeader(type)) { return returnError(HTTPC_ERROR_SEND_HEADER_FAILED); @@ -1222,6 +1237,7 @@ int HTTPClient::handleHeaderResponse() _transferEncoding = HTTPC_TE_IDENTITY; unsigned long lastDataTime = millis(); bool firstLine = true; + String date; while(connected()) { size_t len = _client->available(); @@ -1234,7 +1250,7 @@ int HTTPClient::handleHeaderResponse() log_v("RX: '%s'", headerLine.c_str()); if(firstLine) { - firstLine = false; + firstLine = false; if(_canReuse && headerLine.startsWith("HTTP/1.")) { _canReuse = (headerLine[sizeof "HTTP/1." - 1] != '0'); } @@ -1245,6 +1261,10 @@ int HTTPClient::handleHeaderResponse() String headerValue = headerLine.substring(headerLine.indexOf(':') + 1); headerValue.trim(); + if(headerName.equalsIgnoreCase("Date")) { + date = headerValue; + } + if(headerName.equalsIgnoreCase("Content-Length")) { _size = headerValue.toInt(); } @@ -1263,6 +1283,10 @@ int HTTPClient::handleHeaderResponse() _location = headerValue; } + if (headerName.equalsIgnoreCase("Set-Cookie")) { + setCookie(date, headerValue); + } + for (size_t i = 0; i < _headerKeysCount; i++) { if (_currentHeaders[i].key.equalsIgnoreCase(headerName)) { if (!_currentHeaders[i].value.isEmpty()) { @@ -1498,3 +1522,164 @@ const String &HTTPClient::getLocation(void) { return _location; } + +void HTTPClient::setCookieJar(CookieJar* cookieJar) +{ + _cookieJar = cookieJar; +} + +void HTTPClient::resetCookieJar() +{ + _cookieJar = nullptr; +} + +void HTTPClient::clearAllCookies() +{ + if (_cookieJar) _cookieJar->clear(); +} + +void HTTPClient::setCookie(String date, String headerValue) +{ + #define HTTP_TIME_PATTERN "%a, %d %b %Y %H:%M:%S" + + Cookie cookie; + String value; + int pos1, pos2; + + headerValue.toLowerCase(); + + struct tm tm; + strptime(date.c_str(), HTTP_TIME_PATTERN, &tm); + cookie.date = mktime(&tm); + + pos1 = headerValue.indexOf('='); + pos2 = headerValue.indexOf(';'); + + if (pos1 >= 0 && pos2 > pos1){ + cookie.name = headerValue.substring(0, pos1); + cookie.value = headerValue.substring(pos1 + 1, pos2); + } else { + return; // invalid cookie header + } + + // expires + if (headerValue.indexOf("expires=") >= 0){ + pos1 = headerValue.indexOf("expires=") + strlen("expires="); + pos2 = headerValue.indexOf(';', pos1); + + if (pos2 > pos1) + value = headerValue.substring(pos1, pos2); + else + value = headerValue.substring(pos1); + + strptime(value.c_str(), HTTP_TIME_PATTERN, &tm); + cookie.expires.date = mktime(&tm); + cookie.expires.valid = true; + } + + // max-age + if (headerValue.indexOf("max-age=") >= 0){ + pos1 = headerValue.indexOf("max-age=") + strlen("max-age="); + pos2 = headerValue.indexOf(';', pos1); + + if (pos2 > pos1) + value = headerValue.substring(pos1, pos2); + else + value = headerValue.substring(pos1); + + cookie.max_age.duration = value.toInt(); + cookie.max_age.valid = true; + } + + // domain + if (headerValue.indexOf("domain=") >= 0){ + pos1 = headerValue.indexOf("domain=") + strlen("domain="); + pos2 = headerValue.indexOf(';', pos1); + + if (pos2 > pos1) + value = headerValue.substring(pos1, pos2); + else + value = headerValue.substring(pos1); + + if (value.startsWith(".")) value.remove(0, 1); + + if (_host.indexOf(value) >= 0) { + cookie.domain = value; + } else { + return; // server tries to set a cookie on a different domain; ignore it + } + } else { + pos1 = _host.lastIndexOf('.', _host.lastIndexOf('.') - 1); + if (pos1 >= 0) + cookie.domain = _host.substring(pos1 + 1); + else + cookie.domain = _host; + } + + // path + if (headerValue.indexOf("path=") >= 0){ + pos1 = headerValue.indexOf("path=") + strlen("path="); + pos2 = headerValue.indexOf(';', pos1); + + if (pos2 > pos1) + cookie.path = headerValue.substring(pos1, pos2); + else + cookie.path = headerValue.substring(pos1); + } + + // HttpOnly + cookie.http_only = (headerValue.indexOf("httponly") >= 0); + + // secure + cookie.secure = (headerValue.indexOf("secure") >= 0); + + // overwrite or delete cookie in/from cookie jar + time_t now_local = time(NULL); + time_t now_gmt = mktime(gmtime(&now_local)); + + bool found = false; + + for (auto c = _cookieJar->begin(); c != _cookieJar->end(); ++c) { + if (c->domain == cookie.domain && c->name == cookie.name) { + // when evaluating, max-age takes precedence over expires if both are defined + if (cookie.max_age.valid && ((cookie.date + cookie.max_age.duration) < now_gmt || cookie.max_age.duration <= 0) + || (!cookie.max_age.valid && cookie.expires.valid && cookie.expires.date < now_gmt)) { + _cookieJar->erase(c); + c--; + } else { + *c = cookie; + } + found = true; + } + } + + // add cookie to jar + if (!found && !(cookie.max_age.valid && cookie.max_age.duration <= 0)) + _cookieJar->push_back(cookie); + +} + +bool HTTPClient::generateCookieString(String *cookieString) +{ + time_t now_local = time(NULL); + time_t now_gmt = mktime(gmtime(&now_local)); + + *cookieString = ""; + bool found = false; + + for (auto c = _cookieJar->begin(); c != _cookieJar->end(); ++c) { + if (c->max_age.valid && ((c->date + c->max_age.duration) < now_gmt) || (!c->max_age.valid && c->expires.valid && c->expires.date < now_gmt)) { + _cookieJar->erase(c); + c--; + } else if (_host.indexOf(c->domain) >= 0) { + if (*cookieString == "") + *cookieString = c->name + "=" + c->value; + else + *cookieString += " ;" + c->name + "=" + c->value; + found = true; + } + } + return found; +} + + diff --git a/libraries/HTTPClient/src/HTTPClient.h b/libraries/HTTPClient/src/HTTPClient.h index 1bb84d6d6b5..be4c843e616 100644 --- a/libraries/HTTPClient/src/HTTPClient.h +++ b/libraries/HTTPClient/src/HTTPClient.h @@ -34,6 +34,9 @@ #include #include +/// Cookie jar support +#include + #define HTTPCLIENT_DEFAULT_TCP_TIMEOUT (5000) /// HTTP client errors @@ -142,6 +145,28 @@ class TransportTraits; typedef std::unique_ptr TransportTraitsPtr; #endif +// cookie jar support +typedef struct { + String host; // host which tries to set the cookie + time_t date; // timestamp of the response that set the cookie + String name; + String value; + String domain; + String path = ""; + struct { + time_t date = 0; + bool valid = false; + } expires; + struct { + time_t duration = 0; + bool valid = false; + } max_age; + bool http_only = false; + bool secure = false; +} Cookie; +typedef std::vector CookieJar; + + class HTTPClient { public: @@ -215,6 +240,11 @@ class HTTPClient static String errorToString(int error); + /// Cookie jar support + void setCookieJar(CookieJar* cookieJar); + void resetCookieJar(); + void clearAllCookies(); + protected: struct RequestArgument { String key; @@ -230,6 +260,9 @@ class HTTPClient int handleHeaderResponse(); int writeToStreamDataBlock(Stream * stream, int len); + /// Cookie jar support + void setCookie(String date, String headerValue); + bool generateCookieString(String *cookieString); #ifdef HTTPCLIENT_1_1_COMPATIBLE TransportTraitsPtr _transportTraits; @@ -265,6 +298,10 @@ class HTTPClient uint16_t _redirectLimit = 10; String _location; transferEncoding_t _transferEncoding = HTTPC_TE_IDENTITY; + + /// Cookie jar support + CookieJar* _cookieJar = nullptr; + }; From 8e3df763018bf32f6e312c44dd33d713a140c453 Mon Sep 17 00:00:00 2001 From: Matthias Dreher Date: Mon, 31 Jan 2022 18:47:07 +0100 Subject: [PATCH 3/5] Cookie support: Respect `secure` attribute when sending a request --- libraries/HTTPClient/src/HTTPClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/HTTPClient/src/HTTPClient.cpp b/libraries/HTTPClient/src/HTTPClient.cpp index 3e9c91e34f7..63757c9062c 100644 --- a/libraries/HTTPClient/src/HTTPClient.cpp +++ b/libraries/HTTPClient/src/HTTPClient.cpp @@ -1671,7 +1671,7 @@ bool HTTPClient::generateCookieString(String *cookieString) if (c->max_age.valid && ((c->date + c->max_age.duration) < now_gmt) || (!c->max_age.valid && c->expires.valid && c->expires.date < now_gmt)) { _cookieJar->erase(c); c--; - } else if (_host.indexOf(c->domain) >= 0) { + } else if (_host.indexOf(c->domain) >= 0 && (!c->secure || _secure) ) { if (*cookieString == "") *cookieString = c->name + "=" + c->value; else From e682273fc19f075525e7689f83bb278db7ce4652 Mon Sep 17 00:00:00 2001 From: Matthias Dreher Date: Mon, 31 Jan 2022 23:55:22 +0100 Subject: [PATCH 4/5] Fix missing `_secure` flag --- libraries/HTTPClient/src/HTTPClient.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libraries/HTTPClient/src/HTTPClient.cpp b/libraries/HTTPClient/src/HTTPClient.cpp index 63757c9062c..b29d5e01213 100644 --- a/libraries/HTTPClient/src/HTTPClient.cpp +++ b/libraries/HTTPClient/src/HTTPClient.cpp @@ -160,6 +160,7 @@ bool HTTPClient::begin(WiFiClient &client, String url) { } _port = (protocol == "https" ? 443 : 80); + _secure = (protocol == "https"); return beginInternal(url, protocol.c_str()); } @@ -190,6 +191,7 @@ bool HTTPClient::begin(WiFiClient &client, String host, uint16_t port, String ur _port = port; _uri = uri; _protocol = (https ? "https" : "http"); + _secure = https; return true; } From bd8ec7d397549bddc6965c4236db790b0ce3d34e Mon Sep 17 00:00:00 2001 From: Matthias Dreher Date: Fri, 4 Feb 2022 17:51:14 +0100 Subject: [PATCH 5/5] Comment out support concatenation of headers (not needed anymore when using cookie jar) --- libraries/HTTPClient/src/HTTPClient.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/libraries/HTTPClient/src/HTTPClient.cpp b/libraries/HTTPClient/src/HTTPClient.cpp index b29d5e01213..250a695e0fe 100644 --- a/libraries/HTTPClient/src/HTTPClient.cpp +++ b/libraries/HTTPClient/src/HTTPClient.cpp @@ -1291,13 +1291,14 @@ int HTTPClient::handleHeaderResponse() for (size_t i = 0; i < _headerKeysCount; i++) { if (_currentHeaders[i].key.equalsIgnoreCase(headerName)) { - if (!_currentHeaders[i].value.isEmpty()) { - // Existing value, append this one with a comma - _currentHeaders[i].value += ','; - _currentHeaders[i].value += headerValue; - } else { - _currentHeaders[i].value = headerValue; - } + // Uncomment the following lines if you need to add support for multiple headers with the same key: + // if (!_currentHeaders[i].value.isEmpty()) { + // // Existing value, append this one with a comma + // _currentHeaders[i].value += ','; + // _currentHeaders[i].value += headerValue; + // } else { + _currentHeaders[i].value = headerValue; + // } break; // We found a match, stop looking } }