Merge pull request ethereum#12740 from ethereum/remove-locale-dependent-operations

chriseth · web-flow · commit 77c40cd2afbd · 2022-03-07T17:56:58.000+01:00
Replace all locale-dependent operations with locale-agnostic counterparts
diff --git a/libevmasm/AssemblyItem.cpp b/libevmasm/AssemblyItem.cpp
@@ -200,9 +200,7 @@ string AssemblyItem::toAssemblyText(Assembly const& _assembly) const
 	case Operation:
 	{
 		assertThrow(isValidInstruction(instruction()), AssemblyException, "Invalid instruction.");
-		string name = instructionInfo(instruction()).name;
-		transform(name.begin(), name.end(), name.begin(), [](unsigned char _c) { return tolower(_c); });
-		text = name;
+		text = util::toLower(instructionInfo(instruction()).name);
 		break;
 	}
 	case Push:
diff --git a/liblangutil/Token.cpp b/liblangutil/Token.cpp
@@ -40,10 +40,13 @@
 // You should have received a copy of the GNU General Public License
 // along with solidity.  If not, see <http://www.gnu.org/licenses/>.
 
-#include <liblangutil/Token.h>
 #include <liblangutil/Exceptions.h>
+#include <liblangutil/Token.h>
+#include <libsolutil/StringUtils.h>
+
 #include <map>
 
+
 using namespace std;
 
 namespace solidity::langutil
@@ -180,11 +183,11 @@ tuple<Token, unsigned int, unsigned int> fromIdentifierOrKeyword(string const& _
 		return ret;
 	};
 
-	auto positionM = find_if(_literal.begin(), _literal.end(), ::isdigit);
+	auto positionM = find_if(_literal.begin(), _literal.end(), util::isDigit);
 	if (positionM != _literal.end())
 	{
 		string baseType(_literal.begin(), positionM);
-		auto positionX = find_if_not(positionM, _literal.end(), ::isdigit);
+		auto positionX = find_if_not(positionM, _literal.end(), util::isDigit);
 		int m = parseSize(positionM, positionX);
 		Token keyword = keywordByName(baseType);
 		if (keyword == Token::Bytes)
@@ -208,7 +211,7 @@ tuple<Token, unsigned int, unsigned int> fromIdentifierOrKeyword(string const& _
 				positionM < positionX &&
 				positionX < _literal.end() &&
 				*positionX == 'x' &&
-				all_of(positionX + 1, _literal.end(), ::isdigit)
+				all_of(positionX + 1, _literal.end(), util::isDigit)
 			) {
 				int n = parseSize(positionX + 1, _literal.end());
 				if (
diff --git a/libsolidity/ast/Types.cpp b/libsolidity/ast/Types.cpp
@@ -33,6 +33,7 @@
 #include <libsolutil/CommonIO.h>
 #include <libsolutil/FunctionSelector.h>
 #include <libsolutil/Keccak256.h>
+#include <libsolutil/StringUtils.h>
 #include <libsolutil/UTF8.h>
 
 #include <boost/algorithm/string.hpp>
@@ -781,8 +782,8 @@ tuple<bool, rational> RationalNumberType::parseRational(string const& _value)
 		if (radixPoint != _value.end())
 		{
 			if (
-				!all_of(radixPoint + 1, _value.end(), ::isdigit) ||
-				!all_of(_value.begin(), radixPoint, ::isdigit)
+				!all_of(radixPoint + 1, _value.end(), util::isDigit) ||
+				!all_of(_value.begin(), radixPoint, util::isDigit)
 			)
 				return make_tuple(false, rational(0));
 
diff --git a/libsolidity/codegen/ir/IRGeneratorForStatements.cpp b/libsolidity/codegen/ir/IRGeneratorForStatements.cpp
@@ -203,7 +203,7 @@ struct CopyTranslate: public yul::ASTCopier
 		else
 			solAssert(false);
 
-		if (isdigit(value.front()))
+		if (isDigit(value.front()))
 			return yul::Literal{_identifier.debugData, yul::LiteralKind::Number, yul::YulString{value}, {}};
 		else
 			return yul::Identifier{_identifier.debugData, yul::YulString{value}};
diff --git a/libsolidity/formal/CHC.cpp b/libsolidity/formal/CHC.cpp
@@ -33,6 +33,7 @@
 #include <libsmtutil/CHCSmtLib2Interface.h>
 #include <liblangutil/CharStreamProvider.h>
 #include <libsolutil/Algorithms.h>
+#include <libsolutil/StringUtils.h>
 
 #ifdef HAVE_Z3_DLOPEN
 #include <z3_version.h>
@@ -1998,9 +1999,9 @@ map<unsigned, vector<unsigned>> CHC::summaryCalls(CHCSolverInterface::CexGraph c
 			// Predicates that do not have a CALLID have a predicate id at the end of <suffix>,
 			// so the assertion below should still hold.
 			auto beg = _s.data();
-			while (beg != _s.data() + _s.size() && !isdigit(*beg)) ++beg;
+			while (beg != _s.data() + _s.size() && !isDigit(*beg)) ++beg;
 			auto end = beg;
-			while (end != _s.data() + _s.size() && isdigit(*end)) ++end;
+			while (end != _s.data() + _s.size() && isDigit(*end)) ++end;
 
 			solAssert(beg != end, "Expected to find numerical call or predicate id.");
 
diff --git a/libsolutil/CommonData.cpp b/libsolutil/CommonData.cpp
@@ -20,11 +20,12 @@
  * @date 2014
  */
 
+#include <libsolutil/Assertions.h>
 #include <libsolutil/CommonData.h>
 #include <libsolutil/Exceptions.h>
-#include <libsolutil/Assertions.h>
-#include <libsolutil/Keccak256.h>
 #include <libsolutil/FixedHash.h>
+#include <libsolutil/Keccak256.h>
+#include <libsolutil/StringUtils.h>
 
 #include <boost/algorithm/string.hpp>
 
@@ -154,9 +155,9 @@ string solidity::util::getChecksummedAddress(string const& _addr)
 		char addressCharacter = s[i];
 		uint8_t nibble = hash[i / 2u] >> (4u * (1u - (i % 2u))) & 0xf;
 		if (nibble >= 8)
-			ret += static_cast<char>(toupper(addressCharacter));
+			ret += toUpper(addressCharacter);
 		else
-			ret += static_cast<char>(tolower(addressCharacter));
+			ret += toLower(addressCharacter);
 	}
 	return ret;
 }
@@ -211,7 +212,7 @@ string solidity::util::escapeAndQuoteString(string const& _input)
 			out += "\\r";
 		else if (c == '\t')
 			out += "\\t";
-		else if (!isprint(c, locale::classic()))
+		else if (!isPrint(c))
 		{
 			ostringstream o;
 			o << "\\x" << std::hex << setfill('0') << setw(2) << (unsigned)(unsigned char)(c);
diff --git a/libsolutil/StringUtils.h b/libsolutil/StringUtils.h
@@ -27,7 +27,9 @@
 #include <libsolutil/CommonData.h>
 #include <libsolutil/Numeric.h>
 
+#include <algorithm>
 #include <limits>
+#include <locale>
 #include <string>
 #include <vector>
 
@@ -197,4 +199,47 @@ inline std::optional<unsigned> toUnsignedInt(std::string const& _value)
 	}
 }
 
+/// Converts parameter _c to its lowercase equivalent if c is an uppercase letter and has a lowercase equivalent. It uses the classic "C" locale semantics.
+/// @param _c value to be converted
+/// @return the converted value
+inline char toLower(char _c)
+{
+	return tolower(_c, std::locale::classic());
+}
+
+/// Converts parameter _c to its uppercase equivalent if c is an lowercase letter and has a uppercase equivalent. It uses the classic "C" locale semantics.
+/// @param _c value to be converted
+/// @return the converted value
+inline char toUpper(char _c)
+{
+	return toupper(_c, std::locale::classic());
+}
+
+/// Converts parameter _s to its lowercase equivalent. It uses the classic "C" locale semantics.
+/// @param _s value to be converted
+/// @return the converted value
+inline std::string toLower(std::string _s)
+{
+	std::transform(_s.begin(), _s.end(), _s.begin(), [](char _c) {
+		return toLower(_c);
+	});
+	return _s;
+}
+
+/// Checks whether _c is a decimal digit character. It uses the classic "C" locale semantics.
+/// @param _c character to be checked
+/// @return true if _c is a decimal digit character, false otherwise
+inline bool isDigit(char _c)
+{
+	return isdigit(_c, std::locale::classic());
+}
+
+// Checks if character is printable using classic "C" locale
+/// @param _c character to be checked
+/// @return true if _c is a printable character, false otherwise.
+inline bool isPrint(char _c)
+{
+	return isprint(_c, std::locale::classic());
+}
+
 }
diff --git a/libyul/backends/evm/EVMDialect.cpp b/libyul/backends/evm/EVMDialect.cpp
@@ -21,17 +21,17 @@
 
 #include <libyul/backends/evm/EVMDialect.h>
 
-#include <libyul/AsmAnalysisInfo.h>
+#include <libevmasm/Instruction.h>
+#include <libevmasm/SemanticInformation.h>
+#include <liblangutil/Exceptions.h>
+#include <libsolutil/StringUtils.h>
 #include <libyul/AST.h>
-#include <libyul/Object.h>
-#include <libyul/Exceptions.h>
+#include <libyul/AsmAnalysisInfo.h>
 #include <libyul/AsmParser.h>
+#include <libyul/Exceptions.h>
+#include <libyul/Object.h>
 #include <libyul/Utilities.h>
 #include <libyul/backends/evm/AbstractAssembly.h>
-#include <libevmasm/SemanticInformation.h>
-#include <libevmasm/Instruction.h>
-
-#include <liblangutil/Exceptions.h>
 
 #include <range/v3/view/reverse.hpp>
 #include <range/v3/view/tail.hpp>
@@ -121,8 +121,7 @@ set<YulString> createReservedIdentifiers(langutil::EVMVersion _evmVersion)
 	set<YulString> reserved;
 	for (auto const& instr: evmasm::c_instructions)
 	{
-		string name = instr.first;
-		transform(name.begin(), name.end(), name.begin(), [](unsigned char _c) { return tolower(_c); });
+		string name = toLower(instr.first);
 		if (!baseFeeException(instr.second))
 			reserved.emplace(name);
 	}
@@ -142,8 +141,7 @@ map<YulString, BuiltinFunctionForEVM> createBuiltins(langutil::EVMVersion _evmVe
 	map<YulString, BuiltinFunctionForEVM> builtins;
 	for (auto const& instr: evmasm::c_instructions)
 	{
-		string name = instr.first;
-		transform(name.begin(), name.end(), name.begin(), [](unsigned char _c) { return tolower(_c); });
+		string name = toLower(instr.first);
 		auto const opcode = instr.second;
 
 		if (
diff --git a/libyul/optimiser/SimplificationRules.cpp b/libyul/optimiser/SimplificationRules.cpp
@@ -21,15 +21,16 @@
 
 #include <libyul/optimiser/SimplificationRules.h>
 
+#include <libyul/AST.h>
+#include <libyul/Utilities.h>
+#include <libyul/backends/evm/EVMDialect.h>
 #include <libyul/optimiser/ASTCopier.h>
+#include <libyul/optimiser/DataFlowAnalyzer.h>
 #include <libyul/optimiser/Semantics.h>
 #include <libyul/optimiser/SyntacticalEquality.h>
-#include <libyul/optimiser/DataFlowAnalyzer.h>
-#include <libyul/backends/evm/EVMDialect.h>
-#include <libyul/AST.h>
-#include <libyul/Utilities.h>
 
 #include <libevmasm/RuleList.h>
+#include <libsolutil/StringUtils.h>
 
 using namespace std;
 using namespace solidity;
@@ -249,8 +250,7 @@ Expression Pattern::toExpression(shared_ptr<DebugData const> const& _debugData)
 		for (auto const& arg: m_arguments)
 			arguments.emplace_back(arg.toExpression(_debugData));
 
-		string name = instructionInfo(m_instruction).name;
-		transform(begin(name), end(name), begin(name), [](auto _c) { return tolower(_c); });
+		string name = util::toLower(instructionInfo(m_instruction).name);
 
 		return FunctionCall{_debugData,
 			Identifier{_debugData, YulString{name}},
diff --git a/test/CommonSyntaxTest.cpp b/test/CommonSyntaxTest.cpp
@@ -29,7 +29,6 @@
 
 using namespace std;
 using namespace solidity;
-using namespace solidity::util;
 using namespace solidity::util::formatting;
 using namespace solidity::langutil;
 using namespace solidity::frontend;
@@ -43,10 +42,11 @@ namespace
 
 int parseUnsignedInteger(string::iterator& _it, string::iterator _end)
 {
-	if (_it == _end || !isdigit(*_it))
+	auto isDigit = [](char _c) -> bool {return isdigit(_c, std::locale::classic());};
+	if (_it == _end || !isDigit(*_it))
 		BOOST_THROW_EXCEPTION(runtime_error("Invalid test expectation. Source location expected."));
 	int result = 0;
-	while (_it != _end && isdigit(*_it))
+	while (_it != _end && isDigit(*_it))
 	{
 		result *= 10;
 		result += *_it - '0';
@@ -195,6 +195,7 @@ string CommonSyntaxTest::errorMessage(Exception const& _e)
 
 vector<SyntaxTestError> CommonSyntaxTest::parseExpectations(istream& _stream)
 {
+	auto isDigit = [](char _c) -> bool {return isdigit(_c, std::locale::classic());};
 	vector<SyntaxTestError> expectations;
 	string line;
 	while (getline(_stream, line))
@@ -207,14 +208,14 @@ vector<SyntaxTestError> CommonSyntaxTest::parseExpectations(istream& _stream)
 		if (it == line.end()) continue;
 
 		auto typeBegin = it;
-		while (it != line.end() && isalpha(*it))
+		while (it != line.end() && isalpha(*it, locale::classic()))
 			++it;
 		string errorType(typeBegin, it);
 
 		skipWhitespace(it, line.end());
 
 		optional<ErrorId> errorId;
-		if (it != line.end() && isdigit(*it))
+		if (it != line.end() && isDigit(*it))
 			errorId = ErrorId{static_cast<unsigned long long>(parseUnsignedInteger(it, line.end()))};
 
 		expect(it, line.end(), ':');
@@ -227,7 +228,7 @@ vector<SyntaxTestError> CommonSyntaxTest::parseExpectations(istream& _stream)
 		if (it != line.end() && *it == '(')
 		{
 			++it;
-			if (it != line.end() && !isdigit(*it))
+			if (it != line.end() && !isDigit(*it))
 			{
 				auto sourceNameStart = it;
 				while (it != line.end() && *it != ':')
diff --git a/test/TestCase.cpp b/test/TestCase.cpp
@@ -118,7 +118,7 @@ EVMVersionRestrictedTestCase::EVMVersionRestrictedTestCase(string const& _filena
 	string comparator;
 	size_t versionBegin = 0;
 	for (auto character: versionString)
-		if (!isalpha(character))
+		if (!isalpha(character, locale::classic()))
 		{
 			comparator += character;
 			versionBegin++;
diff --git a/test/TestCase.h b/test/TestCase.h
@@ -90,7 +90,7 @@ class TestCase
 	template<typename IteratorType>
 	static void skipWhitespace(IteratorType& _it, IteratorType _end)
 	{
-		while (_it != _end && isspace(*_it))
+		while (_it != _end && std::isspace<char>(*_it, std::locale::classic()))
 			++_it;
 	}
 
diff --git a/test/libsolidity/util/BytesUtils.cpp b/test/libsolidity/util/BytesUtils.cpp
@@ -22,6 +22,7 @@
 
 #include <libsolutil/CommonData.h>
 #include <libsolutil/CommonIO.h>
+#include <libsolutil/StringUtils.h>
 
 #include <boost/algorithm/string.hpp>
 
@@ -216,7 +217,7 @@ string BytesUtils::formatString(bytes const& _bytes, size_t _cutOff)
 				os << "\\n";
 				break;
 			default:
-				if (isprint(v))
+				if (isPrint(static_cast<char>(v)))
 					os << v;
 				else
 					os << "\\x" << toHex(v, HexCase::Lower);
diff --git a/test/libsolidity/util/TestFileParser.cpp b/test/libsolidity/util/TestFileParser.cpp
@@ -762,13 +762,16 @@ string TestFileParser::Scanner::scanString()
 // TODO: use fromHex() from CommonData
 char TestFileParser::Scanner::scanHexPart()
 {
+	auto toLower = [](char _c) -> char { return tolower(_c, locale::classic()); };
+	auto isDigit = [](char _c) -> bool { return isdigit(_c, locale::classic()); };
+
 	advance(); // skip 'x'
 
 	int value{};
-	if (isdigit(current()))
+	if (isDigit(current()))
 		value = current() - '0';
-	else if (tolower(current()) >= 'a' && tolower(current()) <= 'f')
-		value = tolower(current()) - 'a' + 10;
+	else if (toLower(current()) >= 'a' && toLower(current()) <= 'f')
+		value = toLower(current()) - 'a' + 10;
 	else
 		BOOST_THROW_EXCEPTION(TestParserError("\\x used with no following hex digits."));
 
@@ -777,10 +780,10 @@ char TestFileParser::Scanner::scanHexPart()
 		return static_cast<char>(value);
 
 	value <<= 4;
-	if (isdigit(current()))
+	if (isDigit(current()))
 		value |= current() - '0';
-	else if (tolower(current()) >= 'a' && tolower(current()) <= 'f')
-		value |= tolower(current()) - 'a' + 10;
+	else if (toLower(current()) >= 'a' && toLower(current()) <= 'f')
+		value |= toLower(current()) - 'a' + 10;
 
 	advance();
 
diff --git a/test/tools/ossfuzz/strictasm_diff_ossfuzz.cpp b/test/tools/ossfuzz/strictasm_diff_ossfuzz.cpp
diff --git a/test/tools/yulopti.cpp b/test/tools/yulopti.cpp

Original file line number	Diff line number	Diff line change
`@@ -200,9 +200,7 @@ string AssemblyItem::toAssemblyText(Assembly const& _assembly) const`
`200`	`200`	`case Operation:`
`201`	`201`	`{`
`202`	`202`	`assertThrow(isValidInstruction(instruction()), AssemblyException, "Invalid instruction.");`
`203`		`- string name = instructionInfo(instruction()).name;`
`204`		`- transform(name.begin(), name.end(), name.begin(), [](unsigned char _c) { return tolower(_c); });`
`205`		`- text = name;`
	`203`	`+ text = util::toLower(instructionInfo(instruction()).name);`
`206`	`204`	`break;`
`207`	`205`	`}`
`208`	`206`	`case Push:`