Merge pull request #74995 from dalexeev/gds-r-strings

GDScript: Add raw string literals (r-strings)

modules/gdscript/editor/gdscript_highlighter.cpp

@@ -52,6 +52,7 @@ Dictionary GDScriptSyntaxHighlighter::_get_line_syntax_highlighting_impl(int p_l
 	bool in_keyword = false;
 	bool in_word = false;
 	bool in_number = false;
+	bool in_raw_string = false;
 	bool in_node_path = false;
 	bool in_node_ref = false;
 	bool in_annotation = false;
@@ -234,15 +235,33 @@ Dictionary GDScriptSyntaxHighlighter::_get_line_syntax_highlighting_impl(int p_l
 							}
 
 							if (str[from] == '\\') {
+								if (!in_raw_string) {
 									Dictionary escape_char_highlighter_info;
 									escape_char_highlighter_info["color"] = symbol_color;
 									color_map[from] = escape_char_highlighter_info;
+								}
 
 								from++;
 
+								if (!in_raw_string) {
+									int esc_len = 0;
+									if (str[from] == 'u') {
+										esc_len = 4;
+									} else if (str[from] == 'U') {
+										esc_len = 6;
+									}
+									for (int k = 0; k < esc_len && from < line_length - 1; k++) {
+										if (!is_hex_digit(str[from + 1])) {
+											break;
+										}
+										from++;
+									}
+
 									Dictionary region_continue_highlighter_info;
 									region_continue_highlighter_info["color"] = region_color;
 									color_map[from + 1] = region_continue_highlighter_info;
+								}
+
 								continue;
 							}
 
@@ -489,6 +508,12 @@ Dictionary GDScriptSyntaxHighlighter::_get_line_syntax_highlighting_impl(int p_l
 			in_member_variable = false;
 		}
 
+		if (!in_raw_string && in_region == -1 && str[j] == 'r' && j < line_length - 1 && (str[j + 1] == '"' || str[j + 1] == '\'')) {
+			in_raw_string = true;
+		} else if (in_raw_string && in_region == -1) {
+			in_raw_string = false;
+		}
+
 		// Keep symbol color for binary '&&'. In the case of '&&&' use StringName color for the last ampersand.
 		if (!in_string_name && in_region == -1 && str[j] == '&' && !is_binary_op) {
 			if (j >= 2 && str[j - 1] == '&' && str[j - 2] != '&' && prev_is_binary_op) {
@@ -520,7 +545,9 @@ Dictionary GDScriptSyntaxHighlighter::_get_line_syntax_highlighting_impl(int p_l
 			in_annotation = false;
 		}
 
-		if (in_node_ref) {
+		if (in_raw_string) {
+			color = string_color;
+		} else if (in_node_ref) {
 			next_type = NODE_REF;
 			color = node_ref_color;
 		} else if (in_annotation) {
@@ -692,7 +719,7 @@ void GDScriptSyntaxHighlighter::_update_cache() {
 	}
 
 	/* Strings */
-	const Color string_color = EDITOR_GET("text_editor/theme/highlighting/string_color");
+	string_color = EDITOR_GET("text_editor/theme/highlighting/string_color");
 	List<String> strings;
 	gdscript->get_string_delimiters(&strings);
 	for (const String &string : strings) {

modules/gdscript/editor/gdscript_highlighter.h

@@ -78,6 +78,7 @@ private:
 	Color built_in_type_color;
 	Color number_color;
 	Color member_color;
+	Color string_color;
 	Color node_path_color;
 	Color node_ref_color;
 	Color annotation_color;

modules/gdscript/gdscript_editor.cpp

@@ -59,6 +59,7 @@ void GDScriptLanguage::get_string_delimiters(List<String> *p_delimiters) const {
 	p_delimiters->push_back("' '");
 	p_delimiters->push_back("\"\"\" \"\"\"");
 	p_delimiters->push_back("''' '''");
+	// NOTE: StringName, NodePath and r-strings are not listed here.
 }
 
 bool GDScriptLanguage::is_using_templates() {

modules/gdscript/gdscript_tokenizer.cpp

@@ -857,10 +857,14 @@ GDScriptTokenizer::Token GDScriptTokenizer::string() {
 		STRING_NODEPATH,
 	};
 
+	bool is_raw = false;
 	bool is_multiline = false;
 	StringType type = STRING_REGULAR;
 
-	if (_peek(-1) == '&') {
+	if (_peek(-1) == 'r') {
+		is_raw = true;
+		_advance();
+	} else if (_peek(-1) == '&') {
 		type = STRING_NAME;
 		_advance();
 	} else if (_peek(-1) == '^') {
@@ -890,7 +894,12 @@ GDScriptTokenizer::Token GDScriptTokenizer::string() {
 		char32_t ch = _peek();
 
 		if (ch == 0x200E || ch == 0x200F || (ch >= 0x202A && ch <= 0x202E) || (ch >= 0x2066 && ch <= 0x2069)) {
-			Token error = make_error("Invisible text direction control character present in the string, escape it (\"\\u" + String::num_int64(ch, 16) + "\") to avoid confusion.");
+			Token error;
+			if (is_raw) {
+				error = make_error("Invisible text direction control character present in the string, use regular string literal instead of r-string.");
+			} else {
+				error = make_error("Invisible text direction control character present in the string, escape it (\"\\u" + String::num_int64(ch, 16) + "\") to avoid confusion.");
+			}
 			error.start_column = column;
 			error.leftmost_column = error.start_column;
 			error.end_column = column + 1;
@@ -905,6 +914,25 @@ GDScriptTokenizer::Token GDScriptTokenizer::string() {
 				return make_error("Unterminated string.");
 			}
 
+			if (is_raw) {
+				if (_peek() == quote_char) {
+					_advance();
+					if (_is_at_end()) {
+						return make_error("Unterminated string.");
+					}
+					result += '\\';
+					result += quote_char;
+				} else if (_peek() == '\\') { // For `\\\"`.
+					_advance();
+					if (_is_at_end()) {
+						return make_error("Unterminated string.");
+					}
+					result += '\\';
+					result += '\\';
+				} else {
+					result += '\\';
+				}
+			} else {
 				// Grab escape character.
 				char32_t code = _peek();
 				_advance();
@@ -1013,7 +1041,7 @@ GDScriptTokenizer::Token GDScriptTokenizer::string() {
 							prev_pos = column - 2;
 							continue;
 						} else {
-						Token error = make_error("Invalid UTF-16 sequence in string, unpaired lead surrogate");
+							Token error = make_error("Invalid UTF-16 sequence in string, unpaired lead surrogate.");
 							error.start_column = column - 2;
 							error.leftmost_column = error.start_column;
 							push_error(error);
@@ -1022,7 +1050,7 @@ GDScriptTokenizer::Token GDScriptTokenizer::string() {
 						}
 					} else if ((escaped & 0xfffffc00) == 0xdc00) {
 						if (prev == 0) {
-						Token error = make_error("Invalid UTF-16 sequence in string, unpaired trail surrogate");
+							Token error = make_error("Invalid UTF-16 sequence in string, unpaired trail surrogate.");
 							error.start_column = column - 2;
 							error.leftmost_column = error.start_column;
 							push_error(error);
@@ -1033,7 +1061,7 @@ GDScriptTokenizer::Token GDScriptTokenizer::string() {
 						}
 					}
 					if (prev != 0) {
-					Token error = make_error("Invalid UTF-16 sequence in string, unpaired lead surrogate");
+						Token error = make_error("Invalid UTF-16 sequence in string, unpaired lead surrogate.");
 						error.start_column = prev_pos;
 						error.leftmost_column = error.start_column;
 						push_error(error);
@@ -1044,6 +1072,7 @@ GDScriptTokenizer::Token GDScriptTokenizer::string() {
 				if (valid_escape) {
 					result += escaped;
 				}
+			}
 		} else if (ch == quote_char) {
 			if (prev != 0) {
 				Token error = make_error("Invalid UTF-16 sequence in string, unpaired lead surrogate");
@@ -1416,6 +1445,9 @@ GDScriptTokenizer::Token GDScriptTokenizer::scan() {
 
 	if (is_digit(c)) {
 		return number();
+	} else if (c == 'r' && (_peek() == '"' || _peek() == '\'')) {
+		// Raw string literals.
+		return string();
 	} else if (is_unicode_identifier_start(c)) {
 		return potential_identifier();
 	}

modules/gdscript/tests/scripts/parser/errors/bad_r_string_1.gd

@@ -0,0 +1,2 @@
+func test():
+	print(r"\")

modules/gdscript/tests/scripts/parser/errors/bad_r_string_1.out

@@ -0,0 +1,2 @@
+GDTEST_PARSER_ERROR
+Unterminated string.

modules/gdscript/tests/scripts/parser/errors/bad_r_string_2.gd

@@ -0,0 +1,2 @@
+func test():
+	print(r"\\"")

modules/gdscript/tests/scripts/parser/errors/bad_r_string_2.out

@@ -0,0 +1,2 @@
+GDTEST_PARSER_ERROR
+Unterminated string.

modules/gdscript/tests/scripts/parser/errors/bad_r_string_3.gd

@@ -0,0 +1,3 @@
+func test():
+	#         v
+	print(r"['"]*")

modules/gdscript/tests/scripts/parser/errors/bad_r_string_3.out

@@ -0,0 +1,2 @@
+GDTEST_PARSER_ERROR
+Closing "]" doesn't have an opening counterpart.

modules/gdscript/tests/scripts/parser/features/r_strings.gd

@@ -0,0 +1,22 @@
+func test():
+	print(r"test ' \' \" \\ \n \t \u2023 test")
+	print(r"\n\\[\t ]*(\w+)")
+	print(r"")
+	print(r"\"")
+	print(r"\\\"")
+	print(r"\\")
+	print(r"\" \\\" \\\\\"")
+	print(r"\ \\ \\\ \\\\ \\\\\ \\")
+	print(r'"')
+	print(r'"(?:\\.|[^"])*"')
+	print(r"""""")
+	print(r"""test \t "test"="" " \" \\\" \ \\ \\\ test""")
+	print(r'''r"""test \t "test"="" " \" \\\" \ \\ \\\ test"""''')
+	print(r"\t
+			\t")
+	print(r"\t \
+			\t")
+	print(r"""\t
+			\t""")
+	print(r"""\t \
+			\t""")

modules/gdscript/tests/scripts/parser/features/r_strings.out

@@ -0,0 +1,22 @@
+GDTEST_OK
+test ' \' \" \\ \n \t \u2023 test
+\n\\[\t ]*(\w+)
+
+\"
+\\\"
+\\
+\" \\\" \\\\\"
+\ \\ \\\ \\\\ \\\\\ \\
+"
+"(?:\\.|[^"])*"
+
+test \t "test"="" " \" \\\" \ \\ \\\ test
+r"""test \t "test"="" " \" \\\" \ \\ \\\ test"""
+\t
+			\t
+\t \
+			\t
+\t
+			\t
+\t \
+			\t

modules/regex/doc_classes/RegEx.xml

@@ -10,7 +10,7 @@
 		var regex = RegEx.new()
 		regex.compile("\\w-(\\d+)")
 		[/codeblock]
-		The search pattern must be escaped first for GDScript before it is escaped for the expression. For example, [code]compile("\\d+")[/code] would be read by RegEx as [code]\d+[/code]. Similarly, [code]compile("\"(?:\\\\.|[^\"])*\"")[/code] would be read as [code]"(?:\\.|[^"])*"[/code].
+		The search pattern must be escaped first for GDScript before it is escaped for the expression. For example, [code]compile("\\d+")[/code] would be read by RegEx as [code]\d+[/code]. Similarly, [code]compile("\"(?:\\\\.|[^\"])*\"")[/code] would be read as [code]"(?:\\.|[^"])*"[/code]. In GDScript, you can also use raw string literals (r-strings). For example, [code]compile(r'"(?:\\.|[^"])*"')[/code] would be read the same.
 		Using [method search], you can find the pattern within the given text. If a pattern is found, [RegExMatch] is returned and you can retrieve details of the results using methods such as [method RegExMatch.get_string] and [method RegExMatch.get_start].
 		[codeblock]
 		var regex = RegEx.new()