Merge pull request #51395 from angad-k/pseudolocalization-squashed

Add pseudolocalization to Godot. (GSoC'21)

core/string/translation.cpp

@@ -929,6 +929,66 @@ void Translation::_bind_methods() {
 
 ///////////////////////////////////////////////
 
+struct _character_accent_pair {
+	const char32_t character;
+	const char32_t *accented_character;
+};
+
+static _character_accent_pair _character_to_accented[] = {
+	{ 'A', U"Å" },
+	{ 'B', U"ß" },
+	{ 'C', U"Ç" },
+	{ 'D', U"Ð" },
+	{ 'E', U"É" },
+	{ 'F', U"F́" },
+	{ 'G', U"Ĝ" },
+	{ 'H', U"Ĥ" },
+	{ 'I', U"Ĩ" },
+	{ 'J', U"Ĵ" },
+	{ 'K', U"ĸ" },
+	{ 'L', U"Ł" },
+	{ 'M', U"Ḿ" },
+	{ 'N', U"й" },
+	{ 'O', U"Ö" },
+	{ 'P', U"Ṕ" },
+	{ 'Q', U"Q́" },
+	{ 'R', U"Ř" },
+	{ 'S', U"Ŝ" },
+	{ 'T', U"Ŧ" },
+	{ 'U', U"Ũ" },
+	{ 'V', U"Ṽ" },
+	{ 'W', U"Ŵ" },
+	{ 'X', U"X́" },
+	{ 'Y', U"Ÿ" },
+	{ 'Z', U"Ž" },
+	{ 'a', U"á" },
+	{ 'b', U"ḅ" },
+	{ 'c', U"ć" },
+	{ 'd', U"d́" },
+	{ 'e', U"é" },
+	{ 'f', U"f́" },
+	{ 'g', U"ǵ" },
+	{ 'h', U"h̀" },
+	{ 'i', U"í" },
+	{ 'j', U"ǰ" },
+	{ 'k', U"ḱ" },
+	{ 'l', U"ł" },
+	{ 'm', U"m̀" },
+	{ 'n', U"ή" },
+	{ 'o', U"ô" },
+	{ 'p', U"ṕ" },
+	{ 'q', U"q́" },
+	{ 'r', U"ŕ" },
+	{ 's', U"š" },
+	{ 't', U"ŧ" },
+	{ 'u', U"ü" },
+	{ 'v', U"ṽ" },
+	{ 'w', U"ŵ" },
+	{ 'x', U"x́" },
+	{ 'y', U"ý" },
+	{ 'z', U"ź" },
+};
+
 bool TranslationServer::is_locale_valid(const String &p_locale) {
 	const char **ptr = locale_list;
 
@@ -1101,10 +1161,10 @@ StringName TranslationServer::translate(const StringName &p_message, const Strin
 	}
 
 	if (!res) {
-		return p_message;
+		return pseudolocalization_enabled ? pseudolocalize(p_message) : p_message;
 	}
 
-	return res;
+	return pseudolocalization_enabled ? pseudolocalize(res) : res;
 }
 
 StringName TranslationServer::translate_plural(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context) const {
@@ -1217,7 +1277,18 @@ void TranslationServer::setup() {
 	} else {
 		set_locale(OS::get_singleton()->get_locale());
 	}
+
 	fallback = GLOBAL_DEF("internationalization/locale/fallback", "en");
+	pseudolocalization_enabled = GLOBAL_DEF("internationalization/pseudolocalization/use_pseudolocalization", false);
+	pseudolocalization_accents_enabled = GLOBAL_DEF("internationalization/pseudolocalization/replace_with_accents", true);
+	pseudolocalization_double_vowels_enabled = GLOBAL_DEF("internationalization/pseudolocalization/double_vowels", false);
+	pseudolocalization_fake_bidi_enabled = GLOBAL_DEF("internationalization/pseudolocalization/fake_bidi", false);
+	pseudolocalization_override_enabled = GLOBAL_DEF("internationalization/pseudolocalization/override", false);
+	expansion_ratio = GLOBAL_DEF("internationalization/pseudolocalization/expansion_ratio", 0.0);
+	pseudolocalization_prefix = GLOBAL_DEF("internationalization/pseudolocalization/prefix", "[");
+	pseudolocalization_suffix = GLOBAL_DEF("internationalization/pseudolocalization/suffix", "]");
+	pseudolocalization_skip_placeholders_enabled = GLOBAL_DEF("internationalization/pseudolocalization/skip_placeholders", true);
+
 #ifdef TOOLS_ENABLED
 	{
 		String options = "";
@@ -1258,10 +1329,10 @@ StringName TranslationServer::tool_translate(const StringName &p_message, const
 	if (tool_translation.is_valid()) {
 		StringName r = tool_translation->get_message(p_message, p_context);
 		if (r) {
-			return r;
+			return editor_pseudolocalization ? tool_pseudolocalize(r) : r;
 		}
 	}
-	return p_message;
+	return editor_pseudolocalization ? tool_pseudolocalize(p_message) : p_message;
 }
 
 StringName TranslationServer::tool_translate_plural(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context) const {
@@ -1306,6 +1377,181 @@ StringName TranslationServer::doc_translate_plural(const StringName &p_message,
 	return p_message_plural;
 }
 
+bool TranslationServer::is_pseudolocalization_enabled() const {
+	return pseudolocalization_enabled;
+}
+
+void TranslationServer::set_pseudolocalization_enabled(bool p_enabled) {
+	pseudolocalization_enabled = p_enabled;
+
+	if (OS::get_singleton()->get_main_loop()) {
+		OS::get_singleton()->get_main_loop()->notification(MainLoop::NOTIFICATION_TRANSLATION_CHANGED);
+	}
+	ResourceLoader::reload_translation_remaps();
+}
+
+void TranslationServer::set_editor_pseudolocalization(bool p_enabled) {
+	editor_pseudolocalization = p_enabled;
+}
+
+void TranslationServer::reload_pseudolocalization() {
+	pseudolocalization_accents_enabled = GLOBAL_GET("internationalization/pseudolocalization/replace_with_accents");
+	pseudolocalization_double_vowels_enabled = GLOBAL_GET("internationalization/pseudolocalization/double_vowels");
+	pseudolocalization_fake_bidi_enabled = GLOBAL_GET("internationalization/pseudolocalization/fake_bidi");
+	pseudolocalization_override_enabled = GLOBAL_GET("internationalization/pseudolocalization/override");
+	expansion_ratio = GLOBAL_GET("internationalization/pseudolocalization/expansion_ratio");
+	pseudolocalization_prefix = GLOBAL_GET("internationalization/pseudolocalization/prefix");
+	pseudolocalization_suffix = GLOBAL_GET("internationalization/pseudolocalization/suffix");
+	pseudolocalization_skip_placeholders_enabled = GLOBAL_GET("internationalization/pseudolocalization/skip_placeholders");
+
+	if (OS::get_singleton()->get_main_loop()) {
+		OS::get_singleton()->get_main_loop()->notification(MainLoop::NOTIFICATION_TRANSLATION_CHANGED);
+	}
+	ResourceLoader::reload_translation_remaps();
+}
+
+StringName TranslationServer::pseudolocalize(const StringName &p_message) const {
+	String message = p_message;
+	int length = message.length();
+	if (pseudolocalization_override_enabled) {
+		message = get_override_string(message);
+	}
+
+	if (pseudolocalization_double_vowels_enabled) {
+		message = double_vowels(message);
+	}
+
+	if (pseudolocalization_accents_enabled) {
+		message = replace_with_accented_string(message);
+	}
+
+	if (pseudolocalization_fake_bidi_enabled) {
+		message = wrap_with_fakebidi_characters(message);
+	}
+
+	StringName res = add_padding(message, length);
+	return res;
+}
+
+StringName TranslationServer::tool_pseudolocalize(const StringName &p_message) const {
+	String message = p_message;
+	message = double_vowels(message);
+	message = replace_with_accented_string(message);
+	StringName res = "[!!! " + message + " !!!]";
+	return res;
+}
+
+String TranslationServer::get_override_string(String &p_message) const {
+	String res;
+	for (int i = 0; i < p_message.size(); i++) {
+		if (pseudolocalization_skip_placeholders_enabled && is_placeholder(p_message, i)) {
+			res += p_message[i];
+			res += p_message[i + 1];
+			i++;
+			continue;
+		}
+		res += '*';
+	}
+	return res;
+}
+
+String TranslationServer::double_vowels(String &p_message) const {
+	String res;
+	for (int i = 0; i < p_message.size(); i++) {
+		if (pseudolocalization_skip_placeholders_enabled && is_placeholder(p_message, i)) {
+			res += p_message[i];
+			res += p_message[i + 1];
+			i++;
+			continue;
+		}
+		res += p_message[i];
+		if (p_message[i] == 'a' || p_message[i] == 'e' || p_message[i] == 'i' || p_message[i] == 'o' || p_message[i] == 'u' ||
+				p_message[i] == 'A' || p_message[i] == 'E' || p_message[i] == 'I' || p_message[i] == 'O' || p_message[i] == 'U') {
+			res += p_message[i];
+		}
+	}
+	return res;
+};
+
+String TranslationServer::replace_with_accented_string(String &p_message) const {
+	String res;
+	for (int i = 0; i < p_message.size(); i++) {
+		if (pseudolocalization_skip_placeholders_enabled && is_placeholder(p_message, i)) {
+			res += p_message[i];
+			res += p_message[i + 1];
+			i++;
+			continue;
+		}
+		const char32_t *accented = get_accented_version(p_message[i]);
+		if (accented) {
+			res += accented;
+		} else {
+			res += p_message[i];
+		}
+	}
+	return res;
+}
+
+String TranslationServer::wrap_with_fakebidi_characters(String &p_message) const {
+	String res;
+	char32_t fakebidiprefix = U'\u202e';
+	char32_t fakebidisuffix = U'\u202c';
+	res += fakebidiprefix;
+	// The fake bidi unicode gets popped at every newline so pushing it back at every newline.
+	for (int i = 0; i < p_message.size(); i++) {
+		if (p_message[i] == '\n') {
+			res += fakebidisuffix;
+			res += p_message[i];
+			res += fakebidiprefix;
+		} else if (pseudolocalization_skip_placeholders_enabled && is_placeholder(p_message, i)) {
+			res += fakebidisuffix;
+			res += p_message[i];
+			res += p_message[i + 1];
+			res += fakebidiprefix;
+			i++;
+		} else {
+			res += p_message[i];
+		}
+	}
+	res += fakebidisuffix;
+	return res;
+}
+
+String TranslationServer::add_padding(String &p_message, int p_length) const {
+	String res;
+	String prefix = pseudolocalization_prefix;
+	String suffix;
+	for (int i = 0; i < p_length * expansion_ratio / 2; i++) {
+		prefix += "_";
+		suffix += "_";
+	}
+	suffix += pseudolocalization_suffix;
+	res += prefix;
+	res += p_message;
+	res += suffix;
+	return res;
+}
+
+const char32_t *TranslationServer::get_accented_version(char32_t p_character) const {
+	if (!((p_character >= 'a' && p_character <= 'z') || (p_character >= 'A' && p_character <= 'Z'))) {
+		return nullptr;
+	}
+
+	for (unsigned int i = 0; i < sizeof(_character_to_accented) / sizeof(_character_to_accented[0]); i++) {
+		if (_character_to_accented[i].character == p_character) {
+			return _character_to_accented[i].accented_character;
+		}
+	}
+
+	return nullptr;
+}
+
+bool TranslationServer::is_placeholder(String &p_message, int p_index) const {
+	return p_message[p_index] == '%' && p_index < p_message.size() - 1 &&
+		   (p_message[p_index + 1] == 's' || p_message[p_index + 1] == 'c' || p_message[p_index + 1] == 'd' ||
+				   p_message[p_index + 1] == 'o' || p_message[p_index + 1] == 'x' || p_message[p_index + 1] == 'X' || p_message[p_index + 1] == 'f');
+}
+
 void TranslationServer::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("set_locale", "locale"), &TranslationServer::set_locale);
 	ClassDB::bind_method(D_METHOD("get_locale"), &TranslationServer::get_locale);
@@ -1322,6 +1568,12 @@ void TranslationServer::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("clear"), &TranslationServer::clear);
 
 	ClassDB::bind_method(D_METHOD("get_loaded_locales"), &TranslationServer::get_loaded_locales);
+
+	ClassDB::bind_method(D_METHOD("is_pseudolocalization_enabled"), &TranslationServer::is_pseudolocalization_enabled);
+	ClassDB::bind_method(D_METHOD("set_pseudolocalization_enabled", "enabled"), &TranslationServer::set_pseudolocalization_enabled);
+	ClassDB::bind_method(D_METHOD("reload_pseudolocalization"), &TranslationServer::reload_pseudolocalization);
+	ClassDB::bind_method(D_METHOD("pseudolocalize", "message"), &TranslationServer::pseudolocalize);
+	ADD_PROPERTY(PropertyInfo(Variant::Type::BOOL, "pseudolocalization_enabled"), "set_pseudolocalization_enabled", "is_pseudolocalization_enabled");
 }
 
 void TranslationServer::load_translations() {

core/string/translation.h

@@ -77,6 +77,26 @@ class TranslationServer : public Object {
 
 	bool enabled = true;
 
+	bool pseudolocalization_enabled = false;
+	bool pseudolocalization_accents_enabled = false;
+	bool pseudolocalization_double_vowels_enabled = false;
+	bool pseudolocalization_fake_bidi_enabled = false;
+	bool pseudolocalization_override_enabled = false;
+	bool pseudolocalization_skip_placeholders_enabled = false;
+	bool editor_pseudolocalization = false;
+	float expansion_ratio = 0.0;
+	String pseudolocalization_prefix;
+	String pseudolocalization_suffix;
+
+	StringName tool_pseudolocalize(const StringName &p_message) const;
+	String get_override_string(String &p_message) const;
+	String double_vowels(String &p_message) const;
+	String replace_with_accented_string(String &p_message) const;
+	String wrap_with_fakebidi_characters(String &p_message) const;
+	String add_padding(String &p_message, int p_length) const;
+	const char32_t *get_accented_version(char32_t p_character) const;
+	bool is_placeholder(String &p_message, int p_index) const;
+
 	static TranslationServer *singleton;
 	bool _load_translations(const String &p_from);
 
@@ -104,6 +124,13 @@ public:
 	StringName translate(const StringName &p_message, const StringName &p_context = "") const;
 	StringName translate_plural(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context = "") const;
 
+	StringName pseudolocalize(const StringName &p_message) const;
+
+	bool is_pseudolocalization_enabled() const;
+	void set_pseudolocalization_enabled(bool p_enabled);
+	void set_editor_pseudolocalization(bool p_enabled);
+	void reload_pseudolocalization();
+
 	static Vector<String> get_all_locales();
 	static Vector<String> get_all_locale_names();
 	static bool is_locale_valid(const String &p_locale);

doc/classes/ProjectSettings.xml

@@ -741,6 +741,34 @@
 		<member name="internationalization/locale/test" type="String" setter="" getter="" default="&quot;&quot;">
 			If non-empty, this locale will be used when running the project from the editor.
 		</member>
+		<member name="internationalization/pseudolocalization/double_vowels" type="bool" setter="" getter="" default="false">
+			Double vowels in strings during pseudolocalization to simulate the lengthening of text due to localization.
+		</member>
+		<member name="internationalization/pseudolocalization/expansion_ratio" type="float" setter="" getter="" default="0.0">
+			The expansion ratio to use during pseudolocalization. A value of [code]0.3[/code] is sufficient for most practical purposes, and will increase the length of each string by 30%.
+		</member>
+		<member name="internationalization/pseudolocalization/fake_bidi" type="bool" setter="" getter="" default="false">
+			If [code]true[/code], emulate bidirectional (right-to-left) text when pseudolocalization is enabled. This can be used to spot issues with RTL layout and UI mirroring that will crop up if the project is localized to RTL languages such as Arabic or Hebrew.
+		</member>
+		<member name="internationalization/pseudolocalization/override" type="bool" setter="" getter="" default="false">
+			Replace all characters in the string with [code]*[/code]. Useful for finding non-localizable strings.
+		</member>
+		<member name="internationalization/pseudolocalization/prefix" type="String" setter="" getter="" default="&quot;[&quot;">
+			Prefix that will be prepended to the pseudolocalized string.
+		</member>
+		<member name="internationalization/pseudolocalization/replace_with_accents" type="bool" setter="" getter="" default="true">
+			Replace all characters with their accented variants during pseudolocalization.
+		</member>
+		<member name="internationalization/pseudolocalization/skip_placeholders" type="bool" setter="" getter="" default="true">
+			Skip placeholders for string formatting like [code]%s[/code] or [code]%f[/code] during pseudolocalization. Useful to identify strings which need additional control characters to display correctly.
+		</member>
+		<member name="internationalization/pseudolocalization/suffix" type="String" setter="" getter="" default="&quot;]&quot;">
+			Suffix that will be appended to the pseudolocalized string.
+		</member>
+		<member name="internationalization/pseudolocalization/use_pseudolocalization" type="bool" setter="" getter="" default="false">
+			If [code]true[/code], enables pseudolocalization for the project. This can be used to spot untranslatable strings or layout issues that may occur once the project is localized to languages that have longer strings than the source language.
+			[b]Note:[/b] This property is only read when the project starts. To toggle pseudolocalization at run-time, use [member TranslationServer.pseudolocalization_enabled] instead.
+		</member>
 		<member name="internationalization/rendering/force_right_to_left_layout_direction" type="bool" setter="" getter="" default="false">
 			Force layout direction and text writing direction to RTL for all locales.
 		</member>

doc/classes/TranslationServer.xml

@@ -51,6 +51,19 @@
 				It will return a [code]nullptr[/code] if there is no [Translation] instance that matches the [code]locale[/code].
 			</description>
 		</method>
+		<method name="pseudolocalize" qualifiers="const">
+			<return type="StringName" />
+			<argument index="0" name="message" type="StringName" />
+			<description>
+				Returns the pseudolocalized string based on the [code]p_message[/code] passed in.
+			</description>
+		</method>
+		<method name="reload_pseudolocalization">
+			<return type="void" />
+			<description>
+				Reparses the pseudolocalization options and reloads the translation.
+			</description>
+		</method>
 		<method name="remove_translation">
 			<return type="void" />
 			<argument index="0" name="translation" type="Translation" />
@@ -85,6 +98,11 @@
 			</description>
 		</method>
 	</methods>
+	<members>
+		<member name="pseudolocalization_enabled" type="bool" setter="set_pseudolocalization_enabled" getter="is_pseudolocalization_enabled" default="false">
+			If [code]true[/code], enables the use of pseudolocalization. See [member ProjectSettings.internationalization/pseudolocalization/use_pseudolocalization] for details.
+		</member>
+	</members>
 	<constants>
 	</constants>
 </class>

editor/editor_settings.cpp

@@ -374,6 +374,11 @@ void EditorSettings::_load_defaults(Ref<ConfigFile> p_extra_config) {
 	_initial_set("interface/editor/display_scale", 0);
 	// Display what the Auto display scale setting effectively corresponds to.
 	float scale = get_auto_display_scale();
+
+	_initial_set("interface/editor/enable_debugging_pseudolocalization", false);
+	set_restart_if_changed("interface/editor/enable_debugging_pseudolocalization", true);
+	// Use pseudolocalization in editor.
+
 	hints["interface/editor/display_scale"] = PropertyInfo(Variant::INT, "interface/editor/display_scale", PROPERTY_HINT_ENUM, vformat("Auto (%d%%),75%%,100%%,125%%,150%%,175%%,200%%,Custom", Math::round(scale * 100)), PROPERTY_USAGE_DEFAULT | PROPERTY_USAGE_RESTART_IF_CHANGED);
 	_initial_set("interface/editor/custom_display_scale", 1.0f);
 	hints["interface/editor/custom_display_scale"] = PropertyInfo(Variant::FLOAT, "interface/editor/custom_display_scale", PROPERTY_HINT_RANGE, "0.5,3,0.01", PROPERTY_USAGE_DEFAULT | PROPERTY_USAGE_RESTART_IF_CHANGED);
@@ -953,11 +958,11 @@ fail:
 }
 
 void EditorSettings::setup_language() {
+	TranslationServer::get_singleton()->set_editor_pseudolocalization(get("interface/editor/enable_debugging_pseudolocalization"));
 	String lang = get("interface/editor/editor_language");
 	if (lang == "en") {
 		return; // Default, nothing to do.
 	}
-
 	// Load editor translation for configured/detected locale.
 	EditorTranslationList *etl = _editor_translations;
 	while (etl->data) {