Merge pull request #2119 from yiisoft/i18n-language-fallback

Fixes #2079

docs/guide/i18n.md

@@ -40,6 +40,8 @@ Format is `ll-CC` where `ll` is  two- or three-letter lowercase code for a langu
 [ISO-639](http://www.loc.gov/standards/iso639-2/) and `CC` is country code according to
 [ISO-3166](http://www.iso.org/iso/en/prods-services/iso3166ma/02iso-3166-code-lists/list-en1.html).
 
+If there's no translation for `ru-RU` Yii will try `ru` as well before failing.
+
 > **Note**: you can further customize details specifying language
 > [as documented in ICU project](http://userguide.icu-project.org/locale#TOC-The-Locale-Concept).
 
@@ -64,7 +66,7 @@ Yii tries to load appropriate translation from one of the message sources define
 			'app*' => [
 				'class' => 'yii\i18n\PhpMessageSource',
 				//'basePath' => '@app/messages',
-				//'sourceLanguage' => 'en-US',
+				//'sourceLanguage' => 'en',
 				'fileMap' => [
 					'app' => 'app.php',
 					'app/error' => 'error.php',
@@ -273,8 +275,8 @@ You can use i18n in your views to provide support for different languages. For e
 you want to create special case for russian language, you create `ru-RU` folder under the view path of current controller/widget and
 put there file for russian language as follows `views/site/ru-RU/index.php`.
 
-> **Note**: You should note that in **Yii2** language id style has changed, now it use dash **ru-RU, en-US, pl-PL** instead of underscore, because of
-> php **intl** library.
+> **Note**: If language is specified as `en-US` and there are no corresponding views, Yii will try views under `en`
+> before using original ones.
 
 Formatters
 ----------

framework/CHANGELOG.md

@@ -80,6 +80,10 @@ Yii Framework 2 Change Log
 - Enh #2008: `yii message/extract` is now able to save translation strings to database (kate-kate, samdark)
 - Enh #2043: Added support for custom request body parsers (danschmidt5189, cebe)
 - Enh #2051: Do not save null data into database when using RBAC (qiangxue)
+- Enh #2079:
+	- i18n now falls back to `en` from `en-US` if message translation isn't found (samdark)
+	- View now falls back to `en` from `en-US` if file not found (samdark)
+	- Default `sourceLanguage` and `language` are now `en` (samdark)
 - Enh #2101: Gii is now using model labels when generating search (thiagotalma)
 - Enh #2103: Renamed AccessDeniedHttpException to ForbiddenHttpException, added new commonly used HTTP exception classes (danschmidt5189)
 - Enh #2124: Added support for UNION ALL queries (Ivan Pomortsev, iworker)

framework/base/Application.php

@@ -80,13 +80,13 @@ abstract class Application extends Module
 	 * @var string the language that is meant to be used for end users.
 	 * @see sourceLanguage
 	 */
-	public $language = 'en-US';
+	public $language = 'en';
 	/**
 	 * @var string the language that the application is written in. This mainly refers to
 	 * the language that the messages and view files are written in.
 	 * @see language
 	 */
-	public $sourceLanguage = 'en-US';
+	public $sourceLanguage = 'en';
 	/**
 	 * @var Controller the currently active controller instance
 	 */

framework/helpers/BaseFileHelper.php

@@ -42,9 +42,9 @@ class BaseFileHelper
 	 * The searching is based on the specified language code. In particular,
 	 * a file with the same name will be looked for under the subdirectory
 	 * whose name is the same as the language code. For example, given the file "path/to/view.php"
-	 * and language code "zh_CN", the localized file will be looked for as
-	 * "path/to/zh_CN/view.php". If the file is not found, the original file
-	 * will be returned.
+	 * and language code "zh-CN", the localized file will be looked for as
+	 * "path/to/zh-CN/view.php". If the file is not found, it will try a fallback with just a language code that is
+	 * "zh" i.e. "path/to/zh/view.php". If it is not found as well the original file will be returned.
 	 *
 	 * If the target and the source language codes are the same,
 	 * the original file will be returned.
@@ -69,8 +69,17 @@ class BaseFileHelper
 			return $file;
 		}
 		$desiredFile = dirname($file) . DIRECTORY_SEPARATOR . $language . DIRECTORY_SEPARATOR . basename($file);
+		if (is_file($desiredFile)) {
+			return $desiredFile;
+		} else {
+			$language = substr($language, 0, 2);
+			if ($language === $sourceLanguage) {
+				return $file;
+			}
+			$desiredFile = dirname($file) . DIRECTORY_SEPARATOR . $language . DIRECTORY_SEPARATOR . basename($file);
 			return is_file($desiredFile) ? $desiredFile : $file;
 		}
+	}
 
 	/**
 	 * Determines the MIME type of the specified file.

framework/i18n/DbMessageSource.php

@@ -111,8 +111,9 @@ class DbMessageSource extends MessageSource
 
 	/**
 	 * Loads the message translation for the specified language and category.
-	 * Child classes should override this method to return the message translations of
-	 * the specified language and category.
+	 * If translation for specific locale code such as `en-US` isn't found it
+	 * tries more generic `en`.
+	 *
 	 * @param string $category the message category
 	 * @param string $language the target language
 	 * @return array the loaded messages. The keys are original messages, and the values
@@ -146,13 +147,25 @@ class DbMessageSource extends MessageSource
 	 */
 	protected function loadMessagesFromDb($category, $language)
 	{
-		$query = new Query();
-		$messages = $query->select(['t1.message message', 't2.translation translation'])
+		$mainQuery = new Query();
+		$mainQuery->select(['t1.message message', 't2.translation translation'])
 			->from([$this->sourceMessageTable . ' t1', $this->messageTable . ' t2'])
 			->where('t1.id = t2.id AND t1.category = :category AND t2.language = :language')
-			->params([':category' => $category, ':language' => $language])
-			->createCommand($this->db)
-			->queryAll();
+			->params([':category' => $category, ':language' => $language]);
+
+		$fallbackLanguage = substr($language, 0, 2);
+		if ($fallbackLanguage != $language) {
+			$fallbackQuery = new Query();
+			$fallbackQuery->select(['t1.message message', 't2.translation translation'])
+				->from([$this->sourceMessageTable . ' t1', $this->messageTable . ' t2'])
+				->where('t1.id = t2.id AND t1.category = :category AND t2.language = :fallbackLanguage')
+				->andWhere('t2.id NOT IN (SELECT id FROM '.$this->messageTable.' WHERE language = :language)')
+				->params([':category' => $category, ':language' => $language, ':fallbackLanguage' => $fallbackLanguage]);
+
+			$mainQuery->union($fallbackQuery, true);
+		}
+
+		$messages = $mainQuery->createCommand($this->db)->queryAll();
 		return ArrayHelper::map($messages, 'message', 'translation');
 	}
 }

framework/i18n/GettextMessageSource.php

@@ -50,14 +50,51 @@ class GettextMessageSource extends MessageSource
 
 	/**
 	 * Loads the message translation for the specified language and category.
-	 * Child classes should override this method to return the message translations of
-	 * the specified language and category.
+	 * If translation for specific locale code such as `en-US` isn't found it
+	 * tries more generic `en`.
+	 *
 	 * @param string $category the message category
 	 * @param string $language the target language
 	 * @return array the loaded messages. The keys are original messages, and the values
 	 * are translated messages.
 	 */
 	protected function loadMessages($category, $language)
+	{
+		$messageFile = $this->getMessageFilePath($category, $language);
+		$messages = $this->loadMessagesFromFile($messageFile);
+
+		$fallbackLanguage = substr($language, 0, 2);
+		if ($fallbackLanguage != $language) {
+			$fallbackMessageFile = $this->getMessageFilePath($category, $fallbackLanguage);
+			$fallbackMessages = $this->loadMessagesFromFile($fallbackMessageFile);
+
+			if ($messages === null && $fallbackMessages === null && $fallbackLanguage != $this->sourceLanguage) {
+				Yii::error("The message file for category '$category' does not exist: $messageFile Fallback file does not exist as well: $fallbackMessageFile", __METHOD__);
+			} else if (empty($messages)) {
+				return $fallbackMessages;
+			} else if (!empty($fallbackMessages)) {
+				foreach ($fallbackMessages as $key => $value) {
+					if (!empty($value) && empty($messages[$key])) {
+						$messages[$key] = $fallbackMessages[$key];
+					}
+				}
+			}
+		} else {
+			if ($messages === null) {
+				Yii::error("The message file for category '$category' does not exist: $messageFile", __METHOD__);
+			}
+		}
+		return (array)$messages;
+	}
+
+	/**
+	 * Returns message file path for the specified language and category.
+	 *
+	 * @param string $category the message category
+	 * @param string $language the target language
+	 * @return string path to message file
+	 */
+	protected function getMessageFilePath($category, $language)
 	{
 		$messageFile = Yii::getAlias($this->basePath) . '/' . $language . '/' . $this->catalog;
 		if ($this->useMoFile) {
@@ -65,7 +102,17 @@ class GettextMessageSource extends MessageSource
 		} else {
 			$messageFile .= static::PO_FILE_EXT;
 		}
+		return $messageFile;
+	}
 
+	/**
+	 * Loads the message translation for the specified language and category or returns null if file doesn't exist.
+	 *
+	 * @param $messageFile string path to message file
+	 * @return array|null array of messages or null if file not found
+	 */
+	protected function loadMessagesFromFile($messageFile)
+	{
 		if (is_file($messageFile)) {
 			if ($this->useMoFile) {
 				$gettextFile = new GettextMoFile(['useBigEndian' => $this->useBigEndian]);
@@ -78,8 +125,7 @@ class GettextMessageSource extends MessageSource
 			}
 			return $messages;
 		} else {
-			Yii::error("The message file for category '$category' does not exist: $messageFile", __METHOD__);
-			return [];
+			return null;
 		}
 	}
 }

framework/i18n/I18N.php

@@ -54,14 +54,14 @@ class I18N extends Component
 		if (!isset($this->translations['yii'])) {
 			$this->translations['yii'] = [
 				'class' => 'yii\i18n\PhpMessageSource',
-				'sourceLanguage' => 'en-US',
+				'sourceLanguage' => 'en',
 				'basePath' => '@yii/messages',
 			];
 		}
 		if (!isset($this->translations['app'])) {
 			$this->translations['app'] = [
 				'class' => 'yii\i18n\PhpMessageSource',
-				'sourceLanguage' => 'en-US',
+				'sourceLanguage' => 'en',
 				'basePath' => '@app/messages',
 			];
 		}

framework/i18n/MessageSource.php

@@ -53,8 +53,9 @@ class MessageSource extends Component
 
 	/**
 	 * Loads the message translation for the specified language and category.
-	 * Child classes should override this method to return the message translations of
-	 * the specified language and category.
+	 * If translation for specific locale code such as `en-US` isn't found it
+	 * tries more generic `en`.
+	 *
 	 * @param string $category the message category
 	 * @param string $language the target language
 	 * @return array the loaded messages. The keys are original messages, and the values

framework/i18n/PhpMessageSource.php

@@ -53,11 +53,51 @@ class PhpMessageSource extends MessageSource
 
 	/**
 	 * Loads the message translation for the specified language and category.
+	 * If translation for specific locale code such as `en-US` isn't found it
+	 * tries more generic `en`.
+	 *
 	 * @param string $category the message category
 	 * @param string $language the target language
-	 * @return array the loaded messages
+	 * @return array the loaded messages. The keys are original messages, and the values
+	 * are translated messages.
 	 */
 	protected function loadMessages($category, $language)
+	{
+		$messageFile = $this->getMessageFilePath($category, $language);
+		$messages = $this->loadMessagesFromFile($messageFile);
+
+		$fallbackLanguage = substr($language, 0, 2);
+		if ($fallbackLanguage != $language) {
+			$fallbackMessageFile = $this->getMessageFilePath($category, $fallbackLanguage);
+			$fallbackMessages = $this->loadMessagesFromFile($fallbackMessageFile);
+
+			if ($messages === null && $fallbackMessages === null && $fallbackLanguage != $this->sourceLanguage) {
+				Yii::error("The message file for category '$category' does not exist: $messageFile Fallback file does not exist as well: $fallbackMessageFile", __METHOD__);
+			} else if (empty($messages)) {
+				return $fallbackMessages;
+			} else if (!empty($fallbackMessages)) {
+				foreach ($fallbackMessages as $key => $value) {
+					if (!empty($value) && empty($messages[$key])) {
+						$messages[$key] = $fallbackMessages[$key];
+					}
+				}
+			}
+		} else {
+			if ($messages === null) {
+				Yii::error("The message file for category '$category' does not exist: $messageFile", __METHOD__);
+			}
+		}
+		return (array)$messages;
+	}
+
+	/**
+	 * Returns message file path for the specified language and category.
+	 *
+	 * @param string $category the message category
+	 * @param string $language the target language
+	 * @return string path to message file
+	 */
+	protected function getMessageFilePath($category, $language)
 	{
 		$messageFile = Yii::getAlias($this->basePath) . "/$language/";
 		if (isset($this->fileMap[$category])) {
@@ -65,6 +105,17 @@ class PhpMessageSource extends MessageSource
 		} else {
 			$messageFile .= str_replace('\\', '/', $category) . '.php';
 		}
+		return $messageFile;
+	}
+
+	/**
+	 * Loads the message translation for the specified language and category or returns null if file doesn't exist.
+	 *
+	 * @param $messageFile string path to message file
+	 * @return array|null array of messages or null if file not found
+	 */
+	protected function loadMessagesFromFile($messageFile)
+	{
 		if (is_file($messageFile)) {
 			$messages = include($messageFile);
 			if (!is_array($messages)) {
@@ -72,8 +123,7 @@ class PhpMessageSource extends MessageSource
 			}
 			return $messages;
 		} else {
-			Yii::error("The message file for category '$category' does not exist: $messageFile", __METHOD__);
-			return [];
+			return null;
 		}
 	}
 }

framework/views/errorHandler/exception.php

@@ -6,7 +6,7 @@
 ?>
 <?php if (method_exists($this, 'beginPage')) $this->beginPage(); ?>
 <!doctype html>
-<html lang="en-us">
+<html lang="en">
 
 <head>
 	<meta charset="utf-8"/>

tests/unit/data/i18n/messages/de/test.php

@@ -0,0 +1,7 @@
+<?php
+/**
+ *
+ */
+return [
+	'Hello world!' => 'Hallo Welt!',
+];

tests/unit/data/i18n/messages/ru/test.php

@@ -0,0 +1,7 @@
+<?php
+/**
+ *
+ */
+return [
+	'The dog runs fast.' => 'Собака бегает быстро.',
+];

tests/unit/framework/i18n/I18NTest.php

@@ -40,8 +40,18 @@ class I18NTest extends TestCase
 	public function testTranslate()
 	{
 		$msg = 'The dog runs fast.';
-		$this->assertEquals('The dog runs fast.', $this->i18n->translate('test', $msg, [], 'en-US'));
+
+		// source = target. Should be returned as is.
+		$this->assertEquals('The dog runs fast.', $this->i18n->translate('test', $msg, [], 'en'));
+
+		// exact match
 		$this->assertEquals('Der Hund rennt schnell.', $this->i18n->translate('test', $msg, [], 'de-DE'));
+
+		// fallback to just language code with absent exact match
+		$this->assertEquals('Собака бегает быстро.', $this->i18n->translate('test', $msg, [], 'ru-RU'));
+
+		// fallback to just langauge code with present exact match
+		$this->assertEquals('Hallo Welt!', $this->i18n->translate('test', 'Hello world!', [], 'de-DE'));
 	}
 
 	public function testTranslateParams()