php/yii2
1
0
Fork 0
mirror of https://github.com/yiisoft/yii2.git synced 2025-08-26 14:26:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fixed i18n message sources to load fallback messages in a smarter way

Browse Source 
Added migration and unit-tests for `yii\i18n\DbMessageSource`

Closes #7964
This commit is contained in:
SilverFire - Dmitry Naumenko
2016-02-07 23:08:52 +02:00
parent 0f30b0843b
commit 32c424da10
11 changed files with 396 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
framework/CHANGELOG.md
View File

@ -8,6 +8,7 @@ Yii Framework 2 Change Log
- Bug #6876: Fixed RBAC migration MSSQL cascade problem (thejahweh)
- Bug #7627: Fixed `yii\widgets\ActiveField` to handle inputs validation with changed ID properly (dynasource, cebe)
- Bug #7806: Fixed `yii\grid\CheckboxColumn` fixed `_all` checkbox column name generation (cebe, silverfire)
- Bug #7964: Fixed i18n message sources to load fallback messages in a smarter way (silverfire)
- Bug #8348: Fixed `yii\helpers\BaseArrayHelper` fixed PHP Fatal Error: Nesting level too deep - recursive dependency? (andrewnester)
- Bug #8466: Fixed `yii\validators\FileValidator` to display error for `tooBig` and `tooSmall` with formatted unit (silverfire)
- Bug #8573: Fixed incorrect data type mapping for NUMBER to INTEGER or DECIMAL in Oracle (vbelogai)
@ -111,6 +112,7 @@ Yii Framework 2 Change Log
- Enh #10390: Added ability to disable outer tag for `\yii\helpers\BaseHtml::radiolist()`, `::checkboxList()` (TianJinRong, githubjeka, silverfire)
- Enh #10535: Allow passing a `yii\db\Expression` to `Query::orderBy()` and `Query::groupBy()` (andrewnester, cebe)
- Enh #10545: `yii\web\XMLResponseFormatter` changed to format models in a proper way (andrewnester)
- Enh #10783: Added migration and unit-tests for `yii\i18n\DbMessageSource` (silverfire)
- Enh: Added last resort measure for `FileHelper::removeDirectory()` fail to unlink symlinks under Windows (samdark)
- Chg #9369: `Yii::$app->user->can()` now returns `false` instead of erroring in case `authManager` component is not configured (creocoder)
- Chg #9411: `DetailView` now automatically sets container tag ID in case it's not specified (samdark)

75
framework/i18n/DbMessageSource.php
View File

@ -9,6 +9,7 @@ namespace yii\i18n;
use Yii;
use yii\base\InvalidConfigException;
use yii\db\Expression;
use yii\di\Instance;
use yii\helpers\ArrayHelper;
use yii\caching\Cache;
@ -19,29 +20,18 @@ use yii\db\Query;
* DbMessageSource extends [[MessageSource]] and represents a message source that stores translated
* messages in database.
*
* The database must contain the following two tables:
*
* ```sql
* CREATE TABLE source_message (
* id INTEGER PRIMARY KEY AUTO_INCREMENT,
* category VARCHAR(32),
* message TEXT
* );
*
* CREATE TABLE message (
* id INTEGER,
* language VARCHAR(16),
* translation TEXT,
* PRIMARY KEY (id, language),
* CONSTRAINT fk_message_source_message FOREIGN KEY (id)
* REFERENCES source_message (id) ON DELETE CASCADE ON UPDATE RESTRICT
* );
* ```
* The database must contain the following two tables: source_message and message.
*
* The `source_message` table stores the messages to be translated, and the `message` table stores
* the translated messages. The name of these two tables can be customized by setting [[sourceMessageTable]]
* and [[messageTable]], respectively.
*
* The database connection is specified by [[db]]. Database schema could be initialized by applying migration:
*
* ```
* yii migrate --migrationPath=@yii/i18n/migrations/
* ```
*
* @author resurtm <resurtm@gmail.com>
* @since 2.0
*/
@ -149,26 +139,49 @@ class DbMessageSource extends MessageSource
*/
protected function loadMessagesFromDb($category, $language)
{
$mainQuery = new Query();
$mainQuery->select(['t1.message message', 't2.translation translation'])
->from(["$this->sourceMessageTable t1", "$this->messageTable t2"])
->where('t1.id = t2.id AND t1.category = :category AND t2.language = :language')
->params([':category' => $category, ':language' => $language]);
$mainQuery = (new Query())->select(['message' => 't1.message', 'translation' => 't2.translation'])
->from(['t1' => $this->sourceMessageTable, 't2' => $this->messageTable])
->where([
't1.id' => new Expression('[[t2.id]]'),
't1.category' => $category,
't2.language' => $language
]);
$fallbackLanguage = substr($language, 0, 2);
if ($fallbackLanguage !== $language) {
$fallbackQuery = new Query();
$fallbackQuery->select(['t1.message message', 't2.translation translation'])
->from(["$this->sourceMessageTable t1", "$this->messageTable t2"])
->where('t1.id = t2.id AND t1.category = :category AND t2.language = :fallbackLanguage')
->andWhere("t2.id NOT IN (SELECT id FROM $this->messageTable WHERE language = :language)")
->params([':category' => $category, ':language' => $language, ':fallbackLanguage' => $fallbackLanguage]);
$fallbackSourceLanguage = substr($this->sourceLanguage, 0, 2);
$mainQuery->union($fallbackQuery, true);
if ($fallbackLanguage !== $language) {
$mainQuery->union($this->createFallbackQuery($category, $language, $fallbackLanguage), true);
} elseif ($language === $fallbackSourceLanguage) {
$mainQuery->union($this->createFallbackQuery($category, $language, $fallbackSourceLanguage), true);
}
$messages = $mainQuery->createCommand($this->db)->queryAll();
return ArrayHelper::map($messages, 'message', 'translation');
}
/**
* The method builds the [[Query]] object for the fallback language messages search.
* Normally is called from [[loadMessagesFromDb]].
*
* @param string $category the message category
* @param string $language the originally requested language
* @param string $fallbackLanguage the target fallback language
* @return Query
* @see loadMessagesFromDb
* @since 2.0.7
*/
protected function createFallbackQuery($category, $language, $fallbackLanguage)
{
return (new Query())->select(['message' => 't1.message', 'translation' => 't2.translation'])
->from(['t1' => $this->sourceMessageTable, 't2' => $this->messageTable])
->where([
't1.id' => new Expression('[[t2.id]]'),
't1.category' => $category,
't2.language' => $fallbackLanguage
])->andWhere([
'NOT IN', 't2.id', (new Query())->select('[[id]]')->from($this->messageTable)->where(['language' => $language])
]);
}
}

50
framework/i18n/GettextMessageSource.php
View File

@ -50,14 +50,19 @@ class GettextMessageSource extends MessageSource
/**
* Loads the message translation for the specified language and category.
* Loads the message translation for the specified $language and $category.
* If translation for specific locale code such as `en-US` isn't found it
* tries more generic `en`.
* tries more generic `en`. When both are present, the `en-US` messages will be merged
* over `en`. See [[loadFallbackMessages]] for details.
* If the $language is less specific than [[sourceLanguage]], the method will try to
* load the messages for [[sourceLanguage]]. For example: [[sourceLanguage]] is `en-GB`,
* $language is `en`. The method will load the messages for `en` and merge them over `en-GB`.
*
* @param string $category the message category
* @param string $language the target language
* @return array the loaded messages. The keys are original messages, and the values
* are translated messages.
* @return array the loaded messages. The keys are original messages, and the values are translated messages.
* @see loadFallbackMessages
* @see sourceLanguage
*/
protected function loadMessages($category, $language)
{
@ -65,12 +70,42 @@ class GettextMessageSource extends MessageSource
$messages = $this->loadMessagesFromFile($messageFile, $category);
$fallbackLanguage = substr($language, 0, 2);
$fallbackSourceLanguage = substr($this->sourceLanguage, 0, 2);
if ($fallbackLanguage !== $language) {
$this->loadFallbackMessages($category, $fallbackLanguage, $messages, $messageFile);
} elseif ($language === $fallbackSourceLanguage) {
$messages = $this->loadFallbackMessages($category, $this->sourceLanguage, $messages, $messageFile);
} else {
if ($messages === null) {
Yii::error("The message file for category '$category' does not exist: $messageFile", __METHOD__);
}
}
return (array) $messages;
}
/**
* The method is normally called by [[loadMessages]] to load the fallback messages for the language.
* Method tries to load the $category messages for the $fallbackLanguage and adds them to the $messages array.
*
* @param string $category the message category
* @param string $fallbackLanguage the target fallback language
* @param array $messages the array of previously loaded translation messages.
* The keys are original messages, and the values are the translated messages.
* @param string $originalMessageFile the path to the file with messages. Used to log an error message
* in case when no translations were found.
* @return array the loaded messages. The keys are original messages, and the values are the translated messages.
* @since 2.0.7
*/
protected function loadFallbackMessages($category, $fallbackLanguage, $messages, $originalMessageFile)
{
$fallbackMessageFile = $this->getMessageFilePath($fallbackLanguage);
$fallbackMessages = $this->loadMessagesFromFile($fallbackMessageFile, $category);
if ($messages === null && $fallbackMessages === null && $fallbackLanguage !== $this->sourceLanguage) {
Yii::error("The message file for category '$category' does not exist: $messageFile Fallback file does not exist as well: $fallbackMessageFile", __METHOD__);
Yii::error("The message file for category '$category' does not exist: $originalMessageFile "
. "Fallback file does not exist as well: $fallbackMessageFile", __METHOD__);
} elseif (empty($messages)) {
return $fallbackMessages;
} elseif (!empty($fallbackMessages)) {
@ -80,11 +115,6 @@ class GettextMessageSource extends MessageSource
}
}
}
} else {
if ($messages === null) {
Yii::error("The message file for category '$category' does not exist: $messageFile", __METHOD__);
}
}
return (array) $messages;
}

52
framework/i18n/PhpMessageSource.php
View File

@ -52,14 +52,19 @@ class PhpMessageSource extends MessageSource
/**
* Loads the message translation for the specified language and category.
* Loads the message translation for the specified $language and $category.
* If translation for specific locale code such as `en-US` isn't found it
* tries more generic `en`.
* tries more generic `en`. When both are present, the `en-US` messages will be merged
* over `en`. See [[loadFallbackMessages]] for details.
* If the $language is less specific than [[sourceLanguage]], the method will try to
* load the messages for [[sourceLanguage]]. For example: [[sourceLanguage]] is `en-GB`,
* $language is `en`. The method will load the messages for `en` and merge them over `en-GB`.
*
* @param string $category the message category
* @param string $language the target language
* @return array the loaded messages. The keys are original messages, and the values
* are translated messages.
* @return array the loaded messages. The keys are original messages, and the values are the translated messages.
* @see loadFallbackMessages
* @see sourceLanguage
*/
protected function loadMessages($category, $language)
{
@ -67,12 +72,42 @@ class PhpMessageSource extends MessageSource
$messages = $this->loadMessagesFromFile($messageFile);
$fallbackLanguage = substr($language, 0, 2);
if ($fallbackLanguage !== $language) {
$fallbackSourceLanguage = substr($this->sourceLanguage, 0, 2);
if ($language !== $fallbackLanguage) {
$messages = $this->loadFallbackMessages($category, $fallbackLanguage, $messages, $messageFile);
} elseif ($language === $fallbackSourceLanguage) {
$messages = $this->loadFallbackMessages($category, $this->sourceLanguage, $messages, $messageFile);
} else {
if ($messages === null) {
Yii::error("The message file for category '$category' does not exist: $messageFile", __METHOD__);
}
}
return (array) $messages;
}
/**
* The method is normally called by [[loadMessages]] to load the fallback messages for the language.
* Method tries to load the $category messages for the $fallbackLanguage and adds them to the $messages array.
*
* @param string $category the message category
* @param string $fallbackLanguage the target fallback language
* @param array $messages the array of previously loaded translation messages.
* The keys are original messages, and the values are the translated messages.
* @param string $originalMessageFile the path to the file with messages. Used to log an error message
* in case when no translations were found.
* @return array the loaded messages. The keys are original messages, and the values are the translated messages.
* @since 2.0.7
*/
protected function loadFallbackMessages($category, $fallbackLanguage, $messages, $originalMessageFile)
{
$fallbackMessageFile = $this->getMessageFilePath($category, $fallbackLanguage);
$fallbackMessages = $this->loadMessagesFromFile($fallbackMessageFile);
if ($messages === null && $fallbackMessages === null && $fallbackLanguage !== $this->sourceLanguage) {
Yii::error("The message file for category '$category' does not exist: $messageFile Fallback file does not exist as well: $fallbackMessageFile", __METHOD__);
Yii::error("The message file for category '$category' does not exist: $originalMessageFile "
. "Fallback file does not exist as well: $fallbackMessageFile", __METHOD__);
} elseif (empty($messages)) {
return $fallbackMessages;
} elseif (!empty($fallbackMessages)) {
@ -82,11 +117,6 @@ class PhpMessageSource extends MessageSource
}
}
}
} else {
if ($messages === null) {
Yii::error("The message file for category '$category' does not exist: $messageFile", __METHOD__);
}
}
return (array) $messages;
}

48
framework/i18n/migrations/m150207_210500_i18n_init.php Normal file
View File

@ -0,0 +1,48 @@
<?php
/**
* @link http://www.yiiframework.com/
* @copyright Copyright (c) 2008 Yii Software LLC
* @license http://www.yiiframework.com/license/
*/
use yii\db\Migration;
/**
* Initializes i18n messages tables.
*
* @author Dmitry Naumenko <d.naumenko.a@gmail.com>
* @since 2.0.7
*/
class m150207_210500_i18n_init extends Migration
{
public function up()
{
$tableOptions = null;
if ($this->db->driverName === 'mysql') {
// http://stackoverflow.com/questions/766809/whats-the-difference-between-utf8-general-ci-and-utf8-unicode-ci
$tableOptions = 'CHARACTER SET utf8 COLLATE utf8_unicode_ci ENGINE=InnoDB';
}
$this->createTable('source_message', [
'id' => $this->primaryKey(),
'category' => $this->string(),
'message' => $this->text(),
], $tableOptions);
$this->createTable('message', [
'id' => $this->integer(),
'language' => $this->string(16),
'translation' => $this->text(),
], $tableOptions);
$this->addPrimaryKey('pk_message_id_language', 'message', ['id', 'language']);
$this->addForeignKey('fk_message_source_message', 'message', 'id', 'source_message', 'id', 'CASCADE', 'RESTRICT');
}
public function down()
{
$this->dropForeignKey('fk_message_source_message', 'message');
$this->dropTable('message');
$this->dropTable('source_message');
}
}

3
tests/data/i18n/messages/de-DE/test.php
View File

@ -1,6 +1,7 @@
<?php
/**
*
* Messages are copy-pasted in \yiiunit\framework\i18n\DbMessageSourceTest::setUpBeforeClass()
* Do not forget to update it in after changing this file!
*/
return [
'The dog runs fast.' => 'Der Hund rennt schnell.',

3
tests/data/i18n/messages/de/test.php
View File

@ -1,6 +1,7 @@
<?php
/**
*
* Messages are copy-pasted in \yiiunit\framework\i18n\DbMessageSourceTest::setUpBeforeClass()
* Do not forget to update it in after changing this file!
*/
return [
'Hello world!' => 'Hallo Welt!',

5
tests/data/i18n/messages/en-US/test.php
View File

@ -1,7 +1,8 @@
<?php
/**
*
* Messages are copy-pasted in \yiiunit\framework\i18n\DbMessageSourceTest::setUpBeforeClass()
* Do not forget to update it in after changing this file!
*/
return [
'The dog runs fast.' => 'Der Hund rennt schell.',
'The dog runs fast.' => 'The dog runs fast (en-US).',
];

3
tests/data/i18n/messages/ru/test.php
View File

@ -1,6 +1,7 @@
<?php
/**
*
* Messages are copy-pasted in \yiiunit\framework\i18n\DbMessageSourceTest::setUpBeforeClass()
* Do not forget to update it in after changing this file!
*/
return [
'The dog runs fast.' => 'Собака бегает быстро.',

156
tests/framework/i18n/DbMessageSourceTest.php Normal file
View File

@ -0,0 +1,156 @@
<?php
namespace yiiunit\framework\i18n;
use Yii;
use yii\base\Event;
use yii\db\Connection;
use yii\i18n\DbMessageSource;
use yii\i18n\I18N;
use yiiunit\framework\console\controllers\EchoMigrateController;
/**
* @group i18n
* @group mysql
* @author Dmitry Naumenko <d.naumenko.a@gmail.com>
* @since 2.0.7
*/
class DbMessageSourceTest extends I18NTest
{
protected static $database;
protected static $driverName = 'mysql';
/**
* @var Connection
*/
protected static $db;
protected function setI18N()
{
$this->i18n = new I18N([
'translations' => [
'test' => new DbMessageSource([
'db' => static::$db,
])
]
]);
}
protected static function runConsoleAction($route, $params = [])
{
if (Yii::$app === null) {
new \yii\console\Application([
'id' => 'Migrator',
'basePath' => '@yiiunit',
'controllerMap' => [
'migrate' => EchoMigrateController::className(),
],
'components' => [
'db' => static::getConnection(),
],
]);
}
ob_start();
$result = Yii::$app->runAction($route, $params);
echo "Result is " . $result;
if ($result !== \yii\console\Controller::EXIT_CODE_NORMAL) {
ob_end_flush();
} else {
ob_end_clean();
}
}
public static function setUpBeforeClass()
{
parent::setUpBeforeClass();
$databases = static::getParam('databases');
static::$database = $databases[static::$driverName];
$pdo_database = 'pdo_' . static::$driverName;
if (!extension_loaded('pdo') || !extension_loaded($pdo_database)) {
static::markTestSkipped('pdo and ' . $pdo_database . ' extension are required.');
}
static::runConsoleAction('migrate/up', ['migrationPath' => '@yii/i18n/migrations/', 'interactive' => false]);
static::$db->createCommand()->batchInsert('source_message', ['id', 'category', 'message'], [
[1, 'test', 'Hello world!'],
[2, 'test', 'The dog runs fast.'],
[3, 'test', 'His speed is about {n} km/h.'],
[4, 'test', 'His name is {name} and his speed is about {n, number} km/h.'],
[5, 'test', 'There {n, plural, =0{no cats} =1{one cat} other{are # cats}} on lying on the sofa!'],
])->execute();
static::$db->createCommand()->batchInsert('message', ['id', 'language', 'translation'], [
[1, 'de', 'Hallo Welt!'],
[2, 'de-DE', 'Der Hund rennt schnell.'],
[2, 'en-US', 'The dog runs fast (en-US).'],
[2, 'ru', 'Собака бегает быстро.'],
[3, 'de-DE', 'Seine Geschwindigkeit beträgt {n} km/h.'],
[4, 'de-DE', 'Er heißt {name} und ist {n, number} km/h schnell.'],
[5, 'ru', 'На диване {n, plural, =0{нет кошек} =1{лежит одна кошка} one{лежит # кошка} few{лежит # кошки} many{лежит # кошек} other{лежит # кошки}}!'],
])->execute();
}
public static function tearDownAfterClass()
{
static::runConsoleAction('migrate/down', ['migrationPath' => '@yii/i18n/migrations/', 'interactive' => false]);
if (static::$db) {
static::$db->close();
}
Yii::$app = null;
parent::tearDownAfterClass();
}
/**
* @throws \yii\base\InvalidParamException
* @throws \yii\db\Exception
* @throws \yii\base\InvalidConfigException
* @return \yii\db\Connection
*/
public static function getConnection()
{
if (static::$db == null) {
$db = new Connection;
$db->dsn = static::$database['dsn'];
if (isset(static::$database['username'])) {
$db->username = static::$database['username'];
$db->password = static::$database['password'];
}
if (isset(static::$database['attributes'])) {
$db->attributes = static::$database['attributes'];
}
if (!$db->isActive) {
$db->open();
}
static::$db = $db;
}
return static::$db;
}
public function testMissingTranslationEvent()
{
$this->assertEquals('Hallo Welt!', $this->i18n->translate('test', 'Hello world!', [], 'de-DE'));
$this->assertEquals('Missing translation message.', $this->i18n->translate('test', 'Missing translation message.', [], 'de-DE'));
$this->assertEquals('Hallo Welt!', $this->i18n->translate('test', 'Hello world!', [], 'de-DE'));
Event::on(DbMessageSource::className(), DbMessageSource::EVENT_MISSING_TRANSLATION, function ($event) {});
$this->assertEquals('Hallo Welt!', $this->i18n->translate('test', 'Hello world!', [], 'de-DE'));
$this->assertEquals('Missing translation message.', $this->i18n->translate('test', 'Missing translation message.', [], 'de-DE'));
$this->assertEquals('Hallo Welt!', $this->i18n->translate('test', 'Hello world!', [], 'de-DE'));
Event::off(DbMessageSource::className(), DbMessageSource::EVENT_MISSING_TRANSLATION);
Event::on(DbMessageSource::className(), DbMessageSource::EVENT_MISSING_TRANSLATION, function ($event) {
if ($event->message == 'New missing translation message.') {
$event->translatedMessage = 'TRANSLATION MISSING HERE!';
}
});
$this->assertEquals('Hallo Welt!', $this->i18n->translate('test', 'Hello world!', [], 'de-DE'));
$this->assertEquals('Another missing translation message.', $this->i18n->translate('test', 'Another missing translation message.', [], 'de-DE'));
$this->assertEquals('Missing translation message.', $this->i18n->translate('test', 'Missing translation message.', [], 'de-DE'));
$this->assertEquals('TRANSLATION MISSING HERE!', $this->i18n->translate('test', 'New missing translation message.', [], 'de-DE'));
$this->assertEquals('Hallo Welt!', $this->i18n->translate('test', 'Hello world!', [], 'de-DE'));
Event::off(DbMessageSource::className(), DbMessageSource::EVENT_MISSING_TRANSLATION);
}
}

42
tests/framework/i18n/I18NTest.php
View File

@ -28,6 +28,11 @@ class I18NTest extends TestCase
{
parent::setUp();
$this->mockApplication();
$this->setI18N();
}
protected function setI18N()
{
$this->i18n = new I18N([
'translations' => [
'test' => new PhpMessageSource([
@ -42,7 +47,7 @@ class I18NTest extends TestCase
$msg = 'The dog runs fast.';
// source = target. Should be returned as is.
$this->assertEquals('The dog runs fast.', $this->i18n->translate('test', $msg, [], 'en'));
$this->assertEquals('The dog runs fast.', $this->i18n->translate('test', $msg, [], 'en-US'));
// exact match
$this->assertEquals('Der Hund rennt schnell.', $this->i18n->translate('test', $msg, [], 'de-DE'));
@ -71,7 +76,7 @@ class I18NTest extends TestCase
$msg = 'The dog runs fast.';
// source = target. Should be returned as is.
$this->assertEquals($msg, $i18n->translate('test', $msg, [], 'en'));
$this->assertEquals($msg, $i18n->translate('test', $msg, [], 'en-US'));
// exact match
$this->assertEquals('Der Hund rennt schnell.', $i18n->translate('test', $msg, [], 'de-DE'));
@ -85,6 +90,39 @@ class I18NTest extends TestCase
$this->assertEquals('Hallo Welt!', $i18n->translate('test', 'Hello world!', [], 'de-DE'));
}
/**
* https://github.com/yiisoft/yii2/issues/7964
*/
public function testSourceLanguageFallback()
{
$i18n = new I18N([
'translations' => [
'*' => new PhpMessageSource([
'basePath' => '@yiiunit/data/i18n/messages',
'sourceLanguage' => 'de-DE',
'fileMap' => [
'test' => 'test.php',
'foo' => 'test.php',
],
]
)
]
]);
$msg = 'The dog runs fast.';
// source = target. Should be returned as is.
$this->assertEquals($msg, $i18n->translate('test', $msg, [], 'de-DE'));
// target is less specific, than a source. Messages from sourceLanguage file should be loaded as a fallback
$this->assertEquals('Der Hund rennt schnell.', $i18n->translate('test', $msg, [], 'de'));
$this->assertEquals('Hallo Welt!', $i18n->translate('test', 'Hello world!', [], 'de'));
// target is a different language than source
$this->assertEquals('Собака бегает быстро.', $i18n->translate('test', $msg, [], 'ru-RU'));
$this->assertEquals('Собака бегает быстро.', $i18n->translate('test', $msg, [], 'ru'));
}
public function testTranslateParams()
{
$msg = 'His speed is about {n} km/h.';