From bc3200cef1c94808b86ab7e2ae907155cf065db6 Mon Sep 17 00:00:00 2001 From: Alexander Makarov Date: Thu, 17 Jul 2014 19:27:26 +0400 Subject: [PATCH] Fixes #4295: reworked message extraction for PO files --- .../console/controllers/MessageController.php | 219 ++++++++++++------ framework/views/messageConfig.php | 31 ++- .../controllers/MessageControllerTest.php | 10 +- 3 files changed, 175 insertions(+), 85 deletions(-) diff --git a/framework/console/controllers/MessageController.php b/framework/console/controllers/MessageController.php index 22f1b03890..519a09bfe2 100644 --- a/framework/console/controllers/MessageController.php +++ b/framework/console/controllers/MessageController.php @@ -12,6 +12,7 @@ use yii\console\Controller; use yii\console\Exception; use yii\helpers\FileHelper; use yii\helpers\VarDumper; +use yii\i18n\GettextPoFile; /** * Extracts messages to be translated from source files. @@ -88,14 +89,16 @@ class MessageController extends Controller 'format' => 'php', ], require($configFile)); - if (!isset($config['sourcePath'], $config['messagePath'], $config['languages'])) { - throw new Exception('The configuration file must specify "sourcePath", "messagePath" and "languages".'); + if (!isset($config['sourcePath'], $config['languages'])) { + throw new Exception('The configuration file must specify "sourcePath" and "languages".'); } if (!is_dir($config['sourcePath'])) { throw new Exception("The source path {$config['sourcePath']} is not a valid directory."); } if (in_array($config['format'], ['php', 'po'])) { - if (!is_dir($config['messagePath'])) { + if (!isset($config['messagePath'])) { + throw new Exception('The configuration file must specify "messagePath".'); + } elseif (!is_dir($config['messagePath'])) { throw new Exception("The message path {$config['messagePath']} is not a valid directory."); } } @@ -118,14 +121,11 @@ class MessageController extends Controller if (!is_dir($dir)) { @mkdir($dir); } - foreach ($messages as $category => $msgs) { - $file = str_replace("\\", '/', "$dir/$category." . $config['format']); - $path = dirname($file); - if (!is_dir($path)) { - mkdir($path, 0755, true); - } - $msgs = array_values(array_unique($msgs)); - $this->generateMessageFile($msgs, $file, $config['overwrite'], $config['removeUnused'], $config['sort'], $config['format']); + if ($config['format'] === 'po') { + $catalog = isset($config['catalog']) ? $config['catalog'] : 'messages'; + $this->saveMessagesToPO($messages, $dir, $config['overwrite'], $config['removeUnused'], $config['sort'], $catalog); + } else { + $this->saveMessagesToPHP($messages, $dir, $config['overwrite'], $config['removeUnused'], $config['sort']); } } } elseif ($config['format'] === 'db') { @@ -200,11 +200,11 @@ class MessageController extends Controller $savedFlag = true; $db->createCommand() - ->insert($sourceMessageTable, ['category' => $category, 'message' => $m])->execute(); + ->insert($sourceMessageTable, ['category' => $category, 'message' => $m])->execute(); $lastId = $db->getLastInsertID(); foreach ($languages as $language) { $db->createCommand() - ->insert($messageTable, ['id' => $lastId, 'language' => $language])->execute(); + ->insert($messageTable, ['id' => $lastId, 'language' => $language])->execute(); } } } @@ -217,19 +217,19 @@ class MessageController extends Controller } else { if ($removeUnused) { $db->createCommand() - ->delete($sourceMessageTable, ['in', 'id', $obsolete])->execute(); + ->delete($sourceMessageTable, ['in', 'id', $obsolete])->execute(); echo "deleted.\n"; } else { $last_id = $db->getLastInsertID(); $db->createCommand() - ->update( - $sourceMessageTable, - ['message' => new \yii\db\Expression("CONCAT('@@',message,'@@')")], - ['in', 'id', $obsolete] - )->execute(); + ->update( + $sourceMessageTable, + ['message' => new \yii\db\Expression("CONCAT('@@',message,'@@')")], + ['in', 'id', $obsolete] + )->execute(); foreach ($languages as $language) { $db->createCommand() - ->insert($messageTable, ['id' => $last_id, 'language' => $language])->execute(); + ->insert($messageTable, ['id' => $last_id, 'language' => $language])->execute(); } echo "updated.\n"; } @@ -273,27 +273,41 @@ class MessageController extends Controller } /** - * Writes messages into file + * Writes messages into PHP files + * + * @param array $messages + * @param string $dirName name of the directory to write to + * @param boolean $overwrite if existing file should be overwritten without backup + * @param boolean $removeUnused if obsolete translations should be removed + * @param boolean $sort if translations should be sorted + */ + protected function saveMessagesToPHP($messages, $dirName, $overwrite, $removeUnused, $sort) + { + foreach ($messages as $category => $msgs) { + $file = str_replace("\\", '/', "$dirName/$category.php"); + $path = dirname($file); + if (!is_dir($path)) { + mkdir($path, 0755, true); + } + $msgs = array_values(array_unique($msgs)); + echo "Saving messages to $file...\n"; + $this->saveMessagesCategoryToPHP($msgs, $file, $overwrite, $removeUnused, $sort); + } + } + + /** + * Writes category messages into PHP file * * @param array $messages * @param string $fileName name of the file to write to * @param boolean $overwrite if existing file should be overwritten without backup * @param boolean $removeUnused if obsolete translations should be removed * @param boolean $sort if translations should be sorted - * @param string $format output format */ - protected function generateMessageFile($messages, $fileName, $overwrite, $removeUnused, $sort, $format) + protected function saveMessagesCategoryToPHP($messages, $fileName, $overwrite, $removeUnused, $sort) { - echo "Saving messages to $fileName..."; if (is_file($fileName)) { - if ($format === 'po') { - $translated = file_get_contents($fileName); - preg_match_all('/(?<=msgid ").*(?="\n(#*)msgstr)/', $translated, $keys); - preg_match_all('/(?<=msgstr ").*(?="\n\n)/', $translated, $values); - $translated = array_combine($keys[0], $values[0]); - } else { - $translated = require($fileName); - } + $translated = require($fileName); sort($messages); ksort($translated); if (array_keys($translated) == $messages) { @@ -304,9 +318,6 @@ class MessageController extends Controller $merged = []; $untranslated = []; foreach ($messages as $message) { - if ($format === 'po') { - $message = preg_replace('/\"/', '\"', $message); - } if (array_key_exists($message, $translated) && strlen($translated[$message]) > 0) { $merged[$message] = $translated[$message]; } else { @@ -336,47 +347,19 @@ class MessageController extends Controller if (false === $overwrite) { $fileName .= '.merged'; } - if ($format === 'po') { - $output = ''; - foreach ($merged as $k => $v) { - $k = preg_replace('/(\")|(\\\")/', "\\\"", $k); - $v = preg_replace('/(\")|(\\\")/', "\\\"", $v); - if (substr($v, 0, 2) === '@@' && substr($v, -2) === '@@') { - $output .= "#msgid \"$k\"\n"; - $output .= "#msgstr \"$v\"\n"; - } else { - $output .= "msgid \"$k\"\n"; - $output .= "msgstr \"$v\"\n"; - } - $output .= "\n"; - } - $merged = $output; - } - echo "translation merged.\n"; + echo "Translation merged.\n"; } else { - if ($format === 'po') { - $merged = ''; - sort($messages); - foreach ($messages as $message) { - $message = preg_replace('/(\")|(\\\")/', '\\\"', $message); - $merged .= "msgid \"$message\"\n"; - $merged .= "msgstr \"\"\n"; - $merged .= "\n"; - } - } else { - $merged = []; - foreach ($messages as $message) { - $merged[$message] = ''; - } - ksort($merged); + $merged = []; + foreach ($messages as $message) { + $merged[$message] = ''; } - echo "saved.\n"; + ksort($merged); } - if ($format === 'po') { - $content = $merged; - } else { - $array = VarDumper::export($merged); - $content = << $msgs) { + $msgs = array_values(array_unique($msgs)); + + if (is_file($file)) { + $existingMessages = $poFile->load($file, $category); + + sort($msgs); + ksort($existingMessages); + if (array_keys($existingMessages) == $msgs) { + echo "Nothing new... skipped.\n"; + return self::EXIT_CODE_NORMAL; + } + + // merge existing message translations with new message translations + foreach ($msgs as $message) { + if (array_key_exists($message, $existingMessages) && strlen($existingMessages[$message]) > 0) { + $merged[$category . chr(4) . $message] = $existingMessages[$message]; + } else { + $notTranslatedYet[] = $message; + } + } + ksort($merged); + sort($notTranslatedYet); + + // collect not yet translated messages + foreach ($notTranslatedYet as $message) { + $todos[$category . chr(4) . $message] = ''; + } + + // add obsolete unused messages + foreach ($existingMessages as $message => $translation) { + if (!isset($merged[$category . chr(4) . $message]) && !isset($todos[$category . chr(4) . $message]) && !$removeUnused) { + if (substr($translation, 0, 2) === '@@' && substr($translation, -2) === '@@') { + $todos[$category . chr(4) . $message] = $translation; + } else { + $todos[$category . chr(4) . $message] = '@@' . $translation . '@@'; + } + } + } + + $merged = array_merge($todos, $merged); + if ($sort) { + ksort($merged); + } + + if ($overwrite === false) { + $file .= '.merged'; + } + + echo "Translation merged.\n"; + } else { + sort($msgs); + foreach ($msgs as $message) { + $merged[$category . chr(4) . $message] = ''; + } + ksort($merged); + } + } + $poFile->save($file, $merged); + echo "Saved.\n"; + } } diff --git a/framework/views/messageConfig.php b/framework/views/messageConfig.php index 6b4e8c9b7e..22fa10afbd 100644 --- a/framework/views/messageConfig.php +++ b/framework/views/messageConfig.php @@ -3,8 +3,6 @@ return [ // string, required, root directory of all source files 'sourcePath' => __DIR__, - // string, required, root directory containing message translations. - 'messagePath' => __DIR__ . DIRECTORY_SEPARATOR . 'messages', // array, required, list of language codes that the extracted messages // should be translated to. For example, ['zh-CN', 'de']. 'languages' => ['de'], @@ -44,9 +42,30 @@ return [ '.hgkeep', '/messages', ], - // Generated file format. Can be either "php", "po" or "db". + + // 'php' output format is for saving messages to php files. 'format' => 'php', - // When format is "db", you may specify the following two options - //'db' => 'db', - //'sourceMessageTable' => '{{%source_message}}', + // Root directory containing message translations. + 'messagePath' => __DIR__ . DIRECTORY_SEPARATOR . 'messages', + + + /* + // 'db' output format is for saving messages to database. + 'format' => 'db', + // Connection component to use. Optional. + 'db' => 'db', + // Custom source message table. Optional. + // 'sourceMessageTable' => '{{%source_message}}', + // Custom name for translation message table. Optional. + // 'messageTable' => '{{%message}}', + */ + + /* + // 'po' output format is for saving messages to gettext po files. + 'format' => 'po', + // Root directory containing message translations. + 'messagePath' => __DIR__ . DIRECTORY_SEPARATOR . 'messages', + // Name of the file that will be used for translations. + 'catalog' => 'messages', + */ ]; diff --git a/tests/unit/framework/console/controllers/MessageControllerTest.php b/tests/unit/framework/console/controllers/MessageControllerTest.php index 73df4a98d7..38e3f69dbe 100644 --- a/tests/unit/framework/console/controllers/MessageControllerTest.php +++ b/tests/unit/framework/console/controllers/MessageControllerTest.php @@ -255,13 +255,13 @@ class MessageControllerTest extends TestCase 'messagePath' => $this->messagePath, 'overwrite' => true, ]); - $this->runMessageControllerAction('extract', [$this->configFileName]); + $commandOutput = $this->runMessageControllerAction('extract', [$this->configFileName]); $messages = require($this->messagePath . DIRECTORY_SEPARATOR . $messageFileName); - $this->assertTrue(array_key_exists($newMessage, $messages), 'Unable to add new message!'); - $this->assertTrue(array_key_exists($existingMessage, $messages), 'Unable to keep existing message!'); - $this->assertEquals('', $messages[$newMessage], 'Wrong new message content!'); - $this->assertEquals($existingMessageContent, $messages[$existingMessage], 'Unable to keep existing message content!'); + $this->assertTrue(array_key_exists($newMessage, $messages), 'Unable to add new message: "' . $newMessage . '". Command output was:' . "\n" . $commandOutput); + $this->assertTrue(array_key_exists($existingMessage, $messages), 'Unable to keep existing message: "' . $existingMessage . '". Command output was:' . "\n" . $commandOutput); + $this->assertEquals('', $messages[$newMessage], 'Wrong new message content!. Command output was:\n' . $commandOutput); + $this->assertEquals($existingMessageContent, $messages[$existingMessage], 'Unable to keep existing message content!. Command output was:' . "\n" . $commandOutput); } /**