Newer
Older
<?php
require_once dirname(__file__) . "/class.module.php";
require_once dirname(__file__) . "/../cli.inc.php";
require_once INCLUDE_DIR . 'class.format.php';
class i18n_Compiler extends Module {
var $prologue = "Manages translation files from Crowdin";
var $arguments = array(
"command" => array(
'help' => "Action to be performed.",
"options" => array(
'list' => 'Show list of available translations',
'build' => 'Compile a language pack',
'make-pot' => 'Build the PO file for gettext translations',
'sign' => 'Sign a language pack',
);
var $options = array(
"key" => array('-k','--key','metavar'=>'API-KEY',
'help'=>'Crowdin project API key. This can be omitted if
CROWDIN_API_KEY is defined in the ost-config.php file'),
"lang" => array('-L', '--lang', 'metavar'=>'code',
'help'=>'Language code (used for building)'),
'file' => array('-f', '--file', 'metavar'=>'FILE',
'help' => "Language pack to be signed"),
'pkey' => array('-P', '--pkey', 'metavar'=>'key-file',
'help' => 'Private key for signing'),
'root' => array('-R', '--root', 'matavar'=>'path',
'help' => 'Specify a root folder for `make-pot`'),
'domain' => array('-D', '--domain', 'metavar'=>'name',
'default' => '',
'help' => 'Add a domain to the path/context of PO strings'),
static $project = 'osticket-official';
static $crowdin_api_url = 'http://i18n.osticket.com/api/project/{project}/{command}';
$this->stdout->write(">>> Downloading $url\n");
#curl post
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_USERAGENT, 'osTicket/'.THIS_VERSION);
curl_setopt($ch, CURLOPT_HEADER, FALSE);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, FALSE);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
$result=curl_exec($ch);
$code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);
return array($code, $result);
}
function _request($command, $args=array()) {
$url = str_replace(array('{command}', '{project}'),
array($command, self::$project),
self::$crowdin_api_url);
$args += array('key' => $this->key);
foreach ($args as &$a)
$a = urlencode($a);
unset($a);
$url .= '?' . Format::array_implode('=', '&', $args);
return $this->_http_get($url);
}
function run($args, $options) {
$this->key = $options['key'];
if (!$this->key && defined('CROWDIN_API_KEY'))
$this->key = CROWDIN_API_KEY;
function get_osticket_root_path() { return ROOT_DIR; }
require_once(ROOT_DIR.'setup/test/tests/class.test.php');
switch (strtolower($args['command'])) {
case 'list':
if (!$this->key)
$this->fail('API key is required');
$this->_list();
break;
case 'build':
if (!$this->key)
$this->fail('API key is required');
if (!$options['lang'])
$this->fail('Language code is required. See `list`');
$this->_build($options['lang']);
break;
case 'sign':
if (!$options['file'] || !file_exists($options['file']))
$this->fail('Specify a language pack to sign with --file=');
$this->_sign($options['file'], $options);
break;
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
}
}
function _list() {
error_reporting(E_ALL);
list($code, $body) = $this->_request('status');
$d = new DOMDocument();
$d->loadXML($body);
$xp = new DOMXpath($d);
foreach ($xp->query('//language') as $c) {
$name = $code = '';
foreach ($c->childNodes as $n) {
switch (strtolower($n->nodeName)) {
case 'name':
$name = $n->textContent;
break;
case 'code':
$code = $n->textContent;
break;
}
}
if (!$code)
continue;
$this->stdout->write(sprintf("%s (%s)\n", $code, $name));
}
}
function _build($lang) {
list($code, $zip) = $this->_request("download/$lang.zip");
if ($code !== 200)
$this->fail('Language is not available'."\n");
$temp = tempnam('/tmp', 'osticket-cli');
$f = fopen($temp, 'w');
fwrite($f, $zip);
fclose($f);
$zip = new ZipArchive();
$zip->open($temp);
unlink($temp);
$lang = str_replace('-','_',$lang);
@unlink(I18N_DIR."$lang.phar");
$phar = new Phar(I18N_DIR."$lang.phar");
$phar->startBuffering();
for ($i=0; $i<$zip->numFiles; $i++) {
$info = $zip->statIndex($i);
if (!$contents)
continue;
if (strpos($info['name'], '/messages.po') !== false) {
$po_file = $contents;
// Don't add the PO file as-is to the PHAR file
continue;
}
$phar->addFromString($info['name'], $contents);
}
// TODO: Add i18n extras (like fonts)
$langs = array($lang);
if (strpos($lang, '_') !== false) {
@list($short) = explode('_', $lang);
$langs[] = $short;
}
foreach ($langs as $l) {
list($code, $js) = $this->_http_get(
'http://imperavi.com/webdownload/redactor/lang/?lang='
.strtolower($l));
if ($code == 200 && ($js != 'File not found')) {
$phar->addFromString('js/redactor.js', $js);
break;
}
}
if ($code != 200)
$this->stderr->write($lang . ": Unable to fetch Redactor language file\n");
// JQuery UI Datepicker
// http://jquery-ui.googlecode.com/svn/tags/latest/ui/i18n/jquery.ui.datepicker-de.js
foreach ($langs as $l) {
list($code, $js) = $this->_http_get(
'http://jquery-ui.googlecode.com/svn/tags/latest/ui/i18n/jquery.ui.datepicker-'
.str_replace('_','-',$l).'.js');
// If locale-specific version is not available, use the base
// language version (de if de_CH is not available)
if ($code == 200)
break;
}
if ($code == 200)
$phar->addFromString('js/jquery.ui.datepicker.js', $js);
else
$this->stderr->write(str_replace('_','-',$lang)
.": Unable to fetch jQuery UI Datepicker locale file\n");
// True type fonts for PDF printing
$langs = (include I18N_DIR . 'langs.php');
$info = $langs[$lang];
if (isset($info['fonts'])) {
foreach ($info['fonts'] as $name => $types) {
foreach ($types as $code => $fullname) {
list($ttf, $url) = $fullname;
if (!$url)
continue;
list($code, $file) = $this->_http_get($url);
if ($code == 200)
$phar->addFromString('fonts/'.$ttf, $file);
else
$this->stderr->write(
"*** Unable to fetch $url\n");
}
}
}
if ($po_file) {
$pipes = array();
$msgfmt = proc_open('msgfmt -o- -',
array(0=>array('pipe','r'), 1=>array('pipe','w')),
$pipes);
if (is_resource($msgfmt)) {
fwrite($pipes[0], $po_file);
fclose($pipes[0]);
$mo_input = fopen('php://temp', 'r+b');
fwrite($mo_input, stream_get_contents($pipes[1]));
rewind($mo_input);
require_once INCLUDE_DIR . 'class.translation.php';
$mo = Translation::buildHashFile($mo_input, false, true);
$phar->addFromString('LC_MESSAGES/messages.mo.php', $mo);
fclose($mo_input);
}
}
// Add in translation of javascript strings
if ($mo && ($js = $this->__getAllJsPhrases())) {
$mo = (eval (substr($mo, 5))); # Chop off <?php
foreach ($js as $c) {
foreach ($c['forms'] as $f) {
$phrases[$f] = @$mo[$f] ?: $f;
}
}
$phar->addFromString(
'js/osticket-strings.js',
sprintf('(function($){$.oststrings=%s;})(jQuery);',
JsonDataEncoder::encode($phrases))
);
list($code, $zip) = $this->_request("download/$lang.zip");
// Include a manifest
include_once INCLUDE_DIR . 'class.mailfetch.php';
$po_header = Mail_Parse::splitHeaders($mo['']);
$info = array(
'Build-Date' => date(DATE_RFC822),
'Build-Version' => trim(`git describe`),
'Language' => $po_header['Language'],
#'Phrases' =>
#'Translated' =>
#'Approved' =>
'Id' => 'lang:' . $lang,
'Last-Revision' => $po_header['PO-Revision-Date'],
'Version' => (int)(strtotime($po_header['PO-Revision-Date']) / 10000),
);
$phar->addFromString(
'MANIFEST.php',
sprintf('<?php return %s;', var_export($info, true)));
// Use a very small stub
$phar->setStub('<?php __HALT_COMPILER();');
$phar->setSignatureAlgorithm(Phar::SHA1);
$phar->stopBuffering();
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
function _sign($plugin, $options) {
if (!file_exists($plugin))
$this->fail($plugin.': Cannot find file');
elseif (!file_exists("phar://$plugin/MANIFEST.php"))
$this->fail($plugin.': Should be a plugin PHAR file');
$info = (include "phar://$plugin/MANIFEST.php");
$phar = new Phar($plugin);
if (!function_exists('openssl_get_privatekey'))
$this->fail('OpenSSL extension required for signing');
$private = openssl_get_privatekey(
file_get_contents($options['pkey']));
if (!$private)
$this->fail('Unable to read private key');
$signature = $phar->getSignature();
$seal = '';
openssl_sign($signature['hash'], $seal, $private,
OPENSSL_ALGO_SHA1);
if (!$seal)
$this->fail('Unable to generate verify signature');
$this->stdout->write(sprintf("Signature: %s\n",
strtolower($signature['hash'])));
$this->stdout->write(
sprintf("Seal: \"v=1; i=%s; s=%s; V=%s;\"\n",
$info['Id'], base64_encode($seal), $info['Version']));
}
function __read_next_string($tokens) {
$string = array();
while (list(,$T) = each($tokens)) {
switch ($T[0]) {
case T_CONSTANT_ENCAPSED_STRING:
// Strip leading and trailing ' and " chars
$string['form'] = preg_replace(array("`^{$T[1][0]}`","`{$T[1][0]}$`"),array("",""), $T[1]);
$string['line'] = $T[2];
break;
case T_DOC_COMMENT:
case T_COMMENT:
switch ($T[1][0]) {
case '/':
if ($T[1][1] == '*')
$text = trim($T[1], '/* ');
else
$text = ltrim($T[1], '/ ');
break;
case '#':
$text = ltrim($T[1], '# ');
}
$string['comments'][] = $text;
break;
case T_WHITESPACE:
// noop
continue;
case T_STRING_VARNAME:
case T_NUM_STRING:
case T_ENCAPSED_AND_WHITESPACE:
case '.':
$string['constant'] = false;
break;
case '[':
// Not intended to be translated — array index
return null;
default:
return array($string, $T);
}
}
}
function __read_args($tokens, $proto=false) {
$args = array('forms'=>array());
$arg = null;
$proto = $proto ?: array('forms'=>1);
while (list($string,$T) = $this->__read_next_string($tokens)) {
// Add context and forms
if (isset($proto['context']) && !isset($args['context'])) {
$args['context'] = $string['form'];
}
elseif (count($args['forms']) < $proto['forms'] && $string) {
if (isset($string['constant']) && !$string['constant']) {
throw new Exception($string['form'] . ': Untranslatable string');
}
$args['forms'][] = $string['form'];
}
elseif ($string) {
$this->stderr->write(sprintf("%s: %s: Too many arguments\n",
$string['line'] ?: '?', $string['form']));
}
// Add usage and comment info
if (!isset($args['line']) && isset($string['line']))
$args['line'] = $string['line'];
if (isset($string['comments']))
$args['comments'] = array_merge(
@$args['comments'] ?: array(), $string['comments']);
// Handle the terminating token from ::__read_next_string()
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
switch ($T[0]) {
case ')':
return $args;
}
}
}
function __get_func_args($tokens, $args) {
while (list(,$T) = each($tokens)) {
switch ($T[0]) {
case T_WHITESPACE:
continue;
case '(':
return $this->__read_args($tokens, $args);
default:
// Not a function call
return false;
}
}
}
function __find_strings($tokens, $funcs, $parens=0) {
$T_funcs = array();
$funcdef = false;
while (list(,$T) = each($tokens)) {
switch ($T[0]) {
case T_STRING:
if ($T[1] == 'sprintf') {
foreach ($this->__find_strings($tokens, $funcs) as $i=>$f) {
// Only the first on gets the php-format flag
if ($i == 0)
$f['flags'] = array('php-format');
$T_funcs[] = $f;
}
break;
}
if (!isset($funcs[$T[1]]))
continue;
$constants = $funcs[$T[1]];
if ($info = $this->__get_func_args($tokens, $constants))
$T_funcs[] = $info;
break;
case T_COMMENT:
case T_DOC_COMMENT:
$translate = false;
$hints = array();
if (preg_match('`^/\*+\s*@(\w+)`m', $T[1])) {
foreach (preg_split('`,\s*`m', $T[1]) as $command) {
$command = trim($command, " \n\r\t\"*/\\");
@list($command, $args) = explode(' ', $command, 2);
switch ($command) {
case '@context':
$hints['context'] = trim($args, " \"*\n\t\r");
case '@trans':
$translate = true;
default:
continue;
}
}
}
if ($translate) {
// Find the next textual token
list($S, $T) = $this->__read_next_string($tokens);
$string = array('forms'=>array($S['form']), 'line'=>$S['line'])
+ $hints;
$string['comments'] = $S['comments'];
$T_funcs[] = $string;
}
break;
// Track function definitions of the gettext functions
case T_FUNCTION:
$funcdef = true;
break;
case '{';
$funcdef = false;
case '(':
$parens++;
break;
case ')':
// End of scope?
if (--$parens == 0)
return $T_funcs;
}
}
return $T_funcs;
}
function __write_string($string) {
// Unescape single quote (') and escape unescaped double quotes (")
$string = preg_replace(array("`\\\(['$])`", '`(?<!\\\)"`'), array("$1", '\"'), $string);
// Preserve embedded newlines -- preserve up to on
$string = preg_replace("`\n`u", "\\n\n", $string);
// Word-wrap long lines
$string = rtrim(preg_replace('/(?=[\s\p{Ps}])(.{1,76})(\s|$|(\p{Ps}))/uS',
"$1$2\n", $string), "\n");
$strings = array_filter(explode("\n", $string));
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
if (count($strings) > 1)
array_unshift($strings, "");
foreach ($strings as $line) {
print "\"{$line}\"\n";
}
}
function __write_pot_header() {
$lines = array(
'msgid ""',
'msgstr ""',
'"Project-Id-Version: osTicket '.trim(`git describe`).'\n"',
'"POT-Create-Date: '.date('Y-m-d H:i O').'\n"',
'"Report-Msgid-Bugs-To: support@osticket.com\n"',
'"Language: en_US\n"',
'"MIME-Version: 1.0\n"',
'"Content-Type: text/plain; charset=UTF-8\n"',
'"Content-Transfer-Encoding: 8bit\n"',
'"X-Generator: osTicket i18n CLI\n"',
);
print implode("\n", $lines);
print "\n";
}
function __write_pot($strings) {
$this->__write_pot_header();
foreach ($strings as $S) {
print "\n";
if ($c = @$S['comments']) {
foreach ($c as $comment) {
foreach (explode("\n", $comment) as $line) {
if ($line = trim($line))
print "#. {$line}\n";
}
}
}
foreach ($S['usage'] as $ref) {
print "#: ".$ref."\n";
}
if ($f = @$S['flags']) {
print "#, ".implode(', ', $f)."\n";
}
if (isset($S['context'])) {
print "msgctxt ";
$this->__write_string($S['context']);
}
print "msgid ";
$this->__write_string($S['forms'][0]);
if (count($S['forms']) == 2) {
print "msgid_plural ";
$this->__write_string($S['forms'][1]);
print 'msgstr[0] ""'."\n";
print 'msgstr[1] ""'."\n";
}
else {
print 'msgstr ""'."\n";
}
}
}
$funcs = array(
'__' => array('forms'=>1),
'_S' => array('forms'=>1),
'_N' => array('forms'=>2),
'_NS' => array('forms'=>2),
'_P' => array('context'=>1, 'forms'=>1),
'_NP' => array('context'=>1, 'forms'=>2),
// This is an error
'_' => array('forms'=>0),
$root = realpath($options['root'] ?: ROOT_DIR);
$domain = $options['domain'] ? '('.$options['domain'].')/' : '';
$files = Test::getAllScripts(true, $root);
$strings = array();
foreach ($files as $f) {
$F = str_replace($root.'/', $domain, $f);
$this->stderr->write("$F\n");
$tokens = new ArrayObject(token_get_all(fread(fopen($f, 'r'), filesize($f))));
foreach ($this->__find_strings($tokens, $funcs, 1) as $call) {
self::__addString($strings, $call, $F);
$strings = array_merge($strings, $this->__getAllJsPhrases($root));
$this->__write_pot($strings);
}
static function __addString(&$strings, $call, $file=false) {
if (!($forms = @$call['forms']))
// Transation of non-constant
return;
$primary = $forms[0];
// Normalize the $primary string
$primary = preg_replace(array("`\\\(['$])`", '`(?<!\\\)"`'), array("$1", '\"'), $primary);
if (isset($call['context']))
$primary = $call['context'] . "\x04" . $primary;
if (!isset($strings[$primary])) {
$strings[$primary] = array('forms' => $forms);
}
$E = &$strings[$primary];
if (isset($call['line']) && $file)
$E['usage'][] = "{$file}:{$call['line']}";
if (isset($call['flags']))
$E['flags'] = array_unique(array_merge(@$E['flags'] ?: array(), $call['flags']));
if (isset($call['comments']))
$E['comments'] = array_merge(@$E['comments'] ?: array(), $call['comments']);
if (isset($call['context']))
$E['context'] = $call['context'];
function __getAllJsPhrases($root=ROOT_DIR) {
$root = rtrim($root, '/') . '/';
$funcs = array('__'=>array('forms'=>1));
foreach (glob_recursive($root . "*.js") as $s) {
$this->stderr->write($s."\n");
$calls = array();
preg_match_all('/(?:function\s+)?__\(\s*[^\'"]*(([\'"])(?:(?<!\\\\)\2|.)+\2)\s*[^)]*\)/',
$script, $calls, PREG_OFFSET_CAPTURE | PREG_SET_ORDER);
foreach ($calls as $c) {
if (!($call = $this->__find_strings(token_get_all('<?php '.$c[0][0]), $funcs, 0)))
continue;
$call = $call[0];
list($lhs) = str_split($script, $c[1][1]);
$call['line'] = strlen($lhs) - strlen(str_replace("\n", "", $lhs)) + 1;
self::__addString($strings, $call, $s);
}
}
return $strings;
}
}
Module::register('i18n', 'i18n_Compiler');
?>