diff --git a/include/class.export.php b/include/class.export.php index a3335a2e21e33a7a8290358dfdd210f457408451..f112b15a30006c6d5ce79b219f89db7034b649c0 100644 --- a/include/class.export.php +++ b/include/class.export.php @@ -213,7 +213,7 @@ class ResultSetExporter { $this->options = $options; $this->output = $options['output'] ?: fopen('php://output', 'w'); - $this->_res = db_query($sql); + $this->_res = db_query($sql, true, true); if ($row = db_fetch_array($this->_res)) { $query_fields = array_keys($row); $this->headers = array(); @@ -296,9 +296,13 @@ require_once INCLUDE_DIR . 'class.json.php'; require_once INCLUDE_DIR . 'class.migrater.php'; require_once INCLUDE_DIR . 'class.signal.php'; +define('OSTICKET_BACKUP_SIGNATURE', 'osTicket-Backup'); +define('OSTICKET_BACKUP_VERSION', 'B'); + class DatabaseExporter { var $stream; + var $options; var $tables = array(CONFIG_TABLE, SYSLOG_TABLE, FILE_TABLE, FILE_CHUNK_TABLE, STAFF_TABLE, DEPT_TABLE, TOPIC_TABLE, GROUP_TABLE, GROUP_DEPT_TABLE, TEAM_TABLE, TEAM_MEMBER_TABLE, FAQ_TABLE, @@ -311,21 +315,21 @@ class DatabaseExporter { TIMEZONE_TABLE, SESSION_TABLE, PAGE_TABLE, FORM_SEC_TABLE, FORM_FIELD_TABLE, LIST_TABLE, LIST_ITEM_TABLE, FORM_ENTRY_TABLE, FORM_ANSWER_TABLE, USER_TABLE, USER_EMAIL_TABLE, + PLUGIN_TABLE, TICKET_COLLABORATOR_TABLE, + USER_ACCOUNT_TABLE, ORGANIZATION_TABLE, NOTE_TABLE ); - function DatabaseExporter($stream) { + function DatabaseExporter($stream, $options=array()) { $this->stream = $stream; + $this->options = $options; } function write_block($what) { fwrite($this->stream, JsonDataEncoder::encode($what)); - fwrite($this->stream, "\x1e"); + fwrite($this->stream, "\n"); } - function dump($error_stream) { - // Allow plugins to change the tables exported - Signal::send('export.tables', $this, $this->tables); - + function dump_header() { $header = array( array(OSTICKET_BACKUP_SIGNATURE, OSTICKET_BACKUP_VERSION), array( @@ -338,30 +342,32 @@ class DatabaseExporter { ), ); $this->write_block($header); + } + + function dump($error_stream) { + // Allow plugins to change the tables exported + Signal::send('export.tables', $this, $this->tables); + $this->dump_header(); foreach ($this->tables as $t) { if ($error_stream) $error_stream->write("$t\n"); + // Inspect schema - $table = $indexes = array(); - $res = db_query("show columns from $t"); - while ($field = db_fetch_array($res)) + $table = array(); + $res = db_query("select column_name from information_schema.columns + where table_schema=DATABASE() and table_name='$t'"); + while (list($field) = db_fetch_row($res)) $table[] = $field; - $res = db_query("show indexes from $t"); - while ($col = db_fetch_array($res)) - $indexes[] = $col; - - $res = db_query("select * from $t"); - $types = array(); - if (!$table) { if ($error_stream) $error_stream->write( $t.': Cannot export table with no fields'."\n"); die(); } $this->write_block( - array('table', substr($t, strlen(TABLE_PREFIX)), $table, - $indexes)); + array('table', substr($t, strlen(TABLE_PREFIX)), $table)); + + db_query("select * from $t"); // Dump row data while ($row = db_fetch_row($res)) @@ -370,4 +376,32 @@ class DatabaseExporter { $this->write_block(array('end-table')); } } + + function transfer($destination, $query, $callback=false, $options=array()) { + $header_out = false; + $res = db_query($query, true, false); + $i = 0; + while ($row = db_fetch_array($res)) { + if (is_callable($callback)) + $callback($row); + if (!$header_out) { + $fields = array_keys($row); + $this->write_block( + array('table', $destination, $fields, $options)); + $header_out = true; + + } + $this->write_block(array_values($row)); + } + $this->write_block(array('end-table')); + } + + function transfer_array($destination, $array, $keys, $options=array()) { + $this->write_block( + array('table', $destination, $keys, $options)); + foreach ($array as $row) { + $this->write_block(array_values($row)); + } + $this->write_block(array('end-table')); + } } diff --git a/include/i18n/en_US/form.yaml b/include/i18n/en_US/form.yaml index 0d59d1bc67e1c43e09001d55cb220c0399e19838..fe4be2f66098d0c8de9daee25c785ed3d04260f9 100644 --- a/include/i18n/en_US/form.yaml +++ b/include/i18n/en_US/form.yaml @@ -78,7 +78,8 @@ tickets, and will be searchable with advanced search and filterable. deletable: false fields: - - type: text # notrans + - id: 20 + type: text # notrans name: subject # notrans label: Issue Summary required: true @@ -88,7 +89,8 @@ size: 40 length: 50 - - type: thread # notrans + - id: 21 + type: thread # notrans name: message # notrans label: Issue Details hint: Details on the reason(s) for opening the ticket. @@ -96,7 +98,8 @@ edit_mask: 15 sort: 2 - - type: priority # notrans + - id: 22 + type: priority # notrans name: priority # notrans label: Priority Level required: false diff --git a/include/mysqli.php b/include/mysqli.php index 8955ea99f335685bba44f0b3570c7715ab39ee49..e8bfeb32ebcfa7f6acd5fe26991a6fc07b32c136 100644 --- a/include/mysqli.php +++ b/include/mysqli.php @@ -143,12 +143,18 @@ function db_create_database($database, $charset='utf8', * (mixed) MysqliResource if SELECT query succeeds, true if an INSERT, * UPDATE, or DELETE succeeds, false or null if the query fails. */ -function db_query($query, $logError=true) { +function db_query($query, $logError=true, $buffered=true) { global $ost, $__db; + if ($__db->unbuffered_result) { + $__db->unbuffered_result->free(); + $__db->unbuffered_result = false; + } + $tries = 3; do { - $res = $__db->query($query); + $res = $__db->query($query, + $buffered ? MYSQLI_STORE_RESULT : MYSQLI_USE_RESULT); // Retry the query due to deadlock error (#1213) // TODO: Consider retry on #1205 (lock wait timeout exceeded) // TODO: Log warning @@ -164,6 +170,9 @@ function db_query($query, $logError=true) { //echo $msg; #uncomment during debuging or dev. } + if (is_object($res) && !$buffered) + $__db->unbuffered_result = $res; + return $res; } @@ -182,11 +191,13 @@ function db_count($query) { return db_result(db_query($query)); } -function db_result($res, $row=0) { +function db_result($res, $row=false) { if (!$res) return NULL; - $res->data_seek($row); + if ($row !== false) + $res->data_seek($row); + list($value) = db_output($res->fetch_row()); return $value; } diff --git a/include/pear/Mail/RFC822.php b/include/pear/Mail/RFC822.php index 58d36465cba21779887651cec4204f222f9271ec..f22e4ecfe880546e3558479066d1ac497a4ee586 100644 --- a/include/pear/Mail/RFC822.php +++ b/include/pear/Mail/RFC822.php @@ -362,13 +362,17 @@ class Mail_RFC822 { */ function _hasUnclosedQuotes($string) { + $matches = array(); + if (!preg_match_all('/\\|"/S', $string, $matches, PREG_SET_ORDER)) + return false; + $string = trim($string); $iMax = strlen($string); $in_quote = false; $i = $slashes = 0; - for (; $i < $iMax; ++$i) { - switch ($string[$i]) { + foreach ($matches[0] as $m) { + switch ($m) { case '\\': ++$slashes; break; @@ -425,7 +429,7 @@ class Mail_RFC822 { function _hasUnclosedBracketsSub($string, &$num, $char) { $parts = explode($char, $string); - for ($i = 0; $i < count($parts); $i++){ + for ($i = 0, $k = count($parts); $i < $k; $i++){ if (substr($parts[$i], -1) == '\\' || $this->_hasUnclosedQuotes($parts[$i])) $num--; if (isset($parts[$i + 1])) diff --git a/setup/cli/modules/export.php b/setup/cli/modules/export.php index f74647726e0c80ba632de5e61639582f1b01d62e..a6c45fcee9f0f43de791ea17e2745bef70953dc3 100644 --- a/setup/cli/modules/export.php +++ b/setup/cli/modules/export.php @@ -14,9 +14,7 @@ vim: expandtab sw=4 ts=4 sts=4: **********************************************************************/ require_once dirname(__file__) . "/class.module.php"; - -define('OSTICKET_BACKUP_SIGNATURE', 'osTicket-Backup'); -define('OSTICKET_BACKUP_VERSION', 'A'); +require_once dirname(__file__) . "../../cli.inc.php"; class Exporter extends Module { var $prologue = @@ -30,19 +28,63 @@ class Exporter extends Module { 'help'=> "Send zlib compress data to the output stream"), ); + var $arguments = array( + 'module' => array( + 'required' => false, + 'help' => 'Module used for export (see help)' + ), + ); + + var $autohelp = false; + function run($args, $options) { - require_once dirname(__file__) . '/../../../main.inc.php'; + require_once dirname(__file__) . '/../../../bootstrap.php'; require_once INCLUDE_DIR . 'class.export.php'; - global $ost; + if (!$args['module']) { + $exporter = 'DatabaseExporter'; + } + else { + $module = (include dirname(__file__)."/importer/{$args['module']}.php"); + if ($module) { + $module = new $module(); + return $module->_run($args['module']); + } + else { + $this->stderr->write("Unknown importer module given\n"); + $this->showHelp(); + } + } + if ($exporter) + $this->dump($exporter); + } - $stream = $options['stream']; - if ($options['compress']) $stream = "compress.zlib://$stream"; + function dump($module) { + $stream = $this->getOption('stream'); + if ($this->getOption('compress')) $stream = "compress.zlib://$stream"; $stream = fopen($stream, 'w'); - $x = new DatabaseExporter($stream); + $x = new $module($stream, $this->_options); $x->dump($this->stderr); } + + function showHelp() { + $modules = array(); + foreach (glob(dirname(__file__).'/importer/*.php') as $script) { + $info = pathinfo($script); + $modules[] = $info['filename']; + } + + $this->epilog = + "Currently available modules follow. Use 'manage.php export <module> + --help' for usage regarding each respective module:"; + + parent::showHelp(); + + echo "\n"; + foreach ($modules as $name) + echo str_pad($name, 20) . "\n"; + } } Module::register('export', 'Exporter'); diff --git a/setup/cli/modules/file.php b/setup/cli/modules/file.php index 01deb67b54f108526f39957220db0504836c7a01..ee83169958c3e25a9c514e979564b7f6b699d588 100644 --- a/setup/cli/modules/file.php +++ b/setup/cli/modules/file.php @@ -11,9 +11,12 @@ class FileManager extends Module { 'options' => array( 'list' => 'List files matching criteria', 'export' => 'Export files from the system', + 'import' => 'Load files exported via `export`', 'dump' => 'Dump file content to stdout', + 'load' => 'Load file contents from stdin', 'migrate' => 'Migrate a file to another backend', 'backends' => 'List configured storage backends', + 'expunge' => 'Remove matching files from the system', ), ), ); @@ -21,7 +24,7 @@ class FileManager extends Module { var $options = array( 'ticket' => array('-T', '--ticket', 'metavar'=>'id', 'help' => 'Search by internal ticket id'), - 'file' => array('-F', '--file', 'metavar'=>'id', + 'file-id' => array('-F', '--file-id', 'metavar'=>'id', 'help' => 'Search by file id'), 'name' => array('-N', '--name', 'metavar'=>'name', 'help' => 'Search by file name (subsring match)'), @@ -42,6 +45,9 @@ class FileManager extends Module { 'help' => 'Target backend for migration. See `backends` action for a list of available backends'), + 'file' => array('-f', '--file', 'metavar'=>'FILE', + 'help' => 'Filename used for import and export'), + 'verbose' => array('-v', '--verbose', 'action'=>'store_true', 'help' => 'Be more verbose'), ); @@ -65,8 +71,11 @@ class FileManager extends Module { $files = FileModel::objects(); $this->_applyCriteria($options, $files); foreach ($files as $f) { - printf("% 5d %s % 8d %s % 12s %s\n", $f->id, $f->bk, + printf("% 5d %s % 8d %s % 16s %s\n", $f->id, $f->bk, $f->size, $f->created, $f->type, $f->name); + if ($f->attrs) { + printf(" %s\n", $f->attrs); + } } break; @@ -81,6 +90,62 @@ class FileManager extends Module { $bk->passthru(); break; + case 'load': + // Load file content from STDIN + $files = FileModel::objects(); + $this->_applyCriteria($options, $files); + if ($files->count() != 1) + $this->fail('Criteria must select exactly 1 file'); + + $f = AttachmentFile::lookup($files[0]->id); + try { + if ($bk = $f->open()) + $bk->unlink(); + } + catch (Exception $e) {} + + if ($options['to']) + $bk = FileStorageBackend::lookup($options['to'], $f); + else + // Use the system default + $bk = AttachmentFile::getBackendForFile($f); + + $type = false; + $signature = ''; + $finfo = new finfo(FILEINFO_MIME_TYPE); + if ($options['file'] && $options['file'] != '-') { + if (!file_exists($options['file'])) + $this->fail($options['file'].': Cannot open file'); + if (!$bk->upload($options['file'])) + $this->fail('Unable to upload file contents to backend'); + $type = $finfo->file($options['file']); + list(, $signature) = AttachmentFile::_getKeyAndHash($options['file'], true); + } + else { + $stream = fopen('php://stdin', 'rb'); + while ($block = fread($stream, $bk->getBlockSize())) { + if (!$bk->write($block)) + $this->fail('Unable to send file contents to backend'); + if (!$type) + $type = $finfo->buffer($block); + } + if (!$bk->flush()) + $this->fail('Unable to commit file contents to backend'); + } + + // TODO: Update file metadata + $sql = 'UPDATE '.FILE_TABLE.' SET bk='.db_input($bk->getBkChar()) + .', created=CURRENT_TIMESTAMP' + .', type='.db_input($type) + .', signature='.db_input($signature) + .' WHERE id='.db_input($f->getId()); + + if (!db_query($sql) || db_affected_rows()!=1) + $this->fail('Unable to update file metadata'); + + $this->stdout->write("Successfully saved contents\n"); + break; + case 'migrate': if (!$options['to']) $this->fail('Please specify a target backend for migration'); @@ -110,9 +175,46 @@ class FileManager extends Module { } $this->stdout->write("Migrated $count files\n"); break; - } + case 'export': + // Create a temporary ZIP file + $files = FileModel::objects(); + $this->_applyCriteria($options, $files); + + if (!$options['file']) + $this->fail('Please specify zip file with `-f`'); + + $zip = new ZipArchive(); + if (true !== ($reason = $zip->open($options['file'], + ZipArchive::CREATE))) + $this->fail($reason.': Unable to create zip file'); + + $manifest = array(); + foreach ($files as $m) { + $f = AttachmentFile::lookup($m->id); + $zip->addFromString($f->getId(), $f->getData()); + $zip->setCommentName($f->getId(), $f->getName()); + // TODO: Log %attachment and %ticket_attachment entries + $info = array('file' => $f->getInfo()); + foreach ($m->tickets as $t) + $info['tickets'][] = $t->ht; + $manifest[$f->getId()] = $info; + } + $zip->addFromString('MANIFEST', serialize($manifest)); + $zip->close(); + break; + + case 'expunge': + // Create a temporary ZIP file + $files = FileModel::objects(); + $this->_applyCriteria($options, $files); + + foreach ($files as $f) { + $f->tickets->expunge(); + $f->unlink() && $f->delete(); + } + } } function _applyCriteria($options, $qs) { @@ -122,7 +224,7 @@ class FileManager extends Module { case 'ticket': $qs->filter(array('tickets__ticket_id'=>$val)); break; - case 'file': + case 'file-id': $qs->filter(array('id'=>$val)); break; case 'name': @@ -191,5 +293,12 @@ class TicketAttachmentModel extends VerySimpleModel { ); } +class AttachmentModel extends VerySimpleModel { + static $meta = array( + 'table' => ATTACHMENT_TABLE, + 'pk' => array('object_id', 'type', 'file_id'), + ); +} + Module::register('file', 'FileManager'); ?> diff --git a/setup/cli/modules/import.php b/setup/cli/modules/import.php index 84c27396d4b790c4a5612c2146490e5da42d9090..68979f330cc8f5f959190448d0f639af4b9a0d03 100644 --- a/setup/cli/modules/import.php +++ b/setup/cli/modules/import.php @@ -14,6 +14,9 @@ vim: expandtab sw=4 ts=4 sts=4: **********************************************************************/ require_once dirname(__file__) . "/class.module.php"; +require_once dirname(__file__) . "../../cli.inc.php"; +require_once INCLUDE_DIR . "class.export.php"; +require_once INCLUDE_DIR . 'class.json.php'; class Importer extends Module { var $prologue = @@ -33,6 +36,8 @@ class Importer extends Module { tables. This option can be specified more than once"), 'drop' => array('-D', '--drop', 'action'=>'store_true', 'help'=> 'Issue DROP TABLE statements before the create statemente'), + 'go-baby' => array('-P', '--prime-time', 'action'=>'store_true', + 'help'=>'Load data into live database. Use at your own risk'), ); var $epilog = @@ -41,11 +46,46 @@ class Importer extends Module { var $stream; var $header; var $source_ost_info; + var $parent; + + function __construct($parent=false) { + parent::__construct(); + if ($parent) { + $this->parent = $parent; + $this->stream = $parent->stream; + } + } + function __get($what) { + return $this->parent->{$what}; + } + function __call($what, $how) { + return call_user_func_array(array($this->parent, $what), $how); + } + + function run($args, $options) { + $stream = $options['stream']; + if ($options['compress']) $stream = "compress.zlib://$stream"; + if (!($this->stream = fopen($stream, 'rb'))) { + $this->stderr->write('Unable to open input stream'); + die(); + } + + if (!$this->verify_header()) + die('Unable to verify backup header'); + + Bootstrap::connect(); + + $class = 'Importer_' . $this->header[1]; + $importer = new $class($this); + + while ($importer->import_table()); + @fclose($this->stream); + } function verify_header() { list($header, $info) = $this->read_block(); if (!$header || $header[0] != OSTICKET_BACKUP_SIGNATURE) { - $this->stderr->write('Header mismatch -- not an osTicket backup'); + $this->stderr->write("Header mismatch -- not an osTicket backup\n"); return false; } else @@ -60,9 +100,7 @@ class Importer extends Module { } function read_block() { - $block = ''; - while (!feof($this->stream) && (($c = fgetc($this->stream)) != "\x1e")) - $block .= $c; + $block = fgets($this->stream); if ($json = JsonDataParser::decode($block)) return $json; @@ -73,6 +111,37 @@ class Importer extends Module { } } + function load_row($info, $row, $flush=false) { + static $header = null; + static $rows = array(); + static $length = 0; + + if ($info && $header === null) { + $header = "INSERT INTO `".TABLE_PREFIX.$info[1].'` ('; + $cols = array(); + foreach ($info[2] as $col) + $cols[] = "`{$col['Field']}`"; + $header .= implode(', ', $cols); + $header .= ") VALUES "; + } + if ($row) { + $values = array(); + foreach ($info[2] as $i=>$col) + $values[] = (is_numeric($row[$i])) + ? $row[$i] + : ($row[$i] ? '0x'.bin2hex($row[$i]) : "''"); + $values = "(" . implode(', ', $values) . ")"; + $length += strlen($values); + $rows[] = &$values; + } + if (($flush || $length > 16000) && $header) { + $this->send_statement($header . implode(',', $rows)); + $header = null; + $rows = array(); + $length = 0; + } + } + function send_statement($stmt) { if ($this->getOption('prime-time')) db_query($stmt); @@ -81,6 +150,9 @@ class Importer extends Module { $this->stdout->write(";\n"); } } +} + +class Importer_A extends Importer { function import_table() { if (!($header = $this->read_block())) @@ -173,18 +245,44 @@ class Importer extends Module { .')'); } } +} +class Importer_B extends Importer { function truncate_table($info) { $this->send_statement('TRUNCATE TABLE '.TABLE_PREFIX.$info[1]); - $indexes = array(); - foreach ($info[3] as $idx) { - if ($idx['Key_name'] == 'PRIMARY') - continue; - $indexes[$idx['Key_name']] = - '`'.TABLE_PREFIX.$info[1].'`.`'.$idx['Key_name'].'`'; + } + + function import_table() { + if (!($header = $this->read_block())) + return false; + + else if ($header[0] != 'table') { + $this->stderr->write('Unable to read table header'); + return false; } - foreach ($indexes as $T=>$fqn) - $this->send_statement('DROP INDEX IF EXISTS '.$fqn); + + // TODO: Consider included tables and excluded tables + + $this->stderr->write("Importing table: {$header[1]}\n"); + + $res = db_query("select column_name from information_schema.columns + where table_schema=DATABASE() and table_name='$t'"); + while (list($field) = db_fetch_row($res)) + if (!in_array($field, $header[2])) + $this->fail($header[1] + .": Destination table does not have the `$field` field"); + + if (!isset($header[3]['truncate']) || $header[3]['truncate']) + $this->truncate_table($header); + + while (($row=$this->read_block())) { + if (isset($row[0]) && ($row[0] == 'end-table')) { + $this->load_row(null, null, true); + return true; + } + $this->load_row($header, $row); + } + return false; } function load_row($info, $row, $flush=false) { @@ -196,7 +294,7 @@ class Importer extends Module { $header = "INSERT INTO `".TABLE_PREFIX.$info[1].'` ('; $cols = array(); foreach ($info[2] as $col) - $cols[] = "`{$col['Field']}`"; + $cols[] = "`$col`"; $header .= implode(', ', $cols); $header .= ") VALUES "; } @@ -218,23 +316,6 @@ class Importer extends Module { } } - function run($args, $options) { - require_once dirname(__file__) . '/../../../main.inc.php'; - require_once INCLUDE_DIR . 'class.json.php'; - - $stream = $options['stream']; - if ($options['compress']) $stream = "compress.zlib://$stream"; - if (!($this->stream = fopen($stream, 'rb'))) { - $this->stderr->write('Unable to open input stream'); - die(); - } - - if (!$this->verify_header()) - die('Unable to verify backup header'); - - while ($this->import_table()); - @fclose($this->stream); - } } Module::register('import', 'Importer'); diff --git a/setup/inc/class.installer.php b/setup/inc/class.installer.php index 31213d4c678fcaacc6f720fc42652727d4e0be00..c008566f5546d829ce5375bdc69a0d6cd7c0bf2a 100644 --- a/setup/inc/class.installer.php +++ b/setup/inc/class.installer.php @@ -155,19 +155,19 @@ class Installer extends SetupWizard { $i18n->loadDefaultData(); $sql='SELECT `id` FROM '.PREFIX.'sla ORDER BY `id` LIMIT 1'; - $sla_id_1 = db_result(db_query($sql, false), 0); + $sla_id_1 = db_result(db_query($sql, false)); $sql='SELECT `dept_id` FROM '.PREFIX.'department ORDER BY `dept_id` LIMIT 1'; - $dept_id_1 = db_result(db_query($sql, false), 0); + $dept_id_1 = db_result(db_query($sql, false)); $sql='SELECT `tpl_id` FROM '.PREFIX.'email_template_group ORDER BY `tpl_id` LIMIT 1'; - $template_id_1 = db_result(db_query($sql, false), 0); + $template_id_1 = db_result(db_query($sql, false)); $sql='SELECT `group_id` FROM '.PREFIX.'groups ORDER BY `group_id` LIMIT 1'; - $group_id_1 = db_result(db_query($sql, false), 0); + $group_id_1 = db_result(db_query($sql, false)); $sql='SELECT `value` FROM '.PREFIX.'config WHERE namespace=\'core\' and `key`=\'default_timezone_id\' LIMIT 1'; - $default_timezone = db_result(db_query($sql, false), 0); + $default_timezone = db_result(db_query($sql, false)); //Create admin user. $sql='INSERT INTO '.PREFIX.'staff SET created=NOW() ' @@ -194,7 +194,7 @@ class Installer extends SetupWizard { $sql='SELECT `email_id` FROM '.PREFIX."email WHERE `email`='alerts@$domain' LIMIT 1"; - $alert_email_id = db_result(db_query($sql, false), 0); + $alert_email_id = db_result(db_query($sql, false)); //Create config settings---default settings! //XXX: rename ostversion helpdesk_* ??