file: Add import/export support

This patch adds an export+import method to the file command line module. The
file export is agnostic of the file storage backend. It writes the files
with the %file record data to standard-out or to the file given by the -f
option. The file is a considered a stream and can quietly be compressed and
can be read back in as a stream (no seeking is performed in the import
process).

include/class.orm.php

@@ -206,9 +206,9 @@ class VerySimpleModel {
         foreach ($this->dirty as $field=>$old) {
             if ($this->__new__ or !in_array($field, $pk)) {
                 if (@get_class($this->get($field)) == 'SqlFunction')
-                    $fields[] = $field.' = '.$this->get($field)->toSql();
+                    $fields[] = "`$field` = ".$this->get($field)->toSql();
                 else
-                    $fields[] = $field.' = '.db_input($this->get($field));
+                    $fields[] = "`$field` = ".db_input($this->get($field));
             }
         }
         $sql .= ' SET '.implode(', ', $fields);

setup/cli/modules/file.php

@@ -176,37 +176,197 @@ class FileManager extends Module {
             $this->stdout->write("Migrated $count files\n");
             break;
 
+        /**
+         * export
+         *
+         * Export file contents to a stream file. The format of the stream
+         * will be a continuous stream of file information in the following
+         * format:
+         *
+         * FILE<header-length><data-length><header><data>EOF\x1c
+         *
+         * Where
+         *   "FILE"         is the literal text 'FILE'
+         *   header-length  is 'V' packed header length (bytes)
+         *   data-length    is 'V' packed data length (bytes)
+         *   header         is the file record, php serialized
+         *   data           is the raw content of the file
+         *   "EOF"          is the literal text 'EOF'
+         *   \x1c           is an ASCII 0x1c byte (file separator)
+         *
+         * Options:
+         * --file       File to which to direct the stream output, default
+         *              is stdout
+         */
         case 'export':
-            // Create a temporary ZIP file
             $files = FileModel::objects();
             $this->_applyCriteria($options, $files);
 
-            if (!$options['file'])
-                $this->fail('Please specify zip file with `-f`');
+            if (!$options['file'] || $options['file'] == '-')
+                $options['file'] = 'php://stdout';
 
-            $zip = new ZipArchive();
-            if (true !== ($reason = $zip->open($options['file'],
-                    ZipArchive::CREATE)))
-                $this->fail($reason.': Unable to create zip file');
+            if (!($stream = fopen($options['file'], 'wb')))
+                $this->fail($options['file'].': Unable to open file for export stream');
 
-            $manifest = array();
             foreach ($files as $m) {
                 $f = AttachmentFile::lookup($m->id);
-                $zip->addFromString($f->getId(), $f->getData());
-                $zip->setCommentName($f->getId(), $f->getName());
                 // TODO: Log %attachment and %ticket_attachment entries
                 $info = array('file' => $f->getInfo());
                 foreach ($m->tickets as $t)
                     $info['tickets'][] = $t->ht;
+                $header = serialize($info);
+                fwrite($stream, 'FILE'.pack('VV', strlen($header), $f->getSize()));
+                fwrite($stream, $header);
+                $FS = $f->open();
+                while ($block = $FS->read())
+                    fwrite($stream, $block);
+                fwrite($stream, "EOF\x1c");
+            }
+            fclose($stream);
+            break;
+
+        /**
+         * import
+         *
+         * Import a collection of file contents exported by the `export`.
+         * See the export function above for details about the stream
+         * format.
+         *
+         * Options:
+         * --file       File to which to direct the stream output, default
+         *              is stdin
+         * --to         Backend to receive the contents (@see `backends`)
+         * --verbose    Show file names while importing
+         */
+        case 'import':
+            if (!$options['file'] || $options['file'] == '-')
+                $options['file'] = 'php://stdin';
+
+            if (!($stream = fopen($options['file'], 'wb')))
+                $this->fail($options['file'].': Unable to open file for export stream');
+
+            while (true) {
+                // Read the file header
+                $header = fread($stream, 12);
+                if (!$header)
+                    // EOF
+                    break;
+                list(, $mark, $hlen, $dlen) = unpack('V3', $header);
+
+                // FILE written as little-endian 4-byte int is 0x454c4946 (ELIF)
+                if ($mark != 0x454c4946)
+                    $this->fail('Bad file record');
+
+                // Read the header
+                $header = fread($stream, $hlen);
+                if (strlen($header) != $hlen)
+                    $this->fail('Short read getting header info');
+
+                $header = unserialize($header);
+                if (!$header)
+                    $this->fail('Unable to decipher file header');
+
+                // Find or create the file record
+                $finfo = $header['file'];
+                $f = AttachmentFile::lookup($finfo['id']);
+                if ($f) {
+                    // Verify file information
+                    if ($f->getSize() != $finfo['size']
+                        || $f->getSignature() != $finfo['signature']
+                    ) {
+                        $this->fail(sprintf(
+                            '%s: File data does not match existing file record',
+                            $finfo['name']
+                        ));
+                    }
+                    // Drop existing file contents, if any
+                    try {
+                        if ($bk = $f->open())
+                            $bk->unlink();
+                    }
+                    catch (Exception $e) {}
+                }
+                // Create a new file
+                else {
+                    $fm = FileModel::create($finfo);
+                    if (!$fm->save() || !($f = AttachmentFile::lookup($fm->id))) {
+                        $this->fail(sprintf(
+                            '%s: Unable to create new file record',
+                            $finfo['name']));
+                    }
+                }
+
+                // Determine the backend to recieve the file contents
+                if ($options['to']) {
+                    $bk = FileStorageBackend::lookup($options['to'], $f);
+                }
+                // Use the system default
+                else {
+                    $bk = AttachmentFile::getBackendForFile($f);
+                }
+
+                if ($options['verbose'])
+                    $this->stdout->write('Importing '.$f->getName()."\n");
+
+                // Write file contents to the backend
+                $md5 = hash_init('md5');
+                $sha1 = hash_init('sha1');
+                while ($dlen > 0) {
+                    $read_size = min($dlen, $bk->getBlockSize());
+                    $contents = '';
+                    // reading from the stream will likely return an amount
+                    // of data different from the backend requested block
+                    // size. Loop until $read_size bytes are recieved.
+                    while ($read_size > 0 && ($block = fread($stream, $read_size))) {
+                        $contents .= $block;
+                        $read_size -= strlen($block);
+                    }
+                    if ($read_size != 0) {
+                        // short read
+                        $this->fail(sprintf(
+                            '%s: Some contents are missing from the stream',
+                            $f->getName()
+                        ));
+                    }
+                    // Calculate MD5 and SHA1 hashes of the file to verify
+                    // contents after successfully written to backend
+                    if (!$bk->write($contents))
+                        $this->fail('Unable to send file contents to backend');
+                    hash_update($md5, $contents);
+                    hash_update($sha1, $contents);
+                    $dlen -= strlen($contents);
+                }
+                if (!$bk->flush())
+                    $this->fail('Unable to commit file contents to backend');
+
+                // Check the signature hash
+                if ($finfo['signature']) {
+                    $md5 = base64_encode(hash_final($md5, true));
+                    $sha1 = base64_encode(hash_final($sha1, true));
+                    $sig = str_replace(
+                        array('=','+','/'),
+                        array('','-','_'),
+                        substr($sha1, 0, 16) . substr($md5, 0, 16));
+                    if ($sig != $finfo['signature']) {
+                        $this->fail(sprintf(
+                            '%s: Signature verification failed',
+                            $f->getName()
+                        ));
+                    }
+                }
 
-                $manifest[$f->getId()] = $info;
+                // Read file record footer
+                $footer = fread($stream, 4);
+                if (strlen($footer) != 4)
+                    $this->fail('Unable to read file EOF marker');
+                list(,$footer) = unpack('N', $footer);
+                // Footer should be EOF\x1c as an int
+                if ($footer != 0x454f461c)
+                    $this->fail('Incorrect file EOF marker');
             }
-            $zip->addFromString('MANIFEST', serialize($manifest));
-            $zip->close();
             break;
 
         case 'expunge':
-            // Create a temporary ZIP file
             $files = FileModel::objects();
             $this->_applyCriteria($options, $files);