Merge pull request #2243 from greezybacon/feature/16-file-storage

upgrade: Defer migrating attachments from v1.6

Reviewed-By: Peter Rotich <peter@osticket.com>

README.md

@@ -72,11 +72,18 @@ osTicket-1.7, visit the /scp page of you ticketing system. The upgrader will
 be presented and will walk you through the rest of the process. (The couple
 clicks needed to go through the process are pretty boring to describe).
 
+### Upgrading from v1.6
 **WARNING**: If you are upgrading from osTicket 1.6, please ensure that all
     your files in your upload folder are both readable and writable to your
     http server software. Unreadable files will not be migrated to the
     database during the upgrade and will be effectively lost.
 
+After upgrading, we recommend migrating your attachments to the database or
+to the new filesystem plugin. Use the `file` command-line applet to perform
+the migration.
+
+    php manage.php file migrate --backend=6 --to=D
+
 View the UPGRADING.txt file for other todo items to complete your upgrade.
 
 Help

include/class.file.php

@@ -623,6 +623,7 @@ class FileStorageBackend {
     static $desc = false;
     static $registry;
     static $blocksize = 131072;
+    static $private = false;
 
     /**
      * All storage backends should call this function during the request
@@ -632,8 +633,15 @@ class FileStorageBackend {
         self::$registry[$typechar] = $class;
     }
 
-    static function allRegistered() {
-        return self::$registry;
+    static function allRegistered($private=false) {
+        $R = self::$registry;
+        if (!$private) {
+            foreach ($R as $i=>$bk) {
+                if ($bk::$private)
+                    unset($R[$i]);
+            }
+        }
+        return $R;
     }
 
     /**
@@ -848,4 +856,54 @@ class AttachmentChunkedData extends FileStorageBackend {
 }
 FileStorageBackend::register('D', 'AttachmentChunkedData');
 
+/**
+ * This class provides an interface for files attached on the filesystem in
+ * versions previous to v1.7. The upgrader will keep the attachments on the
+ * disk where they were and write the path into the `attrs` field of the
+ * %file table. This module will continue to serve those files until they
+ * are migrated with the `file` cli app
+ */
+class OneSixAttachments extends FileStorageBackend {
+    static $desc = "upload_dir folder (from osTicket v1.6)";
+    static $private = true;
+
+    function read($bytes=32768, $offset=false) {
+        $filename = $this->meta->attrs;
+        if (!$this->fp)
+            $this->fp = @fopen($filename, 'rb');
+        if (!$this->fp)
+            throw new IOException($filename.': Unable to open for reading');
+        if ($offset)
+            fseek($this->fp, $offset);
+        if (($status = @fread($this->fp, $bytes)) === false)
+            throw new IOException($filename.': Unable to read from file');
+        return $status;
+    }
+
+    function passthru() {
+        $filename = $this->meta->attrs;
+        if (($status = @readfile($filename)) === false)
+            throw new IOException($filename.': Unable to read from file');
+        return $status;
+    }
+
+    function write($data) {
+        throw new IOException('This backend does not support new files');
+    }
+
+    function upload($filepath) {
+        throw new IOException('This backend does not support new files');
+    }
+
+    function unlink() {
+        $filename = $this->meta->attrs;
+        if (!@unlink($filename))
+            throw new IOException($filename.': Unable to delete file');
+        // Drop usage of the `attrs` field
+        $this->meta->attrs = null;
+        $this->meta->save();
+        return true;
+    }
+}
+FileStorageBackend::register('6', 'OneSixAttachments');
 ?>

include/cli/modules/file.php

@@ -59,7 +59,7 @@ class FileManager extends Module {
         switch ($args['action']) {
         case 'backends':
             // List configured backends
-            foreach (FileStorageBackend::allRegistered() as $char=>$bk) {
+            foreach (FileStorageBackend::allRegistered(true) as $char=>$bk) {
                 print "$char -- {$bk::$desc} ($bk)\n";
             }
             break;

include/upgrader/streams/core/15b30765-dd0022fb.patch.sql

@@ -18,7 +18,11 @@ INSERT INTO `%TABLE_PREFIX%file_chunk` (`file_id`, `chunk_id`, `filedata`)
     SELECT `id`, 0, `filedata`
     FROM `%TABLE_PREFIX%file`;
 
-ALTER TABLE `%TABLE_PREFIX%file` DROP COLUMN `filedata`;
+ALTER TABLE `%TABLE_PREFIX%file`
+    DROP COLUMN `filedata`,
+    ADD `bk` CHAR(1) NOT NULL DEFAULT 'D' AFTER `id`,
+    ADD `attrs` VARCHAR(255) AFTER `name`;
+
 OPTIMIZE TABLE `%TABLE_PREFIX%file`;
 
 -- Finished with patch

include/upgrader/streams/core/15b30765-dd0022fb.task.php

@@ -95,37 +95,33 @@ class AttachmentMigrater extends MigrationTask {
         # need to be recalculated for every shift() operation.
         $info = array_pop($this->queue);
         # Attach file to the ticket
-        if (!($info['data'] = @file_get_contents($info['path']))) {
+        if (!@is_readable($info['path'])) {
             # Continue with next file
             return $this->skip($info['attachId'],
                 sprintf('%s: Cannot read file contents', $info['path']));
         }
         # Get the mime/type of each file
         # XXX: Use finfo_buffer for PHP 5.3+
-        if(function_exists('mime_content_type')) {
-            //XXX: function depreciated in newer versions of PHP!!!!!
-            $info['type'] = mime_content_type($info['path']);
-        } elseif (function_exists('finfo_file')) { // PHP 5.3.0+
+        if (function_exists('finfo_file')) { // PHP 5.3.0+
             $finfo = finfo_open(FILEINFO_MIME_TYPE);
             $info['type'] = finfo_file($finfo, $info['path']);
         }
+        elseif (function_exists('mime_content_type')) {
+            //XXX: function depreciated in newer versions of PHP!!!!!
+            $info['type'] = mime_content_type($info['path']);
+        }
         # TODO: Add extension-based mime-type lookup
 
-        if (!($fileId = $this->saveAttachment($info))) {
+        $file = $this->saveAttachment($info);
+        if (!$file)
             return $this->skip($info['attachId'],
                 sprintf('%s: Unable to migrate attachment', $info['path']));
-        }
+
         # Update the ATTACHMENT_TABLE record to set file_id
         db_query('update '.TICKET_ATTACHMENT_TABLE
-                .' set file_id='.db_input($fileId)
+                .' set file_id='.db_input($file->id)
                 .' where attach_id='.db_input($info['attachId']));
-        # Remove disk image of the file. If this fails, the migration for
-        # this file would not be retried, because the file_id in the
-        # TICKET_ATTACHMENT_TABLE has a nonzero value now
-        if (!@unlink($info['path'])) //XXX: what should we do on failure?
-            $this->error(
-                sprintf('%s: Unable to remove file from disk',
-                $info['path']));
+
         # TODO: Log an internal note to the ticket?
         return true;
     }
@@ -231,47 +227,44 @@ class AttachmentMigrater extends MigrationTask {
         return $this->errorList;
     }
 
-    // This is the AttachmentFile::create() method from osTicket 1.7.6. It's
-    // been ported here so that further changes to the %file table and the
-    // AttachmentFile::create() method do not affect upgrades from osTicket
-    // 1.6 to osTicket 1.8 and beyond.
+    // This is (similar to) the AttachmentFile::create() method from
+    // osTicket 1.7.6. It's been ported here so that further changes to the
+    // %file table and the AttachmentFile::create() method do not affect
+    // upgrades from osTicket 1.6 to osTicket 1.8 and beyond.
     function saveAttachment($file) {
 
-        if(!$file['hash'])
+        if (!$file['hash'])
             $file['hash']=MD5(md5_file($file['path']).time());
-        $file['data'] = file_get_contents($file['path']);
-        if(!$file['size'])
-            $file['size']=strlen($file['data']);
-
-        $sql='INSERT INTO '.FILE_TABLE.' SET created=NOW() '
-            .',type='.db_input($file['type'])
-            .',size='.db_input($file['size'])
-            .',name='.db_input($file['name'])
-            .',hash='.db_input($file['hash']);
-
-        if (!(db_query($sql) && ($id=db_insert_id())))
-            return false;
-
-        $f = new CompatAttachmentFile($id);
-        $bk = new AttachmentChunkedData($f);
-        if (!$bk->write($file['data']))
-            return false;
-
-        return $id;
+        if (!$file['size'])
+            $file['size'] = filesize($file['path']);
+
+        return OldOneSixFile::create(array(
+            'name' => $file['name'],
+            'size' => $file['size'],
+            'type' => $file['type'],
+            'hash' => $file['hash'],
+            'bk' => '6',
+            'attrs' => $file['path'],
+        ));
     }
 }
 
-class CompatAttachmentFile {
-    var $id;
-
-    function __construct($id) {
-        $this->id = $id;
-    }
-
-    function getId() {
-        return $this->id;
+class OldOneSixFile extends VerySimpleModel {
+    static $meta = array(
+        'table' => FILE_TABLE,
+        'pk' => array('id'),
+        'joins' => array(
+            'attachments' => array(
+                'reverse' => 'Attachment.file'
+            ),
+        ),
+    );
+
+    static function create($info) {
+        $I = parent::create($info);
+        $I->save();
+        return $I;
     }
 }
 
 return 'AttachmentMigrater';
-?>

include/upgrader/streams/core/934954de-f1ccd3bb.patch.sql

@@ -8,15 +8,30 @@
  */
 
 ALTER TABLE `%TABLE_PREFIX%file`
-    ADD `bk` CHAR(1) NOT NULL DEFAULT 'D' AFTER `ft`,
     -- RFC 4288, Section 4.2 declares max MIMEType at 255 ascii chars
     CHANGE `type` `type` varchar(255) collate ascii_general_ci NOT NULL default '',
     CHANGE `size` `size` BIGINT(20) NOT NULL DEFAULT 0,
     CHANGE `hash` `key` VARCHAR(86) COLLATE ascii_general_ci,
     ADD `signature` VARCHAR(86) COLLATE ascii_bin AFTER `key`,
-    ADD `attrs` VARCHAR(255) AFTER `name`,
     ADD INDEX (`signature`);
 
+-- dd0022fb14892c0bb6a9700392df2de7 added `bk` and `attrs` to facilitate
+-- upgrading from osTicket 1.6 without loading files into the database
+SET @s = (SELECT IF(
+    (SELECT COUNT(*)
+        FROM INFORMATION_SCHEMA.COLUMNS
+        WHERE table_name = '%TABLE_PREFIX%file'
+        AND table_schema = DATABASE()
+        AND column_name = 'bk'
+    ) > 0,
+    "SELECT 1",
+    "ALTER TABLE `%TABLE_PREFIX%file`
+        ADD `bk` CHAR(1) NOT NULL DEFAULT 'D' AFTER `ft`,
+        ADD `attrs` VARCHAR(255) AFTER `name`"
+));
+PREPARE stmt FROM @s;
+EXECUTE stmt;
+
 -- Finished with patch
 UPDATE `%TABLE_PREFIX%config`
     SET `value` = 'f1ccd3bb620e314b0ae1dbd0a1a99177'