Merge pull request #263 from greezybacon/issue/chunk-fetch-attachments

Reviewed-By: Peter Rotich <peter@osticket.com>

include/class.file.php

@@ -27,7 +27,8 @@ class AttachmentFile {
         if(!$id && !($id=$this->getId()))
             return false;
 
-        $sql='SELECT f.*, count(DISTINCT c.canned_id) as canned, count(DISTINCT t.ticket_id) as tickets '
+        $sql='SELECT id, type, size, name, hash, f.created, '
+            .' count(DISTINCT c.canned_id) as canned, count(DISTINCT t.ticket_id) as tickets '
             .' FROM '.FILE_TABLE.' f '
             .' LEFT JOIN '.CANNED_ATTACHMENT_TABLE.' c ON(c.file_id=f.id) '
             .' LEFT JOIN '.TICKET_ATTACHMENT_TABLE.' t ON(t.file_id=f.id) '
@@ -90,12 +91,24 @@ class AttachmentFile {
         return $this->ht['hash'];
     }
 
-    function getBinary() {
-        return $this->ht['filedata'];
+    function open() {
+        return new AttachmentChunkedData($this->id);
+    }
+
+    function sendData() {
+        $file = $this->open();
+        while ($chunk = $file->read())
+            echo $chunk;
     }
 
     function getData() {
-        return $this->getBinary();
+        # XXX: This is horrible, and is subject to php's memory
+        #      restrictions, etc. Don't use this function!
+        ob_start();
+        $this->sendData();
+        $data = &ob_get_contents();
+        ob_end_clean();
+        return $data;
     }
 
     function delete() {
@@ -110,7 +123,7 @@ class AttachmentFile {
 
         header('Content-Type: '.($this->getType()?$this->getType():'application/octet-stream'));
         header('Content-Length: '.$this->getSize());
-        echo $this->getData();
+        $this->sendData();
         exit();
     }
 
@@ -132,7 +145,7 @@ class AttachmentFile {
         
         header('Content-Transfer-Encoding: binary');
         header('Content-Length: '.$this->getSize());
-        echo $this->getBinary();
+        $this->sendData();
         exit();
     }
 
@@ -168,15 +181,9 @@ class AttachmentFile {
         if (!(db_query($sql) && ($id=db_insert_id())))
             return false;
 
-        foreach (str_split($file['data'], 1024*100) as $chunk) {
-            $sql='UPDATE '.FILE_TABLE
-                .' SET filedata = CONCAT(filedata,'.db_input($chunk).')'
-                .' WHERE id='.db_input($id);
-            if(!db_query($sql)) {
-                db_query('DELETE FROM '.FILE_TABLE.' WHERE id='.db_input($id).' LIMIT 1');
-                return false;
-            }
-        }
+        $data = new AttachmentChunkedData($id);
+        if (!$data->write($file['data']))
+            return false;
 
         return $id;
     }
@@ -213,44 +220,56 @@ class AttachmentFile {
                     .'SELECT file_id FROM '.FAQ_ATTACHMENT_TABLE
                 .') still_loved'
             .')');
+        AttachmentChunkedData::deleteOrphans();
         return db_affected_rows();
     }
 }
 
-class AttachmentList {
-    function AttachmentList($table, $key) {
-        $this->table = $table;
-        $this->key = $key;
+/**
+ * Attachments stored in the database are cut into 256kB chunks and stored
+ * in the FILE_CHUNK_TABLE to overcome the max_allowed_packet limitation of
+ * LOB fields in the MySQL database
+ */
+define('CHUNK_SIZE', 500*1024); # Beware if you change this...
+class AttachmentChunkedData {
+    function AttachmentChunkedData($file) {
+        $this->_file = $file;
+        $this->_pos = 0;
     }
 
-    function all() {
-        if (!isset($this->list)) {
-            $this->list = array();
-            $res=db_query('SELECT file_id FROM '.$this->table
-                .' WHERE '.$this->key);
-            while(list($id) = db_fetch_row($res)) {
-                $this->list[] = new AttachmentFile($id);
-            }
-        }
-        return $this->list;
+    function length() {
+        list($length) = db_fetch_row(db_query(
+             'SELECT SUM(LENGTH(filedata)) FROM '.FILE_CHUNK_TABLE
+            .' WHERE file_id='.db_input($this->_file)));
+        return $length;
     }
-    
-    function getCount() {
-        return count($this->all());
+
+    function read() {
+        # Read requested length of data from attachment chunks
+        list($buffer) = @db_fetch_row(db_query(
+            'SELECT filedata FROM '.FILE_CHUNK_TABLE.' WHERE file_id='
+            .db_input($this->_file).' AND chunk_id='.$this->_pos++));
+        return $buffer;
     }
 
-    function add($fileId) {
-        db_query(
-            'INSERT INTO '.$this->table
-                .' SET '.$this->key
-                .' file_id='.db_input($fileId));
+    function write($what, $chunk_size=CHUNK_SIZE) {
+        $offset=0;
+        for (;;) {
+            $block = substr($what, $offset, $chunk_size);
+            if (!$block) break;
+            if (!db_query('REPLACE INTO '.FILE_CHUNK_TABLE
+                    .' SET filedata=0x'.bin2hex($block).', file_id='
+                    .db_input($this->_file).', chunk_id='.db_input($this->_pos++)))
+                return false;
+            $offset += strlen($block);
+        }
+        return true;
     }
 
-    function remove($fileId) {
+    function deleteOrpans() {
         db_query(
-            'DELETE FROM '.$this->table
-                .' WHERE '.$this->key
-                .' AND file_id='.db_input($fileId));
+            'DELETE FROM '.FILE_CHUNK_TABLE.' WHERE file_id NOT IN '
+                .'( SELECT id FROM '.FILE_TABLE.') still_loved');
+        return db_affected_rows();
     }
 }
-?>

include/class.upgrader.php

@@ -269,8 +269,6 @@ class Upgrader extends SetupWizard {
         $tasks=array();
         switch($phash) { //Add  patch specific scripted tasks.
             case 'c00511c7-7be60a84': //V1.6 ST- 1.7 * {{MD5('1.6 ST') -> c00511c7c1db65c0cfad04b4842afc57}}
-                $tasks[] = array('func' => 'migrateAttachments2DB',
-                                 'desc' => 'Migrating attachments to database, it might take a while depending on the number of files.');
                 $tasks[] = array('func' => 'migrateSessionFile2DB',
                                  'desc' => 'Transitioning to db-backed sessions');
                 break;
@@ -282,6 +280,10 @@ class Upgrader extends SetupWizard {
                 $tasks[] = array('func' => 'migrateGroupDeptAccess',
                                  'desc' => 'Migrating group\'s department access to a new table');
                 break;
+            case '15b30765-dd0022fb':
+                $tasks[] = array('func' => 'migrateAttachments2DB',
+                                 'desc' => 'Migrating attachments to database, it might take a while depending on the number of files.');
+                break;
         }
 
         //Check IF SQL cleanup exists. 

include/upgrader/sql/15b30765-dd0022fb.cleanup.sql

+
+-- Drop fields we no longer need in the reference table.
+-- NOTE: This was moved from the 1.6* major upgrade script because the
+--       handling of attachments changed with dd0022fb
+ALTER TABLE `%TABLE_PREFIX%ticket_attachment`
+    DROP `file_size`,
+    DROP `file_name`,
+    DROP `file_key`,
+    DROP `updated`,
+    DROP `deleted`;

include/upgrader/sql/15b30765-dd0022fb.patch.sql

+/**
+ * @version v1.7 RC2+
+ * @signature dd0022fb14892c0bb6a9700392df2de7
+ *
+ * Migrate file attachment data from %file to %file_chunk
+ *  
+ */
+
+DROP TABLE IF EXISTS `%TABLE_PREFIX%file_chunk`;
+CREATE TABLE `%TABLE_PREFIX%file_chunk` (
+    `file_id` int(11) NOT NULL,
+    `chunk_id` int(11) NOT NULL,
+    `filedata` longblob NOT NULL,
+    PRIMARY KEY (`file_id`, `chunk_id`)
+) ENGINE=MyISAM  DEFAULT CHARSET=utf8;
+
+INSERT INTO `%TABLE_PREFIX%file_chunk` (`file_id`, `chunk_id`, `filedata`)
+    SELECT `id`, 0, `filedata`
+    FROM `%TABLE_PREFIX%file`;
+
+ALTER TABLE `%TABLE_PREFIX%file` DROP COLUMN `filedata`;
+OPTIMIZE TABLE `%TABLE_PREFIX%file`;
+
+-- Finished with patch
+UPDATE `%TABLE_PREFIX%config`
+    SET `schema_signature`='dd0022fb14892c0bb6a9700392df2de7';

include/upgrader/sql/c00511c7-7be60a84.cleanup.sql

@@ -4,14 +4,6 @@ ALTER TABLE `%TABLE_PREFIX%config`
     DROP COLUMN `timezone_offset`,
     DROP COLUMN `api_passphrase`;
 
--- Drop fields we no longer need in the reference table.
-ALTER TABLE `%TABLE_PREFIX%ticket_attachment`
-    DROP `file_size`,
-    DROP `file_name`,
-    DROP `file_key`,
-    DROP `updated`,
-    DROP `isdeleted`;
-
 -- Drop fields we no longer need in staff table.
 ALTER TABLE `%TABLE_PREFIX%staff`
     DROP `append_signature`,

main.inc.php

@@ -63,7 +63,7 @@
 
     #Current version && schema signature (Changes from version to version)
     define('THIS_VERSION','1.7-RC2+'); //Shown on admin panel
-    define('SCHEMA_SIGNATURE','15b3076533123ff617801d89861136c8'); //MD5 signature of the db schema. (used to trigger upgrades)
+    define('SCHEMA_SIGNATURE','dd0022fb14892c0bb6a9700392df2de7'); //MD5 signature of the db schema. (used to trigger upgrades)
     #load config info
     $configfile='';
     if(file_exists(ROOT_DIR.'ostconfig.php')) //Old installs prior to v 1.6 RC5
@@ -131,6 +131,7 @@
     define('SYSLOG_TABLE',TABLE_PREFIX.'syslog');
     define('SESSION_TABLE',TABLE_PREFIX.'session');
     define('FILE_TABLE',TABLE_PREFIX.'file');
+    define('FILE_CHUNK_TABLE',TABLE_PREFIX.'file_chunk');
 
     define('STAFF_TABLE',TABLE_PREFIX.'staff');
     define('DEPT_TABLE',TABLE_PREFIX.'department');

setup/inc/sql/osTicket-mysql.sql

@@ -327,15 +327,24 @@ CREATE TABLE `%TABLE_PREFIX%file` (
   `size` varchar(25) NOT NULL default '',
   `hash` varchar(125) NOT NULL,
   `name` varchar(255) NOT NULL default '',
-  `filedata` longblob NOT NULL,
   `created` datetime NOT NULL,
   PRIMARY KEY  (`id`),
   KEY `hash` (`hash`)
 ) ENGINE=MyISAM  DEFAULT CHARSET=utf8;
 
-INSERT INTO `%TABLE_PREFIX%file` (`id`, `type`, `size`, `hash`, `name`, `filedata`, `created`) VALUES
-(1, 'text/plain', '25', '670c6cc1d1dfc97fad20e5470251b255', 'osTicket.txt', 0x43616e6e6564206174746163686d656e747320726f636b210a, NOW());
+INSERT INTO `%TABLE_PREFIX%file` (`id`, `type`, `size`, `hash`, `name`, `created`) VALUES
+(1, 'text/plain', '25', '670c6cc1d1dfc97fad20e5470251b255', 'osTicket.txt', NOW());
 
+DROP TABLE IF EXISTS `%TABLE_PREFIX%file_chunk`;
+CREATE TABLE `%TABLE_PREFIX%file_chunk` (
+    `file_id` int(11) NOT NULL,
+    `chunk_id` int(11) NOT NULL,
+    `filedata` longblob NOT NULL,
+    PRIMARY KEY (`file_id`, `chunk_id`)
+) ENGINE=MyISAM  DEFAULT CHARSET=utf8;
+
+INSERT INTO `%TABLE_PREFIX%file_chunk` (`file_id`, `chunk_id`, `filedata`)
+VALUES (1, 0, 0x43616e6e6564206174746163686d656e747320726f636b210a);
 
 DROP TABLE IF EXISTS `%TABLE_PREFIX%groups`;
 CREATE TABLE `%TABLE_PREFIX%groups` (

setup/inc/sql/osTicket-mysql.sql.md5

-15b3076533123ff617801d89861136c8
+dd0022fb14892c0bb6a9700392df2de7