Merge pull request #517 from protich/issue/508

Consistent decoding and transcoding to UTF-8 as anticipated by the rest of osTicket

Reviewed-By: Jared Hancock <jared@osticket.com>

include/class.format.php

@@ -46,11 +46,13 @@ class Format {
             $charset = 'ISO-8859-1';
 
         if(function_exists('iconv') && $charset)
-            return iconv($charset, $encoding.'//IGNORE', $text);
-        elseif(function_exists('iconv_mime_decode'))
-            return iconv_mime_decode($text, 0, $encoding);
-        else //default to utf8 encoding.
-            return utf8_encode($text);
+            $text = iconv($charset, $encoding.'//IGNORE', $text);
+        elseif(function_exists('mb_convert_encoding') && $charset && $encoding)
+            $text = mb_convert_encoding($text, $encoding, $charset);
+        elseif(!strcasecmp($encoding, 'utf-8')) //forced blind utf8 encoding.
+            $text = function_exists('imap_utf8')?imap_utf8($text):utf8_encode($text);
+
+        return $text;
     }
 
     //Wrapper for utf-8 encoding.
@@ -58,6 +60,24 @@ class Format {
         return Format::encode($text, $charset, 'utf-8');
     }
 
+    function mimedecode($text, $encoding='UTF-8') {
+
+        if(function_exists('imap_mime_header_decode')
+                && ($parts = imap_mime_header_decode($text))) {
+            $str ='';
+            foreach ($parts as $part)
+                $str.= Format::encode($part->text, $part->charset, $encoding);
+
+            $text = $str;
+        } elseif(function_exists('iconv_mime_decode')) {
+            $text = iconv_mime_decode($text, 0, $encoding);
+        } elseif(!strcasecmp($encoding, 'utf-8') && function_exists('imap_utf8')) {
+            $text = imap_utf8($text);
+        }
+
+        return $text;
+    }
+
 	function phone($phone) {
 
 		$stripped= preg_replace("/[^0-9]/", "", $phone);

include/class.mailparse.php

@@ -27,18 +27,29 @@ class Mail_Parse {
 
     var $struct;
 
-    function Mail_parse($mimeMessage,$includeBodies=true,$decodeHeaders=TRUE,$decodeBodies=TRUE){
+    var $charset ='UTF-8'; //Default charset.
 
-        $this->mime_message=$mimeMessage;
-        $this->include_bodies=$includeBodies;
-        $this->decode_headers=$decodeHeaders;
-        $this->decode_bodies=$decodeBodies;
+    function Mail_parse($mimeMessage, $charset=null){
+
+        $this->mime_message = $mimeMessage;
+
+        if($charset)
+            $this->charset = $charset;
+
+        $this->include_bodies = true;
+        $this->decode_headers = true;
+        $this->decode_bodies = true;
+
+        //Desired charset
+        if($charset)
+            $this->charset = $charset;
     }
 
     function decode() {
 
         $params = array('crlf'          => "\r\n",
-                        'input'         =>$this->mime_message,
+                        'charset'       => $this->charset,
+                        'input'         => $this->mime_message,
                         'include_bodies'=> $this->include_bodies,
                         'decode_headers'=> $this->decode_headers,
                         'decode_bodies' => $this->decode_bodies);
@@ -146,12 +157,18 @@ class Mail_Parse {
         return $body;
     }
 
-    function getPart($struct,$ctypepart) {
+    function getPart($struct, $ctypepart) {
 
         if($struct && !$struct->parts) {
             $ctype = @strtolower($struct->ctype_primary.'/'.$struct->ctype_secondary);
-            if($ctype && strcasecmp($ctype,$ctypepart)==0)
-                return $struct->body;
+            if($ctype && strcasecmp($ctype,$ctypepart)==0) {
+                $content = $struct->body;
+                //Encode to desired encoding - ONLY if charset is known??
+                if(isset($struct->ctype_parameters['charset']) && strcasecmp($struct->ctype_parameters['charset'], $this->charset))
+                    $content = Format::encode($content, $struct->ctype_parameters['charset'], $this->charset);
+
+                return $content;
+            }
         }
 
         $data='';
@@ -295,8 +312,8 @@ class EmailDataParser {
             }
         }
 
-        $data['subject'] = Format::utf8encode($parser->getSubject());
-        $data['message'] = Format::utf8encode(Format::stripEmptyLines($parser->getBody()));
+        $data['subject'] = $parser->getSubject();
+        $data['message'] = Format::stripEmptyLines($parser->getBody());
         $data['header'] = $parser->getHeader();
         $data['mid'] = $parser->getMessageId();
         $data['priorityId'] = $parser->getPriority();

include/pear/Mail/mimeDecode.php

@@ -28,8 +28,8 @@
  * - Redistributions in binary form must reproduce the above copyright
  *   notice, this list of conditions and the following disclaimer in the
  *   documentation and/or other materials provided with the distribution.
- * - Neither the name of the authors, nor the names of its contributors 
- *   may be used to endorse or promote products derived from this 
+ * - Neither the name of the authors, nor the names of its contributors
+ *   may be used to endorse or promote products derived from this
  *   software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
@@ -174,9 +174,9 @@ class Mail_mimeDecode extends PEAR
     function getHeader() {
         return $this->_header;
     }
-        
 
-  
+
+
 
     /**
      * Begins the decoding process. If called statically
@@ -219,6 +219,8 @@ class Mail_mimeDecode extends PEAR
 	                             $params['decode_bodies']  : false;
             $this->_decode_headers = isset($params['decode_headers']) ?
 	                             $params['decode_headers'] : false;
+            $this->_charset = isset($params['charset']) ?
+                                 $params['charset'] : 'UTF-8';
 
             $structure = $this->_decode($this->_header, $this->_body);
             if ($structure === false) {
@@ -374,7 +376,7 @@ class Mail_mimeDecode extends PEAR
             }
             for ($i = 0; $i < count($structure->parts); $i++) {
 
-            
+
                 if (!empty($structure->headers['content-type']) AND substr(strtolower($structure->headers['content-type']), 0, 8) == 'message/') {
                     $prepend      = $prepend . $mime_number . '.';
                     $_mime_number = '';
@@ -394,7 +396,7 @@ class Mail_mimeDecode extends PEAR
             $structure->mime_id = $prepend . $mime_number;
             $no_refs ? $return[$prepend . $mime_number] = '' : $return[$prepend . $mime_number] = &$structure;
         }
-        
+
         return $return;
     }
 
@@ -567,6 +569,16 @@ class Mail_mimeDecode extends PEAR
                     break;
             }
 
+            //Convert decoded text to the desired charset.
+            if($charset && $this->_charset && strcasecmp($this->_charset, $charset)) {
+                if(function_exists('iconv'))
+                    $text = iconv($charset, $this->_charset.'//IGNORE', $text);
+                elseif(function_exists('mb_convert_encoding'))
+                    $text = mb_convert_encoding($text, $this->_charset, $charset);
+                elseif(!strcasecmp($this->_charset, 'utf-8')) //forced blind utf8 encoding.
+                    $text = function_exists('imap_utf8')?imap_utf8($text):utf8_encode($text);
+            }
+
             $input = str_replace($encoded, $text, $input);
         }
 
@@ -698,7 +710,7 @@ class Mail_mimeDecode extends PEAR
 
     /**
      * getSendArray() returns the arguments required for Mail::send()
-     * used to build the arguments for a mail::send() call 
+     * used to build the arguments for a mail::send() call
      *
      * Usage:
      * $mailtext = Full email (for example generated by a template)
@@ -741,7 +753,7 @@ class Mail_mimeDecode extends PEAR
         }
         $to = substr($to,1);
         return array($to,$header,$this->_body);
-    } 
+    }
 
     /**
      * Returns a xml copy of the output of