Merge pull request #2052 from greezybacon/feature/deploy-manifest

deploy: Add concept of a MANIFEST to the deployment process

Reviewed-By: Peter Rotich <peter@osticket.com>

setup/cli/modules/class.module.php

@@ -4,10 +4,6 @@ class Option {
 
     var $default = false;
 
-    function Option() {
-        call_user_func_array(array($this, "__construct"), func_get_args());
-    }
-
     function __construct($options=false) {
         list($this->short, $this->long) = array_slice($options, 0, 2);
         $this->help = (isset($options['help'])) ? $options['help'] : "";
@@ -119,10 +115,6 @@ class Module {
     var $_options;
     var $_args;
 
-    function Module() {
-        call_user_func_array(array($this, '__construct'), func_get_args());
-    }
-
     function __construct() {
         $this->options['help'] = array("-h","--help",
             'action'=>'store_true',
@@ -262,21 +254,36 @@ class Module {
     function parseArgs($argv) {
         $options = $args = array();
         $argv = array_slice($argv, 0);
+        $more_opts = true;
         while ($arg = array_shift($argv)) {
             if (strpos($arg, '=') !== false) {
                 list($arg, $value) = explode('=', $arg, 2);
                 array_unshift($argv, $value);
             }
+            if ($arg == '--') {
+                $more_opts = false;
+                continue;
+            }
+            // Allow multiple simple args like -Dvt
+            if ($arg[0] == '-' && strlen($arg) > 2) {
+                foreach (str_split(substr($arg, 2)) as $O)
+                    array_unshift($argv, "-{$O}");
+                $arg = substr($arg, 0, 2);
+            }
             $found = false;
-            foreach ($this->options as $opt) {
-                if ($opt->short == $arg || $opt->long == $arg) {
-                    if ($opt->handleValue($options, $argv))
-                        array_shift($argv);
-                    $found = true;
+            if ($more_opts && $arg[0] == '-') {
+                foreach ($this->options as $opt) {
+                    if ($opt->short == $arg || $opt->long == $arg) {
+                        if ($nargs = $opt->handleValue($options, $argv))
+                            while ($nargs--)
+                                array_shift($argv);
+                        $found = true;
+                    }
                 }
             }
-            if (!$found && $arg[0] != '-')
+            if (!$found && (!$more_opts || $arg[0] != '-'))
                 $args[] = $arg;
+            // XXX else show help if $strict?
         }
         return array($options, $args);
     }

setup/cli/modules/deploy.php

@@ -24,6 +24,10 @@ class Deployment extends Unpacker {
             'action'=>'store_true',
             'help'=>'Remove files from the destination that are no longer
                 included in this repository');
+        $this->options['git'] = array('-g','--git',
+            'action'=>'store_true',
+            'help'=>'Use `git ls-files -s` as files source. Eliminates
+                possibility of deploying untracked files');
         # super(*args);
         call_user_func_array(array('parent', '__construct'), func_get_args());
     }
@@ -87,12 +91,26 @@ class Deployment extends Unpacker {
         }
     }
 
-    function copyFile($src, $dest) {
+    function writeManifest($root) {
+        $lines = array();
+        foreach ($this->manifest as $F=>$H)
+            $lines[] = "$H $F";
+
+        return file_put_contents($this->include_path.'/.MANIFEST', implode("\n", $lines));
+    }
+
+    function hashContents($file) {
+        $md5 = md5($file);
+        $sha1 = sha1($file);
+        return substr($md5, -20) . substr($sha1, -20);
+    }
+
+    function getEditedContents($src) {
         static $short = false;
         static $version = false;
 
         if (substr($src, -4) != '.php')
-            return parent::copyFile($src, $dest);
+            return false;
 
         if (!$short) {
             $hash = exec('git rev-parse HEAD');
@@ -103,7 +121,7 @@ class Deployment extends Unpacker {
             $version = exec('git describe');
 
         if (!$short || !$version)
-            return parent::copyFile($src, $dest);
+            return false;
 
         $source = file_get_contents($src);
         $source = preg_replace(':<script(.*) src="([^"]+)\.js"></script>:',
@@ -125,10 +143,68 @@ class Deployment extends Unpacker {
             "$1ini_set('$2', '0'); // Set by installer",
             $source);
 
-        if (!file_put_contents($dest, $source))
-            die("Unable to apply rewrite rules to ".$dest);
+        return $source;
+    }
+
+    function copyFile($source, $dest, $hash=false, $mode=0644) {
+        $contents = $this->getEditedContents($source);
+        if ($contents === false)
+            // Regular file
+            return parent::copyFile($source, $dest, $hash, $mode);
+
+        if (!file_put_contents($dest, $contents))
+            $this->fail($dest.": Unable to apply rewrite rules");
+
+        $this->updateManifest($source, $hash);
+        return chmod($dest, $mode);
+    }
+
+    function unpackage($folder, $destination, $recurse=0, $exclude=false) {
+        $use_git = $this->getOption('git', false);
+        if (!$use_git)
+            return parent::unpackage($folder, $destination, $recurse, $exclude);
 
-        return true;
+        // Attempt to read from git using `git ls-files` for deployment
+        if (substr($destination, -1) !== '/')
+            $destination .= '/';
+        $source = $this->source;
+        if (substr($source, -1) != '/')
+            $source .= '/';
+        $local = str_replace(array($source, '{,.}*'), array('',''), $folder);
+
+        $pipes = array();
+        $patterns = array();
+        foreach ((array) $exclude as $x) {
+            $patterns[] = str_replace($source, '', $x);
+        }
+        $X = implode(' --exclude-per-directory=', $patterns);
+        chdir($source.$local);
+        if (!($files = proc_open(
+            "git ls-files -zs --exclude-standard --exclude-per-directory=$X -- .",
+            array(1 => array('pipe', 'w')),
+            $pipes
+        ))) {
+            return parent::unpackage($folder, $destination, $recurse, $exclude);
+        }
+
+        $dryrun = $this->getOption('dry-run', false);
+        $verbose = $this->getOption('verbose') || $dryrun;
+        while ($line = stream_get_line($pipes[1], 255, "\x00")) {
+            list($mode, $hash, , $path) = preg_split('/\s+/', $line);
+            $src = $source.$local.$path;
+            if ($this->exclude($exclude, $src))
+                continue;
+            if (!$this->isChanged($src, $hash))
+                continue;
+            $dst = $destination.$path;
+            if ($verbose)
+                $this->stdout->write($dst."\n");
+            if ($dryrun)
+                continue;
+            if (!is_dir(dirname($dst)))
+                mkdir(dirname($dst), 0751, true);
+            $this->copyFile($src, $dst, $hash, octdec($mode));
+        }
     }
 
     function run($args, $options) {
@@ -147,16 +223,19 @@ class Deployment extends Unpacker {
         $include = ($upgrade) ? $this->get_include_dir()
             : ($options['include'] ? $options['include']
                 : rtrim($this->destination, '/')."/include");
-        $include = rtrim($include, '/').'/';
+        $this->include_path = $include = rtrim($include, '/').'/';
 
         # Locate the upload folder
-        $root = $this->find_root_folder();
+        $root = $this->source = $this->find_root_folder();
         $rootPattern = str_replace("\\","\\\\", $root); //need for windows case
 
-        $exclusions = array("$rootPattern/include", "$rootPattern/.git*",
+        # Prime the manifest system
+        $this->readManifest($this->destination.'/.MANIFEST');
+
+        $exclusions = array("$rootPattern/include/*", "$rootPattern/.git*",
             "*.sw[a-z]","*.md", "*.txt");
         if (!$options['setup'])
-            $exclusions[] = "$rootPattern/setup";
+            $exclusions[] = "$rootPattern/setup/*";
 
         # Unpack everything but the include/ folder
         $this->unpackage("$root/{,.}*", $this->destination, -1,
@@ -177,6 +256,8 @@ class Deployment extends Unpacker {
                 array("ost-config.php","settings.php","plugins/",
                 "*/.htaccess"));
         }
+
+        $this->writeManifest($this->destination);
     }
 }
 

setup/cli/modules/package.php

+<?php
+require_once "deploy.php";
+
+class Packager extends Deployment {
+    var $prologue = "Creates an osTicket distribution ZIP file";
+
+    var $epilog =
+        "Packaging is based on the `deploy` and `test` cli apps. After
+        running the tests, the system is deployed into a temporary staging
+        area using the files tracked in git if supported. Afterwards, the
+        staging area is packaged as a ZIP file.";
+
+    var $options = array(
+        'format' => array('-F','--format',
+            'default'=>'zip',
+            'help'=>'Output the package in this format. Supported formats are
+                "zip" (the default), and "targz"'
+        ),
+        'skip-test' => array('-S','--skip-test',
+            'action'=>'store_true', 'default'=>false,
+            'help'=>'Skip regression testing (NOT RECOMMENDED)',
+        ),
+    );
+    var $arguments = array();
+
+    function __construct() {
+        // Skip options added to the deploy — options and arguments are
+        // forced in this module
+        call_user_func_array(array('Module', '__construct'), func_get_args());
+    }
+
+    function run($args, $options) {
+        // Set some forced args and options
+        $temp = rtrim(sys_get_temp_dir(), DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR;
+        $stage_path = $temp . 'osticket'
+            . substr(md5('osticket-stage'.getmypid().getcwd()), -8);
+        $args['install-path'] = $stage_path . '/upload';
+
+        // Deployment will auto-create the staging area
+
+        // Ensure that the staging path is cleaned up on exit
+        register_shutdown_function(function() use ($stage_path) {
+            $delTree = function($dir) use (&$delTree) {
+                $files = array_diff(scandir($dir), array('.','..'));
+                foreach ($files as $file) {
+                    (is_dir("$dir/$file")) ? $delTree("$dir/$file") : unlink("$dir/$file");
+                }
+                return rmdir($dir);
+            };
+            return $delTree($stage_path);
+        });
+
+        $options['setup'] = true;
+        $options['git'] = true;
+        $options['verbose'] = true;
+
+        $options['clean'] = false;
+        $options['dry-run'] = false;
+        $options['include'] = false;
+
+        $this->_args = $args;
+        $this->_options = $options;
+
+        // TODO: Run the testing applet first
+        $root = $this->find_root_folder();
+        if (!$this->getOption('skip-test') && $this->run_tests($root) > 0)
+            $this->fail("Regression tests failed. Cowardly refusing to package");
+
+        // Run the deployment
+        // NOTE: The deployment will change the working directory
+        parent::run($args, $options);
+
+        // Deploy the `setup/scripts` folder to `/scripts`
+        $root = $this->source;
+        Unpacker::unpackage("$root/setup/scripts/{,.}*", "$stage_path/scripts", -1);
+
+        // Package up the staging area
+        $version = exec('git describe');
+        switch (strtolower($this->getOption('format'))) {
+        case 'zip':
+        default:
+            $this->packageZip("$root/osTicket-$version.zip", $stage_path);
+        }
+    }
+
+    function run_tests($root) {
+        return (require "$root/setup/test/run-tests.php");
+    }
+
+    function packageZip($name, $path) {
+        $zip = new ZipArchive();
+        if (!$zip->open($name, ZipArchive::CREATE | ZipArchive::OVERWRITE) === true)
+            return false;
+
+        $php56 = version_compare(phpversion(), '5.6.0', '>');
+        $addFiles = function($dir) use (&$addFiles, $zip, $path, $php56) {
+            $files = array_diff(scandir($dir), array('.','..'));
+            $path = rtrim($path, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR;
+            foreach ($files as $file) {
+                $full = "$dir/$file";
+                $local = str_replace($path, '', $full);
+                if (is_dir($full))
+                    $addFiles($full);
+                else
+                    // XXX: AddFile() will keep the file open and run
+                    //      out of OS open file handles
+                    $zip->addFromString($local, file_get_contents($full));
+                    // This only works on PHP >= v5.6
+                    if ($php56) {
+                        // Set the Unix mode of the file
+                        $stat = stat($full);
+                        $zip->setExternalAttributesName($local, ZipArchive::OPSYS_UNIX, $stat['mode']);
+                    }
+            }
+        };
+        $addFiles($path);
+        return $zip->close();
+
+    }
+}
+Module::register('package', 'Packager');

setup/cli/modules/unpack.php

@@ -34,6 +34,10 @@ class Unpacker extends Module {
              main installation path.",
     );
 
+    var $manifest;
+    var $source;
+    var $destination;
+
     function realpath($path) {
         return ($p = realpath($path)) ? $p : $path;
     }
@@ -88,8 +92,52 @@ class Unpacker extends Module {
         return false;
     }
 
-    function copyFile($src, $dest) {
-        return copy($src, $dest);
+    function readManifest($file) {
+        if (isset($this->manifest))
+            return @$this->manifest[$file] ?: null;
+
+        $this->manifest = $lines = array();
+        $path = $this->get_include_dir() . '/.MANIFEST';
+        if (!is_file($path))
+            return null;
+
+        if (!preg_match_all('/^(\w+) (.+)$/mu', file_get_contents($path),
+            $lines, PREG_PATTERN_ORDER)
+        ) {
+            return null;
+        }
+
+        $this->manifest = array_combine($lines[2], $lines[1]);
+        return @$this->manifest[$file] ?: null;
+    }
+
+    function hashFile($file) {
+        static $hashes = array();
+
+        if (!isset($hashes[$file])) {
+            $md5 = md5_file($file);
+            $sha1 = sha1_file($file);
+            $hash = substr($md5, -20) . substr($sha1, -20);
+            $hashes[$file] = $hash;
+        }
+        return $hashes[$file];
+    }
+
+    function isChanged($source, $hash=false) {
+        $local = str_replace($this->source.'/', '', $source);
+        $hash = $hash ?: $this->hashFile($source);
+        return $this->readManifest($local) != $hash;
+    }
+
+    function updateManifest($file, $hash=false) {
+        $hash = $hash ?: $this->hashFile($file);
+        $local = str_replace($this->source.'/', '', $file);
+        $this->manifest[$local] = $hash;
+    }
+
+    function copyFile($src, $dest, $hash=false, $mode=0644) {
+        $this->updateManifest($src, $hash);
+        return copy($src, $dest) && chmod($dest, $mode);
     }
 
     /**
@@ -116,15 +164,17 @@ class Unpacker extends Module {
             if ($this->exclude($exclude, $file))
                 continue;
             if (is_file($file)) {
-                if (!is_dir($destination) && !$dryrun)
-                    mkdir($destination, 0751, true);
                 $target = $destination . basename($file);
-                if (is_file($target) && (md5_file($target) == md5_file($file)))
+                $hash = $this->hashFile($file);
+                if (is_file($target) && !$this->isChanged($file, $hash))
                     continue;
                 if ($verbose)
                     $this->stdout->write($target."\n");
-                if (!$dryrun)
-                    $this->copyFile($file, $target);
+                if ($dryrun)
+                    continue;
+                if (!is_dir($destination))
+                    mkdir($destination, 0751, true);
+                $this->copyFile($file, $target, $hash);
             }
         }
         if ($recurse) {
@@ -144,7 +194,15 @@ class Unpacker extends Module {
     }
 
     function get_include_dir() {
+        static $location;
+
+        if (isset($location))
+            return $location;
+
         $bootstrap_php = $this->destination . '/bootstrap.php';
+        if (!is_file($bootstrap_php))
+            return @$this->include_path ?: '';
+
         $lines = preg_grep("/define\s*\(\s*'INCLUDE_DIR'/",
             explode("\n", file_get_contents($bootstrap_php)));
 
@@ -153,9 +211,9 @@ class Unpacker extends Module {
         if (!defined('ROOT_DIR'))
             define('ROOT_DIR', rtrim($this->destination, '/').'/');
         foreach ($lines as $line)
-            eval($line);
+            @eval($line);
 
-        return rtrim(INCLUDE_DIR, '/').'/';
+        return $location = rtrim(INCLUDE_DIR, '/').'/';
     }
 
     function run($args, $options) {
@@ -169,7 +227,7 @@ class Unpacker extends Module {
         $upgrade = file_exists("{$this->destination}/main.inc.php");
 
         # Locate the upload folder
-        $upload = $this->find_upload_folder();
+        $upload = $this->source = $this->find_upload_folder();
 
         # Unpack the upload folder to the destination, except the include folder
         if ($upgrade)

setup/cli/package.php

-#!/usr/bin/env php
-<?php
-
-if (php_sapi_name() != 'cli')
-    die("Only command-line packaging is supported");
-
-$stage_folder = "stage";
-$stage_path = dirname(__file__) . '/' . $stage_folder;
-
-function get_osticket_root_path() {
-    # Hop up to the root folder
-    $start = dirname(__file__);
-    for (;;) {
-        if (file_exists($start . '/main.inc.php')) break;
-        $start .= '/..';
-    }
-    return realpath($start);
-}
-
-function run_tests($root) {
-    return (require "$root/setup/test/run-tests.php");
-}
-
-# Check PHP syntax across all php files
-function glob_recursive($pattern, $flags = 0) {
-    $files = glob($pattern, $flags);
-    foreach (glob(dirname($pattern).'/*', GLOB_ONLYDIR|GLOB_NOSORT) as $dir) {
-        $files = array_merge($files,
-            glob_recursive($dir.'/'.basename($pattern), $flags));
-    }
-    return $files;
-}
-
-$root = get_osticket_root_path();
-
-function exclude($pattern, $match) {
-    if (is_array($pattern)) {
-        foreach ($pattern as $p)
-            if (fnmatch($p, $match))
-                return true;
-    }
-    else
-        return fnmatch($pattern, $match);
-    return false;
-}
-
-function package($pattern, $destination, $recurse=false, $exclude=false) {
-    global $root, $stage_path;
-    $search = $root . '/' . $pattern;
-    echo "Packaging " . $search . "\n";
-    foreach (glob($search, GLOB_BRACE|GLOB_NOSORT) as $file) {
-        if (is_file($file)) {
-            if ($exclude && exclude($exclude, $file))
-                continue;
-            if (!is_dir("$stage_path/$destination"))
-                mkdir("$stage_path/$destination", 0777, true);
-            copy($file, $stage_path . '/' . $destination . '/' . basename($file));
-        }
-    }
-    if ($recurse) {
-        foreach (glob(dirname($search).'/*', GLOB_ONLYDIR|GLOB_NOSORT) as $dir) {
-            if ($exclude && exclude($exclude, $dir))
-                continue;
-            package(dirname($pattern).'/'.basename($dir).'/'.basename($pattern),
-                $destination.'/'.basename($dir),
-                $recurse-1, $exclude);
-        }
-    }
-}
-
-# Run tests before continuing
-if (run_tests($root) > 0)
-    die("Regression tests failed. Cowardly refusing to package\n");
-
-# Create the stage folder for the install files
-if (!is_dir($stage_path))
-    mkdir($stage_path);
-else {
-    $dirs = array();
-    foreach (glob_recursive($stage_path . '/*') as $file)
-        if (is_dir($file))
-            $dirs[] = $file;
-        else
-            unlink($file);
-    sort($dirs);
-    foreach (array_reverse($dirs) as $dir)
-        rmdir($dir);
-}
-
-# Source code goes into 'upload'
-mkdir($stage_path . '/upload');
-
-# Load the root directory files
-package("*.php", 'upload/');
-package("web.config", 'upload/');
-
-# Load the client interface
-foreach (array('assets','css','images','js') as $dir)
-    package("$dir/*", "upload/$dir", -1, "*less");
-
-# Load API and pages
-package('api/{,.}*', 'upload/api');
-package('pages/{,.}*', 'upload/pages');
-
-# Load the knowledgebase
-package("kb/*.php", "upload/kb");
-
-# Load the staff interface
-package("scp/*.php", "upload/scp/", -1);
-foreach (array('css','images','js') as $dir)
-    package("scp/$dir/*", "upload/scp/$dir", -1);
-
-# Load in the scripts
-mkdir("$stage_path/scripts/");
-package("setup/scripts/*", "scripts/", -1, "*stage");
-
-# Load the heart of the system
-package("include/{,.}*", "upload/include", -1, array('*ost-config.php', '*.sw[a-z]','plugins/*'));
-
-# Include the installer
-package("setup/*.{php,txt,html}", "upload/setup", -1, array("*scripts","*test","*stage"));
-foreach (array('css','images','js') as $dir)
-    package("setup/$dir/*", "upload/setup/$dir", -1);
-package("setup/inc/streams/*.sql", "upload/setup/inc/streams", -1);
-
-# Load the license and documentation
-package("*.{txt,md}", "");
-
-#Rename markdown as text TODO: Do html version before rename.
-if(($mds = glob("$stage_path/*.md"))) {
-    foreach($mds as $md)
-        rename($md, preg_replace('/\.md$/', '.txt', $md));
-}
-
-# Make an archive of the stage folder
-$version = exec('git describe');
-$hash = exec('git rev-parse HEAD');
-$short = substr($hash, 0, 7);
-
-$pwd = getcwd();
-chdir($stage_path);
-
-// Replace THIS_VERSION in the stage/ folder
-
-shell_exec("sed -ri -e \"
-    s/( *)define\('THIS_VERSION'.*/\\1define('THIS_VERSION', '$version');/
-    s/( *)define\('GIT_VERSION'.*/\\1define('GIT_VERSION', '$short');/
-    \" upload/bootstrap.php");
-shell_exec("find upload -name '*.php' -print0 | xargs -0 sed -i -e '
-    s:<script\(.*\) src=\"\(.*\).js\"></script>:<script\\1 src=\"\\2.js?$short\"></script>:
-    s:<link\(.*\) href=\"\(.*\)\.css\"\(.*\)*/*>:<link\\1 href=\"\\2.css?$short\"\\3>:
-   '");
-shell_exec("find upload -name '*.php' -print0 | xargs -0 sed -i -e \"
-    s/\( *\)ini_set( *'display_errors'[^])]*);/\\1ini_set('display_errors', 0);/
-    s/\( *\)ini_set( *'display_startup_errors'[^])]*);/\\1ini_set('display_startup_errors', 0);/
-    \"");
-
-shell_exec("tar cjf '$pwd/osTicket-$version.tar.bz2' *");
-shell_exec("zip -r '$pwd/osTicket-$version.zip' *");
-
-chdir($pwd);
-?>

setup/doc/package.md

+Creating an osTicket distribution
+=================================
+osTicket is packaged using an included script. Access to the packaging
+system is provided via the `manage.php` CLI app.
+
+The packaging system extends the deployment system in order to remove
+similar code between the two processes. Where possible, the files reported
+to be part of the git repository are used in the packaging process, which
+removes the possibility of experimental files and those ignored by git from
+being added to the distribution.
+
+More information is available via the automated help output.
+
+    php setup/cli/manage.php package --help
+
+Creating the ZIP file
+---------------------
+To package the system using the defaults (as a ZIP file), just invoke the
+packager with no other options.
+
+    php setup/cli/manage.php package