deploy: Add concept of a MANIFEST to the deployment process

This allows files which are edited on the fly during deployment to be
tracked so that they are not unnecessarily deployed again in the next
deployment run. It also allows for more creative deployment strategies using
something like `git ls-files -s`

setup/cli/modules/deploy.php

@@ -87,12 +87,26 @@ class Deployment extends Unpacker {
         }
     }
 
-    function copyFile($src, $dest) {
+    function writeManifest($root) {
+        $lines = array();
+        foreach ($this->manifest as $F=>$H)
+            $lines[] = "$H $F";
+
+        return file_put_contents($root.'/.MANIFEST', implode("\n", $lines));
+    }
+
+    function hashContents($file) {
+        $md5 = md5($file);
+        $sha1 = sha1($file);
+        return substr($md5, -20) . substr($sha1, -20);
+    }
+
+    function getEditedContents($src) {
         static $short = false;
         static $version = false;
 
         if (substr($src, -4) != '.php')
-            return parent::copyFile($src, $dest);
+            return false;
 
         if (!$short) {
             $hash = exec('git rev-parse HEAD');
@@ -103,7 +117,7 @@ class Deployment extends Unpacker {
             $version = exec('git describe');
 
         if (!$short || !$version)
-            return parent::copyFile($src, $dest);
+            return false;
 
         $source = file_get_contents($src);
         $source = preg_replace(':<script(.*) src="([^"]+)\.js"></script>:',
@@ -125,12 +139,64 @@ class Deployment extends Unpacker {
             "$1ini_set('$2', '0'); // Set by installer",
             $source);
 
-        if (!file_put_contents($dest, $source))
-            die("Unable to apply rewrite rules to ".$dest);
+        return $source;
+    }
+
+    function copyFile($source, $dest, $hash=false) {
+        $contents = $this->getEditedContents($source);
+        if ($contents === false)
+            // Regular file
+            return parent::copyFile($source, $dest, $hash);
+
+        if (!file_put_contents($dest, $contents))
+            $this->fail($dest.": Unable to apply rewrite rules");
 
         return true;
     }
 
+    function unpackage($folder, $destination, $recurse=0, $exclude=false) {
+        return parent::unpackage($folder, $destination, $recurse, $exclude);
+
+        // TODO: Consider using `git ls-files` for deployment
+        if (substr($destination, -1) !== '/')
+            $destination .= '/';
+        $source = $this->source;
+        if (substr($source, -1) != '/')
+            $source .= '/';
+
+        $pipes = array();
+        $patterns = array();
+        foreach ((array) $exclude as $x) {
+            $patterns[] = str_replace($source, '', $x);
+        }
+        $exclude = implode(' --exclude-per-directory=', $patterns);
+        if (!($files = proc_open(
+            "git ls-files -s --exclude-standard --exclude-per-directory=$exclude $folder",
+            array(1 => array('pipe', 'w')),
+            $pipes
+        ))) {
+            return parent::unpackage($folder, $destination, $recurse, $exclude);
+        }
+
+        $dryrun = $this->getOption('dry-run', false);
+        $verbose = $this->getOption('verbose') || $dryrun;
+        while ($line = stream_get_line($pipes[1], 255, PHP_EOL)) {
+            list($mode, $hash, , $path) = preg_split('/\s+/', $line);
+            $src = $source.$path;
+            $dst = $destination.$path;
+            if (!$this->isChanged($src, false, $hash))
+                continue;
+            if ($verbose)
+                $this->stdout->write($dst."\n");
+            if ($dryrun)
+                continue;
+            if (!is_dir(dirname($dst)))
+                mkdir(dirname($dst), 0751, true);
+            // TODO: Consider the MODE value
+            $this->copyFile($src, $dst, $hash);
+        }
+    }
+
     function run($args, $options) {
         $this->destination = $args['install-path'];
         if (!is_dir($this->destination))
@@ -150,9 +216,12 @@ class Deployment extends Unpacker {
         $include = rtrim($include, '/').'/';
 
         # Locate the upload folder
-        $root = $this->find_root_folder();
+        $root = $this->source = $this->find_root_folder();
         $rootPattern = str_replace("\\","\\\\", $root); //need for windows case
 
+        # Prime the manifest system
+        $this->readManifest($this->destination.'/.MANIFEST');
+
         $exclusions = array("$rootPattern/include", "$rootPattern/.git*",
             "*.sw[a-z]","*.md", "*.txt");
         if (!$options['setup'])
@@ -177,6 +246,8 @@ class Deployment extends Unpacker {
                 array("ost-config.php","settings.php","plugins/",
                 "*/.htaccess"));
         }
+
+        $this->writeManifest($this->destination);
     }
 }
 

setup/cli/modules/unpack.php

@@ -34,6 +34,10 @@ class Unpacker extends Module {
              main installation path.",
     );
 
+    var $manifest;
+    var $source;
+    var $destination;
+
     function realpath($path) {
         return ($p = realpath($path)) ? $p : $path;
     }
@@ -88,7 +92,51 @@ class Unpacker extends Module {
         return false;
     }
 
-    function copyFile($src, $dest) {
+    function readManifest($file) {
+        if (isset($this->manifest))
+            return @$this->manifest[$file] ?: null;
+
+        $this->manifest = $lines = array();
+        $path = $this->destination . '/.MANIFEST';
+        if (!is_file($path))
+            return null;
+
+        if (!preg_match_all('/^(\w+) (.+)$/mu', file_get_contents($path),
+            $lines, PREG_PATTERN_ORDER)
+        ) {
+            return null;
+        }
+
+        $this->manifest = array_combine($lines[2], $lines[1]);
+        return @$this->manifest[$file] ?: null;
+    }
+
+    function hashFile($file) {
+        static $hashes = array();
+
+        if (!isset($hashes[$file])) {
+            $md5 = md5_file($file);
+            $sha1 = sha1_file($file);
+            $hash = substr($md5, -20) . substr($sha1, -20);
+            $hashes[$file] = $hash;
+        }
+        return $hashes[$file];
+    }
+
+    function isChanged($source, $hash=false) {
+        $local = str_replace($this->source.'/', '', $source);
+        $hash = $hash ?: $this->hashFile($source);
+        return $this->readManifest($local) != $hash;
+    }
+
+    function updateManifest($file, $hash=false) {
+        $hash = $hash ?: $this->hashFile($file);
+        $local = str_replace($this->source.'/', '', $file);
+        $this->manifest[$local] = $hash;
+    }
+
+    function copyFile($src, $dest, $hash=false) {
+        $this->updateManifest($src, $hash);
         return copy($src, $dest);
     }
 
@@ -116,15 +164,17 @@ class Unpacker extends Module {
             if ($this->exclude($exclude, $file))
                 continue;
             if (is_file($file)) {
-                if (!is_dir($destination) && !$dryrun)
-                    mkdir($destination, 0751, true);
                 $target = $destination . basename($file);
-                if (is_file($target) && (md5_file($target) == md5_file($file)))
+                $hash = $this->hashFile($file);
+                if (is_file($target) && !$this->isChanged($file, $hash))
                     continue;
                 if ($verbose)
                     $this->stdout->write($target."\n");
-                if (!$dryrun)
-                    $this->copyFile($file, $target);
+                if ($dryrun)
+                    continue;
+                if (!is_dir($destination))
+                    mkdir($destination, 0751, true);
+                $this->copyFile($file, $target);
             }
         }
         if ($recurse) {
@@ -169,7 +219,7 @@ class Unpacker extends Module {
         $upgrade = file_exists("{$this->destination}/main.inc.php");
 
         # Locate the upload folder
-        $upload = $this->find_upload_folder();
+        $upload = $this->source = $this->find_upload_folder();
 
         # Unpack the upload folder to the destination, except the include folder
         if ($upgrade)