Merge pull request #8952 from Seldaek/parallel-installs

Parallel zip installs

src/Composer/Command/ArchiveCommand.php

@@ -23,6 +23,7 @@ use Composer\Plugin\CommandEvent;
 use Composer\Plugin\PluginEvents;
 use Composer\Util\Filesystem;
 use Composer\Util\Loop;
+use Composer\Util\ProcessExecutor;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
@@ -112,9 +113,10 @@ EOT
             $archiveManager = $composer->getArchiveManager();
         } else {
             $factory = new Factory;
+            $process = new ProcessExecutor();
             $httpDownloader = $factory->createHttpDownloader($io, $config);
-            $downloadManager = $factory->createDownloadManager($io, $config, $httpDownloader);
-            $archiveManager = $factory->createArchiveManager($config, $downloadManager, new Loop($httpDownloader));
+            $downloadManager = $factory->createDownloadManager($io, $config, $httpDownloader, $process);
+            $archiveManager = $factory->createArchiveManager($config, $downloadManager, new Loop($httpDownloader, $process));
         }
 
         if ($packageName) {

src/Composer/Command/CreateProjectCommand.php

@@ -201,6 +201,8 @@ EOT
 
         // install dependencies of the created project
         if ($noInstall === false) {
+            $composer->getInstallationManager()->setOutputProgress(!$noProgress);
+
             $installer = Installer::create($io, $composer);
             $installer->setPreferSource($preferSource)
                 ->setPreferDist($preferDist)
@@ -212,6 +214,10 @@ EOT
                 ->setClassMapAuthoritative($config->get('classmap-authoritative'))
                 ->setApcuAutoloader($config->get('apcu-autoloader'));
 
+            if (!$composer->getLocker()->isLocked()) {
+                $installer->setUpdate(true);
+            }
+
             if ($disablePlugins) {
                 $installer->disablePlugins();
             }
@@ -405,7 +411,8 @@ EOT
             ->setPreferDist($preferDist);
 
         $projectInstaller = new ProjectInstaller($directory, $dm, $fs);
-        $im = $factory->createInstallationManager(new Loop($httpDownloader), $io);
+        $im = $factory->createInstallationManager(new Loop($httpDownloader, $process), $io);
+        $im->setOutputProgress(!$noProgress);
         $im->addInstaller($projectInstaller);
         $im->execute(new InstalledFilesystemRepository(new JsonFile('php://memory')), array(new InstallOperation($package)));
         $im->notifyInstalls($io);

src/Composer/Command/InstallCommand.php

@@ -106,6 +106,8 @@ EOT
 
         $ignorePlatformReqs = $input->getOption('ignore-platform-reqs') ?: ($input->getOption('ignore-platform-req') ?: false);
 
+        $composer->getInstallationManager()->setOutputProgress(!$input->getOption('no-progress'));
+
         $install
             ->setDryRun($input->getOption('dry-run'))
             ->setVerbose($input->getOption('verbose'))

src/Composer/Command/RemoveCommand.php

@@ -220,6 +220,8 @@ EOT
         $commandEvent = new CommandEvent(PluginEvents::COMMAND, 'remove', $input, $output);
         $composer->getEventDispatcher()->dispatch($commandEvent->getName(), $commandEvent);
 
+        $composer->getInstallationManager()->setOutputProgress(!$input->getOption('no-progress'));
+
         $install = Installer::create($io, $composer);
 
         $updateDevMode = !$input->getOption('update-no-dev');

src/Composer/Command/RequireCommand.php

@@ -286,6 +286,8 @@ EOT
         $commandEvent = new CommandEvent(PluginEvents::COMMAND, 'require', $input, $output);
         $composer->getEventDispatcher()->dispatch($commandEvent->getName(), $commandEvent);
 
+        $composer->getInstallationManager()->setOutputProgress(!$input->getOption('no-progress'));
+
         $install = Installer::create($io, $composer);
 
         $ignorePlatformReqs = $input->getOption('ignore-platform-reqs') ?: ($input->getOption('ignore-platform-req') ?: false);

src/Composer/Command/UpdateCommand.php

@@ -180,6 +180,8 @@ EOT
         $commandEvent = new CommandEvent(PluginEvents::COMMAND, 'update', $input, $output);
         $composer->getEventDispatcher()->dispatch($commandEvent->getName(), $commandEvent);
 
+        $composer->getInstallationManager()->setOutputProgress(!$input->getOption('no-progress'));
+
         $install = Installer::create($io, $composer);
 
         $config = $composer->getConfig();

src/Composer/Downloader/ArchiveDownloader.php

@@ -16,6 +16,7 @@ use Composer\Package\PackageInterface;
 use Symfony\Component\Finder\Finder;
 use Composer\IO\IOInterface;
 use Composer\Exception\IrrecoverableDownloadException;
+use React\Promise\PromiseInterface;
 
 /**
  * Base downloader for archives
@@ -28,14 +29,12 @@ abstract class ArchiveDownloader extends FileDownloader
 {
     public function download(PackageInterface $package, $path, PackageInterface $prevPackage = null, $output = true)
     {
-        $res = parent::download($package, $path, $prevPackage, $output);
-
         // if not downgrading and the dir already exists it seems we have an inconsistent state in the vendor dir and the user should fix it
         if (!$prevPackage && is_dir($path) && !$this->filesystem->isDirEmpty($path)) {
             throw new IrrecoverableDownloadException('Expected empty path to extract '.$package.' into but directory exists: '.$path);
         }
 
-        return $res;
+        return parent::download($package, $path, $prevPackage, $output);
     }
 
     /**
@@ -60,26 +59,64 @@ abstract class ArchiveDownloader extends FileDownloader
             $temporaryDir = $this->config->get('vendor-dir').'/composer/'.substr(md5(uniqid('', true)), 0, 8);
         } while (is_dir($temporaryDir));
 
+        $this->addCleanupPath($package, $temporaryDir);
+        $this->addCleanupPath($package, $path);
+
+        $this->filesystem->ensureDirectoryExists($temporaryDir);
         $fileName = $this->getFileName($package, $path);
 
+        $filesystem = $this->filesystem;
+        $self = $this;
+
+        $cleanup = function () use ($path, $filesystem, $temporaryDir, $package, $self) {
+            // remove cache if the file was corrupted
+            $self->clearLastCacheWrite($package);
+
+            // clean up
+            $filesystem->removeDirectory($path);
+            $filesystem->removeDirectory($temporaryDir);
+            $self->removeCleanupPath($package, $temporaryDir);
+            $self->removeCleanupPath($package, $path);
+        };
+
+        $promise = null;
         try {
-            $this->filesystem->ensureDirectoryExists($temporaryDir);
-            try {
-                $this->extract($package, $fileName, $temporaryDir);
-            } catch (\Exception $e) {
-                // remove cache if the file was corrupted
-                parent::clearLastCacheWrite($package);
-                throw $e;
-            }
+            $promise = $this->extract($package, $fileName, $temporaryDir);
+        } catch (\Exception $e) {
+            $cleanup();
+            throw $e;
+        }
 
-            $this->filesystem->unlink($fileName);
+        if (!$promise instanceof PromiseInterface) {
+            $promise = \React\Promise\resolve();
+        }
+
+        return $promise->then(function () use ($self, $package, $filesystem, $fileName, $temporaryDir, $path) {
+            $filesystem->unlink($fileName);
+
+            /**
+             * Returns the folder content, excluding .DS_Store
+             *
+             * @param  string         $dir Directory
+             * @return \SplFileInfo[]
+             */
+            $getFolderContent = function ($dir) {
+                $finder = Finder::create()
+                    ->ignoreVCS(false)
+                    ->ignoreDotFiles(false)
+                    ->notName('.DS_Store')
+                    ->depth(0)
+                    ->in($dir);
+
+                return iterator_to_array($finder);
+            };
 
             $renameAsOne = false;
-            if (!file_exists($path) || ($this->filesystem->isDirEmpty($path) && $this->filesystem->removeDirectory($path))) {
+            if (!file_exists($path) || ($filesystem->isDirEmpty($path) && $filesystem->removeDirectory($path))) {
                 $renameAsOne = true;
             }
 
-            $contentDir = $this->getFolderContent($temporaryDir);
+            $contentDir = $getFolderContent($temporaryDir);
             $singleDirAtTopLevel = 1 === count($contentDir) && is_dir(reset($contentDir));
 
             if ($renameAsOne) {
@@ -89,28 +126,28 @@ abstract class ArchiveDownloader extends FileDownloader
                 } else {
                     $extractedDir = $temporaryDir;
                 }
-                $this->filesystem->rename($extractedDir, $path);
+                $filesystem->rename($extractedDir, $path);
             } else {
                 // only one dir in the archive, extract its contents out of it
                 if ($singleDirAtTopLevel) {
-                    $contentDir = $this->getFolderContent((string) reset($contentDir));
+                    $contentDir = $getFolderContent((string) reset($contentDir));
                 }
 
                 // move files back out of the temp dir
                 foreach ($contentDir as $file) {
                     $file = (string) $file;
-                    $this->filesystem->rename($file, $path . '/' . basename($file));
+                    $filesystem->rename($file, $path . '/' . basename($file));
                 }
             }
 
-            $this->filesystem->removeDirectory($temporaryDir);
-        } catch (\Exception $e) {
-            // clean up
-            $this->filesystem->removeDirectory($path);
-            $this->filesystem->removeDirectory($temporaryDir);
+            $filesystem->removeDirectory($temporaryDir);
+            $self->removeCleanupPath($package, $temporaryDir);
+            $self->removeCleanupPath($package, $path);
+        }, function ($e) use ($cleanup) {
+            $cleanup();
 
             throw $e;
-        }
+        });
     }
 
     /**
@@ -119,25 +156,8 @@ abstract class ArchiveDownloader extends FileDownloader
      * @param string $file Extracted file
      * @param string $path Directory
      *
+     * @return PromiseInterface|null
      * @throws \UnexpectedValueException If can not extract downloaded file to path
      */
     abstract protected function extract(PackageInterface $package, $file, $path);
-
-    /**
-     * Returns the folder content, excluding dotfiles
-     *
-     * @param  string         $dir Directory
-     * @return \SplFileInfo[]
-     */
-    private function getFolderContent($dir)
-    {
-        $finder = Finder::create()
-            ->ignoreVCS(false)
-            ->ignoreDotFiles(false)
-            ->notName('.DS_Store')
-            ->depth(0)
-            ->in($dir);
-
-        return iterator_to_array($finder);
-    }
 }

src/Composer/Downloader/FileDownloader.php

@@ -27,7 +27,9 @@ use Composer\EventDispatcher\EventDispatcher;
 use Composer\Util\Filesystem;
 use Composer\Util\HttpDownloader;
 use Composer\Util\Url as UrlUtil;
+use Composer\Util\ProcessExecutor;
 use Composer\Downloader\TransportException;
+use React\Promise\PromiseInterface;
 
 /**
  * Base downloader for files
@@ -51,10 +53,13 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface
     protected $cache;
     /** @var EventDispatcher */
     protected $eventDispatcher;
+    /** @var ProcessExecutor */
+    protected $process;
     /**
      * @private this is only public for php 5.3 support in closures
      */
     public $lastCacheWrites = array();
+    private $additionalCleanupPaths = array();
 
     /**
      * Constructor.
@@ -66,14 +71,15 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface
      * @param Cache            $cache           Cache instance
      * @param Filesystem       $filesystem      The filesystem
      */
-    public function __construct(IOInterface $io, Config $config, HttpDownloader $httpDownloader, EventDispatcher $eventDispatcher = null, Cache $cache = null, Filesystem $filesystem = null)
+    public function __construct(IOInterface $io, Config $config, HttpDownloader $httpDownloader, EventDispatcher $eventDispatcher = null, Cache $cache = null, Filesystem $filesystem = null, ProcessExecutor $process = null)
     {
         $this->io = $io;
         $this->config = $config;
         $this->eventDispatcher = $eventDispatcher;
         $this->httpDownloader = $httpDownloader;
-        $this->filesystem = $filesystem ?: new Filesystem();
         $this->cache = $cache;
+        $this->process = $process ?: new ProcessExecutor($io);
+        $this->filesystem = $filesystem ?: new Filesystem($this->process);
 
         if ($this->cache && $this->cache->gcIsNecessary()) {
             $this->cache->gc($config->get('cache-files-ttl'), $config->get('cache-files-maxsize'));
@@ -258,6 +264,12 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface
             $path,
         );
 
+        if (isset($this->additionalCleanupPaths[$package->getName()])) {
+            foreach ($this->additionalCleanupPaths[$package->getName()] as $path) {
+                $this->filesystem->remove($path);
+            }
+        }
+
         foreach ($dirsToCleanUp as $dir) {
             if (is_dir($dir) && $this->filesystem->isDirEmpty($dir)) {
                 $this->filesystem->removeDirectory($dir);
@@ -291,6 +303,29 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface
         }
     }
 
+    /**
+     * TODO mark private in v3
+     * @protected This is public due to PHP 5.3
+     */
+    public function addCleanupPath(PackageInterface $package, $path)
+    {
+        $this->additionalCleanupPaths[$package->getName()][] = $path;
+    }
+
+    /**
+     * TODO mark private in v3
+     * @protected This is public due to PHP 5.3
+     */
+    public function removeCleanupPath(PackageInterface $package, $path)
+    {
+        if (isset($this->additionalCleanupPaths[$package->getName()])) {
+            $idx = array_search($path, $this->additionalCleanupPaths[$package->getName()]);
+            if (false !== $idx) {
+                unset($this->additionalCleanupPaths[$package->getName()][$idx]);
+            }
+        }
+    }
+
     /**
      * {@inheritDoc}
      */
@@ -303,10 +338,19 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface
         $actionName = VersionParser::isUpgrade($initial->getVersion(), $target->getVersion()) ? 'Upgrading' : 'Downgrading';
         $this->io->writeError("  - " . $actionName . " <info>" . $name . "</info> (<comment>" . $from . "</comment> => <comment>" . $to . "</comment>): ", false);
 
-        $this->remove($initial, $path, false);
-        $this->install($target, $path, false);
+        $promise = $this->remove($initial, $path, false);
+        if (!$promise instanceof PromiseInterface) {
+            $promise = \React\Promise\resolve();
+        }
+        $self = $this;
+        $io = $this->io;
+
+        return $promise->then(function () use ($self, $target, $path, $io) {
+            $promise = $self->install($target, $path, false);
+            $io->writeError('');
 
-        $this->io->writeError('');
+            return $promise;
+        });
     }
 
     /**
@@ -380,9 +424,10 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface
         $output = '';
 
         try {
-            $res = $this->download($package, $targetDir.'_compare', null, false);
+            $this->download($package, $targetDir.'_compare', null, false);
             $this->httpDownloader->wait();
-            $res = $this->install($package, $targetDir.'_compare', false);
+            $this->install($package, $targetDir.'_compare', false);
+            $this->process->wait();
 
             $comparer = new Comparer();
             $comparer->setSource($targetDir.'_compare');

src/Composer/Downloader/GzipDownloader.php

@@ -29,15 +29,6 @@ use Composer\Util\Filesystem;
  */
 class GzipDownloader extends ArchiveDownloader
 {
-    /** @var ProcessExecutor */
-    protected $process;
-
-    public function __construct(IOInterface $io, Config $config, HttpDownloader $downloader, EventDispatcher $eventDispatcher = null, Cache $cache = null, Filesystem $fs = null, ProcessExecutor $process = null)
-    {
-        $this->process = $process ?: new ProcessExecutor($io);
-        parent::__construct($io, $config, $downloader, $eventDispatcher, $cache, $fs);
-    }
-
     protected function extract(PackageInterface $package, $file, $path)
     {
         $filename = pathinfo(parse_url($package->getDistUrl(), PHP_URL_PATH), PATHINFO_FILENAME);

src/Composer/Downloader/PathDownloader.php

@@ -39,15 +39,6 @@ class PathDownloader extends FileDownloader implements VcsCapableDownloaderInter
     const STRATEGY_SYMLINK = 10;
     const STRATEGY_MIRROR = 20;
 
-    /** @var ProcessExecutor */
-    private $process;
-
-    public function __construct(IOInterface $io, Config $config, HttpDownloader $downloader, EventDispatcher $eventDispatcher = null, Cache $cache = null, Filesystem $fs = null, ProcessExecutor $process = null)
-    {
-        $this->process = $process ?: new ProcessExecutor($io);
-        parent::__construct($io, $config, $downloader, $eventDispatcher, $cache, $fs);
-    }
-
     /**
      * {@inheritdoc}
      */

src/Composer/Downloader/RarDownloader.php

@@ -33,15 +33,6 @@ use RarArchive;
  */
 class RarDownloader extends ArchiveDownloader
 {
-    /** @var ProcessExecutor */
-    protected $process;
-
-    public function __construct(IOInterface $io, Config $config, HttpDownloader $downloader, EventDispatcher $eventDispatcher = null, Cache $cache = null, Filesystem $fs = null, ProcessExecutor $process = null)
-    {
-        $this->process = $process ?: new ProcessExecutor($io);
-        parent::__construct($io, $config, $downloader, $eventDispatcher, $cache, $fs);
-    }
-
     protected function extract(PackageInterface $package, $file, $path)
     {
         $processError = null;

src/Composer/Downloader/XzDownloader.php

@@ -29,16 +29,6 @@ use Composer\Util\Filesystem;
  */
 class XzDownloader extends ArchiveDownloader
 {
-    /** @var ProcessExecutor */
-    protected $process;
-
-    public function __construct(IOInterface $io, Config $config, HttpDownloader $downloader, EventDispatcher $eventDispatcher = null, Cache $cache = null, Filesystem $fs = null, ProcessExecutor $process = null)
-    {
-        $this->process = $process ?: new ProcessExecutor($io);
-
-        parent::__construct($io, $config, $downloader, $eventDispatcher, $cache, $fs);
-    }
-
     protected function extract(PackageInterface $package, $file, $path)
     {
         $command = 'tar -xJf ' . ProcessExecutor::escape($file) . ' -C ' . ProcessExecutor::escape($path);

src/Composer/Downloader/ZipDownloader.php

@@ -34,17 +34,9 @@ class ZipDownloader extends ArchiveDownloader
     private static $hasZipArchive;
     private static $isWindows;
 
-    /** @var ProcessExecutor */
-    protected $process;
     /** @var ZipArchive|null */
     private $zipArchiveObject;
 
-    public function __construct(IOInterface $io, Config $config, HttpDownloader $downloader, EventDispatcher $eventDispatcher = null, Cache $cache = null, Filesystem $fs = null, ProcessExecutor $process = null)
-    {
-        $this->process = $process ?: new ProcessExecutor($io);
-        parent::__construct($io, $config, $downloader, $eventDispatcher, $cache, $fs);
-    }
-
     /**
      * {@inheritDoc}
      */
@@ -86,9 +78,8 @@ class ZipDownloader extends ArchiveDownloader
      * @param  string $file         File to extract
      * @param  string $path         Path where to extract file
      * @param  bool   $isLastChance If true it is called as a fallback and should throw an exception
-     * @return bool   Success status
      */
-    protected function extractWithSystemUnzip($file, $path, $isLastChance)
+    private function extractWithSystemUnzip(PackageInterface $package, $file, $path, $isLastChance, $async = false)
     {
         if (!self::$hasZipArchive) {
             // Force Exception throwing if the Other alternative is not available
@@ -98,18 +89,47 @@ class ZipDownloader extends ArchiveDownloader
         if (!self::$hasSystemUnzip && !$isLastChance) {
             // This was call as the favorite extract way, but is not available
             // We switch to the alternative
-            return $this->extractWithZipArchive($file, $path, true);
+            return $this->extractWithZipArchive($package, $file, $path, true);
         }
 
-        $processError = null;
         // When called after a ZipArchive failed, perhaps there is some files to overwrite
         $overwrite = $isLastChance ? '-o' : '';
-
         $command = 'unzip -qq '.$overwrite.' '.ProcessExecutor::escape($file).' -d '.ProcessExecutor::escape($path);
 
+        if ($async) {
+            $self = $this;
+            $io = $this->io;
+            $tryFallback = function ($processError) use ($isLastChance, $io, $self, $file, $path, $package) {
+                if ($isLastChance) {
+                    throw $processError;
+                }
+
+                $io->writeError('    <warning>'.$processError->getMessage().'</warning>');
+                $io->writeError('    The archive may contain identical file names with different capitalization (which fails on case insensitive filesystems)');
+                $io->writeError('    Unzip with unzip command failed, falling back to ZipArchive class');
+
+                return $self->extractWithZipArchive($package, $file, $path, true);
+            };
+
+            try {
+                $promise = $this->process->executeAsync($command);
+
+                return $promise->then(function ($process) use ($tryFallback, $command, $package) {
+                    if (!$process->isSuccessful()) {
+                        return $tryFallback(new \RuntimeException('Failed to extract '.$package->getName().': ('.$process->getExitCode().') '.$command."\n\n".$process->getErrorOutput()));
+                    }
+                });
+            } catch (\Exception $e) {
+                return $tryFallback($e);
+            } catch (\Throwable $e) {
+                return $tryFallback($e);
+            }
+        }
+
+        $processError = null;
         try {
             if (0 === $exitCode = $this->process->execute($command, $ignoredOutput)) {
-                return true;
+                return \React\Promise\resolve();
             }
 
             $processError = new \RuntimeException('Failed to execute ('.$exitCode.') '.$command."\n\n".$this->process->getErrorOutput());
@@ -121,11 +141,11 @@ class ZipDownloader extends ArchiveDownloader
             throw $processError;
         }
 
-        $this->io->writeError('    '.$processError->getMessage());
+        $this->io->writeError('    <warning>'.$processError->getMessage().'</warning>');
         $this->io->writeError('    The archive may contain identical file names with different capitalization (which fails on case insensitive filesystems)');
         $this->io->writeError('    Unzip with unzip command failed, falling back to ZipArchive class');
 
-        return $this->extractWithZipArchive($file, $path, true);
+        return $this->extractWithZipArchive($package, $file, $path, true);
     }
 
     /**
@@ -134,9 +154,11 @@ class ZipDownloader extends ArchiveDownloader
      * @param  string $file         File to extract
      * @param  string $path         Path where to extract file
      * @param  bool   $isLastChance If true it is called as a fallback and should throw an exception
-     * @return bool   Success status
+     *
+     * TODO v3 should make this private once we can drop PHP 5.3 support
+     * @protected
      */
-    protected function extractWithZipArchive($file, $path, $isLastChance)
+    public function extractWithZipArchive(PackageInterface $package, $file, $path, $isLastChance)
     {
         if (!self::$hasSystemUnzip) {
             // Force Exception throwing if the Other alternative is not available
@@ -146,7 +168,7 @@ class ZipDownloader extends ArchiveDownloader
         if (!self::$hasZipArchive && !$isLastChance) {
             // This was call as the favorite extract way, but is not available
             // We switch to the alternative
-            return $this->extractWithSystemUnzip($file, $path, true);
+            return $this->extractWithSystemUnzip($package, $file, $path, true);
         }
 
         $processError = null;
@@ -159,7 +181,7 @@ class ZipDownloader extends ArchiveDownloader
                 if (true === $extractResult) {
                     $zipArchive->close();
 
-                    return true;
+                    return \React\Promise\resolve();
                 }
 
                 $processError = new \RuntimeException(rtrim("There was an error extracting the ZIP file, it is either corrupted or using an invalid format.\n"));
@@ -170,16 +192,18 @@ class ZipDownloader extends ArchiveDownloader
             $processError = new \RuntimeException('The archive may contain identical file names with different capitalization (which fails on case insensitive filesystems): '.$e->getMessage(), 0, $e);
         } catch (\Exception $e) {
             $processError = $e;
+        } catch (\Throwable $e) {
+            $processError = $e;
         }
 
         if ($isLastChance) {
             throw $processError;
         }
 
-        $this->io->writeError('    '.$processError->getMessage());
+        $this->io->writeError('    <warning>'.$processError->getMessage().'</warning>');
         $this->io->writeError('    Unzip with ZipArchive class failed, falling back to unzip command');
 
-        return $this->extractWithSystemUnzip($file, $path, true);
+        return $this->extractWithSystemUnzip($package, $file, $path, true);
     }
 
     /**
@@ -192,10 +216,10 @@ class ZipDownloader extends ArchiveDownloader
     {
         // Each extract calls its alternative if not available or fails
         if (self::$isWindows) {
-            $this->extractWithZipArchive($file, $path, false);
-        } else {
-            $this->extractWithSystemUnzip($file, $path, false);
+            return $this->extractWithZipArchive($package, $file, $path, false);
         }
+
+        return $this->extractWithSystemUnzip($package, $file, $path, false, true);
     }
 
     /**

src/Composer/Factory.php

@@ -336,7 +336,7 @@ class Factory
 
         $httpDownloader = self::createHttpDownloader($io, $config);
         $process = new ProcessExecutor($io);
-        $loop = new Loop($httpDownloader);
+        $loop = new Loop($httpDownloader, $process);
         $composer->setLoop($loop);
 
         // initialize event dispatcher
@@ -495,11 +495,11 @@ class Factory
         $dm->setDownloader('perforce', new Downloader\PerforceDownloader($io, $config, $process, $fs));
         $dm->setDownloader('zip', new Downloader\ZipDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $fs, $process));
         $dm->setDownloader('rar', new Downloader\RarDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $fs, $process));
-        $dm->setDownloader('tar', new Downloader\TarDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $fs));
+        $dm->setDownloader('tar', new Downloader\TarDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $fs, $process));
         $dm->setDownloader('gzip', new Downloader\GzipDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $fs, $process));
         $dm->setDownloader('xz', new Downloader\XzDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $fs, $process));
-        $dm->setDownloader('phar', new Downloader\PharDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $fs));
-        $dm->setDownloader('file', new Downloader\FileDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $fs));
+        $dm->setDownloader('phar', new Downloader\PharDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $fs, $process));
+        $dm->setDownloader('file', new Downloader\FileDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $fs, $process));
         $dm->setDownloader('path', new Downloader\PathDownloader($io, $config, $httpDownloader, $eventDispatcher, $cache, $fs, $process));
 
         return $dm;

src/Composer/IO/ConsoleIO.php

@@ -14,6 +14,7 @@ namespace Composer\IO;
 
 use Composer\Question\StrictConfirmationQuestion;
 use Symfony\Component\Console\Helper\HelperSet;
+use Symfony\Component\Console\Helper\ProgressBar;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\ConsoleOutputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -253,6 +254,15 @@ class ConsoleIO extends BaseIO
         }
     }
 
+    /**
+     * @param int $max
+     * @return ProgressBar
+     */
+    public function getProgressBar($max = 0)
+    {
+        return new ProgressBar($this->getErrorOutput(), $max);
+    }
+
     /**
      * {@inheritDoc}
      */

src/Composer/Installer.php

@@ -685,7 +685,7 @@ class Installer
         }
 
         if ($this->executeOperations) {
-            $this->installationManager->execute($localRepo, $localRepoTransaction->getOperations(), $this->devMode);
+            $this->installationManager->execute($localRepo, $localRepoTransaction->getOperations(), $this->devMode, $this->runScripts);
         } else {
             foreach ($localRepoTransaction->getOperations() as $operation) {
                 // output op, but alias op only in debug verbosity

src/Composer/Installer/InstallationManager.php

@@ -13,6 +13,7 @@
 namespace Composer\Installer;
 
 use Composer\IO\IOInterface;
+use Composer\IO\ConsoleIO;
 use Composer\Package\PackageInterface;
 use Composer\Package\AliasPackage;
 use Composer\Repository\RepositoryInterface;
@@ -49,6 +50,8 @@ class InstallationManager
     private $io;
     /** @var EventDispatcher */
     private $eventDispatcher;
+    /** @var bool */
+    private $outputProgress;
 
     public function __construct(Loop $loop, IOInterface $io, EventDispatcher $eventDispatcher = null)
     {
@@ -173,7 +176,7 @@ class InstallationManager
      * @param RepositoryInterface  $repo       repository in which to add/remove/update packages
      * @param OperationInterface[] $operations operations to execute
      * @param bool                 $devMode    whether the install is being run in dev mode
-     * @param bool                 $operation  whether to dispatch script events
+     * @param bool                 $runScripts whether to dispatch script events
      */
     public function execute(RepositoryInterface $repo, array $operations, $devMode = true, $runScripts = true)
     {
@@ -184,6 +187,8 @@ class InstallationManager
         $runCleanup = function () use (&$cleanupPromises, $loop) {
             $promises = array();
 
+            $loop->abortJobs();
+
             foreach ($cleanupPromises as $cleanup) {
                 $promises[] = new \React\Promise\Promise(function ($resolve, $reject) use ($cleanup) {
                     $promise = $cleanup();
@@ -266,69 +271,44 @@ class InstallationManager
 
             // execute all downloads first
             if (!empty($promises)) {
-                $this->loop->wait($promises);
+                $progress = null;
+                if ($this->outputProgress && $this->io instanceof ConsoleIO && !$this->io->isDebug() && count($promises) > 1) {
+                    $progress = $this->io->getProgressBar();
+                }
+                $this->loop->wait($promises, $progress);
+                if ($progress) {
+                    $progress->clear();
+                }
             }
 
+            // execute operations in batches to make sure every plugin is installed in the
+            // right order and activated before the packages depending on it are installed
+            $batches = array();
+            $batch = array();
             foreach ($operations as $index => $operation) {
-                $opType = $operation->getOperationType();
-
-                // ignoring alias ops as they don't need to execute anything
-                if (!in_array($opType, array('update', 'install', 'uninstall'))) {
-                    // output alias ops in debug verbosity as they have no output otherwise
-                    if ($this->io->isDebug()) {
-                        $this->io->writeError('  - ' . $operation->show(false));
+                if (in_array($operation->getOperationType(), array('update', 'install'), true)) {
+                    $package = $operation->getOperationType() === 'update' ? $operation->getTargetPackage() : $operation->getPackage();
+                    if ($package->getType() === 'composer-plugin' || $package->getType() === 'composer-installer') {
+                        if ($batch) {
+                            $batches[] = $batch;
+                        }
+                        unset($operations[$index]);
+                        $batches[] = array($index => $operation);
+                        $batch = array();
+
+                        continue;
                     }
-                    $this->$opType($repo, $operation);
-
-                    continue;
                 }
+                unset($operations[$index]);
+                $batch[$index] = $operation;
+            }
 
-                if ($opType === 'update') {
-                    $package = $operation->getTargetPackage();
-                    $initialPackage = $operation->getInitialPackage();
-                } else {
-                    $package = $operation->getPackage();
-                    $initialPackage = null;
-                }
-                $installer = $this->getInstaller($package->getType());
-
-                $event = 'Composer\Installer\PackageEvents::PRE_PACKAGE_'.strtoupper($opType);
-                if (defined($event) && $runScripts && $this->eventDispatcher) {
-                    $this->eventDispatcher->dispatchPackageEvent(constant($event), $devMode, $repo, $operations, $operation);
-                }
-
-                $dispatcher = $this->eventDispatcher;
-                $installManager = $this;
-                $loop = $this->loop;
-                $io = $this->io;
-
-                $promise = $installer->prepare($opType, $package, $initialPackage);
-                if (!$promise instanceof PromiseInterface) {
-                    $promise = \React\Promise\resolve();
-                }
-
-                $promise = $promise->then(function () use ($opType, $installManager, $repo, $operation) {
-                    return $installManager->$opType($repo, $operation);
-                })->then($cleanupPromises[$index])
-                ->then(function () use ($opType, $runScripts, $dispatcher, $installManager, $devMode, $repo, $operations, $operation) {
-                    $repo->write($devMode, $installManager);
-
-                    $event = 'Composer\Installer\PackageEvents::POST_PACKAGE_'.strtoupper($opType);
-                    if (defined($event) && $runScripts && $dispatcher) {
-                        $dispatcher->dispatchPackageEvent(constant($event), $devMode, $repo, $operations, $operation);
-                    }
-                }, function ($e) use ($opType, $package, $io) {
-                    $io->writeError('    <error>' . ucfirst($opType) .' of '.$package->getPrettyName().' failed</error>');
-
-                    throw $e;
-                });
-
-                $promises[] = $promise;
+            if ($batch) {
+                $batches[] = $batch;
             }
 
-            // execute all prepare => installs/updates/removes => cleanup steps
-            if (!empty($promises)) {
-                $this->loop->wait($promises);
+            foreach ($batches as $batch) {
+                $this->executeBatch($repo, $batch, $cleanupPromises, $devMode, $runScripts);
             }
         } catch (\Exception $e) {
             $runCleanup();
@@ -356,6 +336,77 @@ class InstallationManager
         $repo->write($devMode, $this);
     }
 
+    private function executeBatch(RepositoryInterface $repo, array $operations, array $cleanupPromises, $devMode, $runScripts)
+    {
+        foreach ($operations as $index => $operation) {
+            $opType = $operation->getOperationType();
+
+            // ignoring alias ops as they don't need to execute anything
+            if (!in_array($opType, array('update', 'install', 'uninstall'))) {
+                // output alias ops in debug verbosity as they have no output otherwise
+                if ($this->io->isDebug()) {
+                    $this->io->writeError('  - ' . $operation->show(false));
+                }
+                $this->$opType($repo, $operation);
+
+                continue;
+            }
+
+            if ($opType === 'update') {
+                $package = $operation->getTargetPackage();
+                $initialPackage = $operation->getInitialPackage();
+            } else {
+                $package = $operation->getPackage();
+                $initialPackage = null;
+            }
+            $installer = $this->getInstaller($package->getType());
+
+            $event = 'Composer\Installer\PackageEvents::PRE_PACKAGE_'.strtoupper($opType);
+            if (defined($event) && $runScripts && $this->eventDispatcher) {
+                $this->eventDispatcher->dispatchPackageEvent(constant($event), $devMode, $repo, $operations, $operation);
+            }
+
+            $dispatcher = $this->eventDispatcher;
+            $installManager = $this;
+            $io = $this->io;
+
+            $promise = $installer->prepare($opType, $package, $initialPackage);
+            if (!$promise instanceof PromiseInterface) {
+                $promise = \React\Promise\resolve();
+            }
+
+            $promise = $promise->then(function () use ($opType, $installManager, $repo, $operation) {
+                return $installManager->$opType($repo, $operation);
+            })->then($cleanupPromises[$index])
+            ->then(function () use ($opType, $runScripts, $dispatcher, $installManager, $devMode, $repo, $operations, $operation) {
+                $repo->write($devMode, $installManager);
+
+                $event = 'Composer\Installer\PackageEvents::POST_PACKAGE_'.strtoupper($opType);
+                if (defined($event) && $runScripts && $dispatcher) {
+                    $dispatcher->dispatchPackageEvent(constant($event), $devMode, $repo, $operations, $operation);
+                }
+            }, function ($e) use ($opType, $package, $io) {
+                $io->writeError('    <error>' . ucfirst($opType) .' of '.$package->getPrettyName().' failed</error>');
+
+                throw $e;
+            });
+
+            $promises[] = $promise;
+        }
+
+        // execute all prepare => installs/updates/removes => cleanup steps
+        if (!empty($promises)) {
+            $progress = null;
+            if ($this->outputProgress && $this->io instanceof ConsoleIO && !$this->io->isDebug() && count($promises) > 1) {
+                $progress = $this->io->getProgressBar();
+            }
+            $this->loop->wait($promises, $progress);
+            if ($progress) {
+                $progress->clear();
+            }
+        }
+    }
+
     /**
      * Executes install operation.
      *
@@ -454,6 +505,11 @@ class InstallationManager
         return $installer->getInstallPath($package);
     }
 
+    public function setOutputProgress($outputProgress)
+    {
+        $this->outputProgress = $outputProgress;
+    }
+
     public function notifyInstalls(IOInterface $io)
     {
         foreach ($this->notifiablePackages as $repoUrl => $packages) {

src/Composer/Util/Http/CurlDownloader.php

@@ -23,6 +23,7 @@ use Composer\Util\HttpDownloader;
 use React\Promise\Promise;
 
 /**
+ * @internal
  * @author Jordi Boggiano <j.boggiano@seld.be>
  * @author Nicolas Grekas <p@tchwork.com>
  */
@@ -90,6 +91,9 @@ class CurlDownloader
         $this->authHelper = new AuthHelper($io, $config);
     }
 
+    /**
+     * @return int internal job id
+     */
     public function download($resolve, $reject, $origin, $url, $options, $copyTo = null)
     {
         $attributes = array();
@@ -101,6 +105,9 @@ class CurlDownloader
         return $this->initDownload($resolve, $reject, $origin, $url, $options, $copyTo, $attributes);
     }
 
+    /**
+     * @return int internal job id
+     */
     private function initDownload($resolve, $reject, $origin, $url, $options, $copyTo = null, array $attributes = array())
     {
         $attributes = array_merge(array(
@@ -199,8 +206,29 @@ class CurlDownloader
         }
 
         $this->checkCurlResult(curl_multi_add_handle($this->multiHandle, $curlHandle));
-// TODO progress
+        // TODO progress
         //$params['notification'](STREAM_NOTIFY_RESOLVE, STREAM_NOTIFY_SEVERITY_INFO, '', 0, 0, 0, false);
+
+        return (int) $curlHandle;
+    }
+
+    public function abortRequest($id)
+    {
+        if (isset($this->jobs[$id]) && isset($this->jobs[$id]['handle'])) {
+            $job = $this->jobs[$id];
+            curl_multi_remove_handle($this->multiHandle, $job['handle']);
+            curl_close($job['handle']);
+            if (is_resource($job['headerHandle'])) {
+                fclose($job['headerHandle']);
+            }
+            if (is_resource($job['bodyHandle'])) {
+                fclose($job['bodyHandle']);
+            }
+            if ($job['filename']) {
+                @unlink($job['filename'].'~');
+            }
+            unset($this->jobs[$id]);
+        }
     }
 
     public function tick()
@@ -235,7 +263,7 @@ class CurlDownloader
             $statusCode = null;
             $response = null;
             try {
-// TODO progress
+                // TODO progress
                 //$this->onProgress($curlHandle, $job['callback'], $progress, $job['progress']);
                 if (CURLE_OK !== $errno || $error) {
                     throw new TransportException($error);
@@ -285,8 +313,6 @@ class CurlDownloader
                 // fail 4xx and 5xx responses and capture the response
                 if ($statusCode >= 400 && $statusCode <= 599) {
                     throw $this->failResponse($job, $response, $response->getStatusMessage());
-// TODO progress
-//                        $this->io->overwriteError("Downloading (<error>failed</error>)", false);
                 }
 
                 if ($job['attributes']['storeAuth']) {

src/Composer/Util/HttpDownloader.php

@@ -31,6 +31,7 @@ class HttpDownloader
     const STATUS_STARTED = 2;
     const STATUS_COMPLETED = 3;
     const STATUS_FAILED = 4;
+    const STATUS_ABORTED = 5;
 
     private $io;
     private $config;
@@ -44,6 +45,7 @@ class HttpDownloader
     private $rfs;
     private $idGen = 0;
     private $disabled;
+    private $allowAsync = false;
 
     /**
      * @param IOInterface $io         The IO instance
@@ -139,6 +141,10 @@ class HttpDownloader
             'origin' => Url::getOrigin($this->config, $request['url']),
         );
 
+        if (!$sync && !$this->allowAsync) {
+            throw new \LogicException('You must use the HttpDownloader instance which is part of a Composer\Loop instance to be able to run async http requests');
+        }
+
         // capture username/password from URL if there is one
         if (preg_match('{^https?://([^:/]+):([^@/]+)@([^/]+)}i', $request['url'], $match)) {
             $this->io->setAuthentication($job['origin'], rawurldecode($match[1]), rawurldecode($match[2]));
@@ -179,8 +185,20 @@ class HttpDownloader
 
         $downloader = $this;
         $io = $this->io;
+        $curl = $this->curl;
 
-        $canceler = function () {};
+        $canceler = function () use (&$job, $curl) {
+            if ($job['status'] === self::STATUS_QUEUED) {
+                $job['status'] = self::STATUS_ABORTED;
+            }
+            if ($job['status'] !== self::STATUS_STARTED) {
+                return;
+            }
+            $job['status'] = self::STATUS_ABORTED;
+            if (isset($job['curl_id'])) {
+                $curl->abortRequest($job['curl_id']);
+            }
+        };
 
         $promise = new Promise($resolver, $canceler);
         $promise->then(function ($response) use (&$job, $downloader) {
@@ -189,7 +207,6 @@ class HttpDownloader
 
             // TODO 3.0 this should be done directly on $this when PHP 5.3 is dropped
             $downloader->markJobDone();
-            $downloader->scheduleNextJob();
 
             return $response;
         }, function ($e) use (&$job, $downloader) {
@@ -197,7 +214,6 @@ class HttpDownloader
             $job['exception'] = $e;
 
             $downloader->markJobDone();
-            $downloader->scheduleNextJob();
 
             throw $e;
         });
@@ -239,9 +255,9 @@ class HttpDownloader
         }
 
         if ($job['request']['copyTo']) {
-            $this->curl->download($resolve, $reject, $origin, $url, $options, $job['request']['copyTo']);
+            $job['curl_id'] = $this->curl->download($resolve, $reject, $origin, $url, $options, $job['request']['copyTo']);
         } else {
-            $this->curl->download($resolve, $reject, $origin, $url, $options);
+            $job['curl_id'] = $this->curl->download($resolve, $reject, $origin, $url, $options);
         }
     }
 
@@ -253,49 +269,58 @@ class HttpDownloader
         $this->runningJobs--;
     }
 
-    /**
-     * @private
-     */
-    public function scheduleNextJob()
+    public function wait($index = null)
     {
-        foreach ($this->jobs as $job) {
-            if ($job['status'] === self::STATUS_QUEUED) {
-                $this->startJob($job['id']);
-                if ($this->runningJobs >= $this->maxJobs) {
-                    return;
-                }
+        while (true) {
+            if (!$this->countActiveJobs($index)) {
+                return;
             }
+
+            usleep(1000);
         }
     }
 
-    public function wait($index = null, $progress = false)
+    /**
+     * @internal
+     */
+    public function enableAsync()
     {
-        while (true) {
-            if ($this->curl) {
-                $this->curl->tick();
-            }
+        $this->allowAsync = true;
+    }
 
-            if (null !== $index) {
-                if ($this->jobs[$index]['status'] === self::STATUS_COMPLETED || $this->jobs[$index]['status'] === self::STATUS_FAILED) {
-                    return;
-                }
-            } else {
-                $done = true;
-                foreach ($this->jobs as $job) {
-                    if (!in_array($job['status'], array(self::STATUS_COMPLETED, self::STATUS_FAILED), true)) {
-                        $done = false;
-                        break;
-                    } elseif (!$job['sync']) {
-                        unset($this->jobs[$job['id']]);
-                    }
-                }
-                if ($done) {
-                    return;
+    /**
+     * @internal
+     *
+     * @return int number of active (queued or started) jobs
+     */
+    public function countActiveJobs($index = null)
+    {
+        if ($this->runningJobs < $this->maxJobs) {
+            foreach ($this->jobs as $job) {
+                if ($job['status'] === self::STATUS_QUEUED && $this->runningJobs < $this->maxJobs) {
+                    $this->startJob($job['id']);
                 }
             }
+        }
 
-            usleep(1000);
+        if ($this->curl) {
+            $this->curl->tick();
         }
+
+        if (null !== $index) {
+            return $this->jobs[$index]['status'] < self::STATUS_COMPLETED ? 1 : 0;
+        }
+
+        $active = 0;
+        foreach ($this->jobs as $job) {
+            if ($job['status'] < self::STATUS_COMPLETED) {
+                $active++;
+            } elseif (!$job['sync']) {
+                unset($this->jobs[$job['id']]);
+            }
+        }
+
+        return $active;
     }
 
     private function getResponse($index)

src/Composer/Util/Loop.php

@@ -14,6 +14,7 @@ namespace Composer\Util;
 
 use Composer\Util\HttpDownloader;
 use React\Promise\Promise;
+use Symfony\Component\Console\Helper\ProgressBar;
 
 /**
  * @author Jordi Boggiano <j.boggiano@seld.be>
@@ -21,13 +22,22 @@ use React\Promise\Promise;
 class Loop
 {
     private $httpDownloader;
+    private $processExecutor;
+    private $currentPromises;
 
-    public function __construct(HttpDownloader $httpDownloader)
+    public function __construct(HttpDownloader $httpDownloader = null, ProcessExecutor $processExecutor = null)
     {
         $this->httpDownloader = $httpDownloader;
+        if ($this->httpDownloader) {
+            $this->httpDownloader->enableAsync();
+        }
+        $this->processExecutor = $processExecutor;
+        if ($this->processExecutor) {
+            $this->processExecutor->enableAsync();
+        }
     }
 
-    public function wait(array $promises)
+    public function wait(array $promises, ProgressBar $progress = null)
     {
         /** @var \Exception|null */
         $uncaught = null;
@@ -39,10 +49,52 @@ class Loop
             }
         );
 
-        $this->httpDownloader->wait();
+        $this->currentPromises = $promises;
+
+        if ($progress) {
+            $totalJobs = 0;
+            if ($this->httpDownloader) {
+                $totalJobs += $this->httpDownloader->countActiveJobs();
+            }
+            if ($this->processExecutor) {
+                $totalJobs += $this->processExecutor->countActiveJobs();
+            }
+            $progress->start($totalJobs);
+        }
 
+        while (true) {
+            $activeJobs = 0;
+
+            if ($this->httpDownloader) {
+                $activeJobs += $this->httpDownloader->countActiveJobs();
+            }
+            if ($this->processExecutor) {
+                $activeJobs += $this->processExecutor->countActiveJobs();
+            }
+
+            if ($progress) {
+                $progress->setProgress($progress->getMaxSteps() - $activeJobs);
+            }
+
+            if (!$activeJobs) {
+                break;
+            }
+
+            usleep(5000);
+        }
+
+        $this->currentPromises = null;
         if ($uncaught) {
             throw $uncaught;
         }
     }
+
+    public function abortJobs()
+    {
+        if ($this->currentPromises) {
+            foreach ($this->currentPromises as $promise) {
+                $promise->cancel();
+            }
+        }
+    }
 }

src/Composer/Util/ProcessExecutor.php

@@ -16,18 +16,32 @@ use Composer\IO\IOInterface;
 use Symfony\Component\Process\Process;
 use Symfony\Component\Process\ProcessUtils;
 use Symfony\Component\Process\Exception\RuntimeException;
+use React\Promise\Promise;
 
 /**
  * @author Robert Schönthal <seroscho@googlemail.com>
+ * @author Jordi Boggiano <j.boggiano@seld.be>
  */
 class ProcessExecutor
 {
+    const STATUS_QUEUED = 1;
+    const STATUS_STARTED = 2;
+    const STATUS_COMPLETED = 3;
+    const STATUS_FAILED = 4;
+    const STATUS_ABORTED = 5;
+
     protected static $timeout = 300;
 
     protected $captureOutput;
     protected $errorOutput;
     protected $io;
 
+    private $jobs = array();
+    private $runningJobs = 0;
+    private $maxJobs = 10;
+    private $idGen = 0;
+    private $allowAsync = false;
+
     public function __construct(IOInterface $io = null)
     {
         $this->io = $io;
@@ -112,6 +126,192 @@ class ProcessExecutor
         return $process->getExitCode();
     }
 
+    /**
+     * starts a process on the commandline in async mode
+     *
+     * @param  string $command the command to execute
+     * @param  mixed  $output  the output will be written into this var if passed by ref
+     *                         if a callable is passed it will be used as output handler
+     * @param  string $cwd     the working directory
+     * @return int    statuscode
+     */
+    public function executeAsync($command, $cwd = null)
+    {
+        if (!$this->allowAsync) {
+            throw new \LogicException('You must use the ProcessExecutor instance which is part of a Composer\Loop instance to be able to run async processes');
+        }
+
+        $job = array(
+            'id' => $this->idGen++,
+            'status' => self::STATUS_QUEUED,
+            'command' => $command,
+            'cwd' => $cwd,
+        );
+
+        $resolver = function ($resolve, $reject) use (&$job) {
+            $job['status'] = ProcessExecutor::STATUS_QUEUED;
+            $job['resolve'] = $resolve;
+            $job['reject'] = $reject;
+        };
+
+        $self = $this;
+        $io = $this->io;
+
+        $canceler = function () use (&$job) {
+            if ($job['status'] === self::STATUS_QUEUED) {
+                $job['status'] = self::STATUS_ABORTED;
+            }
+            if ($job['status'] !== self::STATUS_STARTED) {
+                return;
+            }
+            $job['status'] = self::STATUS_ABORTED;
+            try {
+                if (defined('SIGINT')) {
+                    $job['process']->signal(SIGINT);
+                }
+            } catch (\Exception $e) {
+                // signal can throw in various conditions, but we don't care if it fails
+            }
+            $job['process']->stop(1);
+        };
+
+        $promise = new Promise($resolver, $canceler);
+        $promise = $promise->then(function () use (&$job, $self) {
+            if ($job['process']->isSuccessful()) {
+                $job['status'] = ProcessExecutor::STATUS_COMPLETED;
+            } else {
+                $job['status'] = ProcessExecutor::STATUS_FAILED;
+            }
+
+            // TODO 3.0 this should be done directly on $this when PHP 5.3 is dropped
+            $self->markJobDone();
+
+            return $job['process'];
+        }, function ($e) use (&$job, $self) {
+            $job['status'] = ProcessExecutor::STATUS_FAILED;
+
+            $self->markJobDone();
+
+            throw $e;
+        });
+        $this->jobs[$job['id']] =& $job;
+
+        if ($this->runningJobs < $this->maxJobs) {
+            $this->startJob($job['id']);
+        }
+
+        return $promise;
+    }
+
+    private function startJob($id)
+    {
+        $job =& $this->jobs[$id];
+        if ($job['status'] !== self::STATUS_QUEUED) {
+            return;
+        }
+
+        // start job
+        $job['status'] = self::STATUS_STARTED;
+        $this->runningJobs++;
+
+        $command = $job['command'];
+        $cwd = $job['cwd'];
+
+        if ($this->io && $this->io->isDebug()) {
+            $safeCommand = preg_replace_callback('{://(?P<user>[^:/\s]+):(?P<password>[^@\s/]+)@}i', function ($m) {
+                if (preg_match('{^[a-f0-9]{12,}$}', $m['user'])) {
+                    return '://***:***@';
+                }
+
+                return '://'.$m['user'].':***@';
+            }, $command);
+            $safeCommand = preg_replace("{--password (.*[^\\\\]\') }", '--password \'***\' ', $safeCommand);
+            $this->io->writeError('Executing async command ('.($cwd ?: 'CWD').'): '.$safeCommand);
+        }
+
+        // make sure that null translate to the proper directory in case the dir is a symlink
+        // and we call a git command, because msysgit does not handle symlinks properly
+        if (null === $cwd && Platform::isWindows() && false !== strpos($command, 'git') && getcwd()) {
+            $cwd = realpath(getcwd());
+        }
+
+        // TODO in v3, commands should be passed in as arrays of cmd + args
+        if (method_exists('Symfony\Component\Process\Process', 'fromShellCommandline')) {
+            $process = Process::fromShellCommandline($command, $cwd, null, null, static::getTimeout());
+        } else {
+            $process = new Process($command, $cwd, null, null, static::getTimeout());
+        }
+
+        $job['process'] = $process;
+
+        $process->start();
+    }
+
+    public function wait($index = null)
+    {
+        while (true) {
+            if (!$this->countActiveJobs($index)) {
+                return;
+            }
+
+            usleep(1000);
+        }
+    }
+
+    /**
+     * @internal
+     */
+    public function enableAsync()
+    {
+        $this->allowAsync = true;
+    }
+
+    /**
+     * @internal
+     *
+     * @return int number of active (queued or started) jobs
+     */
+    public function countActiveJobs($index = null)
+    {
+        // tick
+        foreach ($this->jobs as $job) {
+            if ($job['status'] === self::STATUS_STARTED) {
+                if (!$job['process']->isRunning()) {
+                    call_user_func($job['resolve'], $job['process']);
+                }
+            }
+
+            if ($this->runningJobs < $this->maxJobs) {
+                if ($job['status'] === self::STATUS_QUEUED) {
+                    $this->startJob($job['id']);
+                }
+            }
+        }
+
+        if (null !== $index) {
+            return $this->jobs[$index]['status'] < self::STATUS_COMPLETED ? 1 : 0;
+        }
+
+        $active = 0;
+        foreach ($this->jobs as $job) {
+            if ($job['status'] < self::STATUS_COMPLETED) {
+                $active++;
+            } else {
+                unset($this->jobs[$job['id']]);
+            }
+        }
+
+        return $active;
+    }
+
+    /**
+     * @private
+     */
+    public function markJobDone()
+    {
+        $this->runningJobs--;
+    }
+
     public function splitLines($output)
     {
         $output = trim($output);

src/Composer/Util/RemoteFilesystem.php

@@ -20,6 +20,7 @@ use Composer\Util\HttpDownloader;
 use Composer\Util\Http\Response;
 
 /**
+ * @internal
  * @author François Pluchino <francois.pluchino@opendisplay.com>
  * @author Jordi Boggiano <j.boggiano@seld.be>
  * @author Nils Adermann <naderman@naderman.de>

tests/Composer/Test/Downloader/FileDownloaderTest.php

@@ -139,8 +139,8 @@ class FileDownloaderTest extends TestCase
             ->will($this->returnValue($path.'/vendor'));
 
         try {
-            $promise = $downloader->download($packageMock, $path);
             $loop = new Loop($this->httpDownloader);
+            $promise = $downloader->download($packageMock, $path);
             $loop->wait(array($promise));
 
             $this->fail('Download was expected to throw');
@@ -225,8 +225,8 @@ class FileDownloaderTest extends TestCase
         touch($dlFile);
 
         try {
-            $promise = $downloader->download($packageMock, $path);
             $loop = new Loop($this->httpDownloader);
+            $promise = $downloader->download($packageMock, $path);
             $loop->wait(array($promise));
 
             $this->fail('Download was expected to throw');
@@ -296,8 +296,8 @@ class FileDownloaderTest extends TestCase
         mkdir(dirname($dlFile), 0777, true);
         touch($dlFile);
 
-        $promise = $downloader->download($newPackage, $path, $oldPackage);
         $loop = new Loop($this->httpDownloader);
+        $promise = $downloader->download($newPackage, $path, $oldPackage);
         $loop->wait(array($promise));
 
         $downloader->update($oldPackage, $newPackage, $path);

tests/Composer/Test/Downloader/XzDownloaderTest.php

@@ -70,8 +70,8 @@ class XzDownloaderTest extends TestCase
         $downloader = new XzDownloader($io, $config, $httpDownloader = new HttpDownloader($io, $this->getMockBuilder('Composer\Config')->getMock()), null, null, null);
 
         try {
-            $promise = $downloader->download($packageMock, $this->testDir.'/install-path');
             $loop = new Loop($httpDownloader);
+            $promise = $downloader->download($packageMock, $this->testDir.'/install-path');
             $loop->wait(array($promise));
             $downloader->install($packageMock, $this->testDir.'/install-path');
 

tests/Composer/Test/Downloader/ZipDownloaderTest.php

@@ -60,9 +60,6 @@ class ZipDownloaderTest extends TestCase
         }
     }
 
-    /**
-     * @group only
-     */
     public function testErrorMessages()
     {
         if (!class_exists('ZipArchive')) {
@@ -92,8 +89,8 @@ class ZipDownloaderTest extends TestCase
         $this->setPrivateProperty('hasSystemUnzip', false);
 
         try {
-            $promise = $downloader->download($this->package, $path = sys_get_temp_dir().'/composer-zip-test');
             $loop = new Loop($this->httpDownloader);
+            $promise = $downloader->download($this->package, $path = sys_get_temp_dir().'/composer-zip-test');
             $loop->wait(array($promise));
             $downloader->install($this->package, $path);
 
@@ -125,7 +122,8 @@ class ZipDownloaderTest extends TestCase
             ->will($this->returnValue(false));
 
         $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader);
-        $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $promise = $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $this->wait($promise);
     }
 
     /**
@@ -150,12 +148,10 @@ class ZipDownloaderTest extends TestCase
             ->will($this->throwException(new \ErrorException('Not a directory')));
 
         $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader);
-        $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $promise = $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $this->wait($promise);
     }
 
-    /**
-     * @group only
-     */
     public function testZipArchiveOnlyGood()
     {
         if (!class_exists('ZipArchive')) {
@@ -174,45 +170,66 @@ class ZipDownloaderTest extends TestCase
             ->will($this->returnValue(true));
 
         $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader);
-        $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $promise = $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $this->wait($promise);
     }
 
     /**
      * @expectedException \Exception
-     * @expectedExceptionMessage Failed to execute (1) unzip
+     * @expectedExceptionMessage Failed to extract : (1) unzip
      */
     public function testSystemUnzipOnlyFailed()
     {
-        if (!class_exists('ZipArchive')) {
-            $this->markTestSkipped('zip extension missing');
-        }
-
+        $this->setPrivateProperty('isWindows', false);
         $this->setPrivateProperty('hasSystemUnzip', true);
         $this->setPrivateProperty('hasZipArchive', false);
+
+        $procMock = $this->getMockBuilder('Symfony\Component\Process\Process')->disableOriginalConstructor()->getMock();
+        $procMock->expects($this->any())
+            ->method('getExitCode')
+            ->will($this->returnValue(1));
+        $procMock->expects($this->any())
+            ->method('isSuccessful')
+            ->will($this->returnValue(false));
+        $procMock->expects($this->any())
+            ->method('getErrorOutput')
+            ->will($this->returnValue('output'));
+
         $processExecutor = $this->getMockBuilder('Composer\Util\ProcessExecutor')->getMock();
         $processExecutor->expects($this->at(0))
-            ->method('execute')
-            ->will($this->returnValue(1));
+            ->method('executeAsync')
+            ->will($this->returnValue(\React\Promise\resolve($procMock)));
 
         $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, null, $processExecutor);
-        $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $promise = $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $this->wait($promise);
     }
 
     public function testSystemUnzipOnlyGood()
     {
-        if (!class_exists('ZipArchive')) {
-            $this->markTestSkipped('zip extension missing');
-        }
-
+        $this->setPrivateProperty('isWindows', false);
         $this->setPrivateProperty('hasSystemUnzip', true);
         $this->setPrivateProperty('hasZipArchive', false);
+
+        $procMock = $this->getMockBuilder('Symfony\Component\Process\Process')->disableOriginalConstructor()->getMock();
+        $procMock->expects($this->any())
+            ->method('getExitCode')
+            ->will($this->returnValue(0));
+        $procMock->expects($this->any())
+            ->method('isSuccessful')
+            ->will($this->returnValue(true));
+        $procMock->expects($this->any())
+            ->method('getErrorOutput')
+            ->will($this->returnValue('output'));
+
         $processExecutor = $this->getMockBuilder('Composer\Util\ProcessExecutor')->getMock();
         $processExecutor->expects($this->at(0))
-            ->method('execute')
-            ->will($this->returnValue(0));
+            ->method('executeAsync')
+            ->will($this->returnValue(\React\Promise\resolve($procMock)));
 
         $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, null, $processExecutor);
-        $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $promise = $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $this->wait($promise);
     }
 
     public function testNonWindowsFallbackGood()
@@ -225,10 +242,21 @@ class ZipDownloaderTest extends TestCase
         $this->setPrivateProperty('hasSystemUnzip', true);
         $this->setPrivateProperty('hasZipArchive', true);
 
+        $procMock = $this->getMockBuilder('Symfony\Component\Process\Process')->disableOriginalConstructor()->getMock();
+        $procMock->expects($this->any())
+            ->method('getExitCode')
+            ->will($this->returnValue(1));
+        $procMock->expects($this->any())
+            ->method('isSuccessful')
+            ->will($this->returnValue(false));
+        $procMock->expects($this->any())
+            ->method('getErrorOutput')
+            ->will($this->returnValue('output'));
+
         $processExecutor = $this->getMockBuilder('Composer\Util\ProcessExecutor')->getMock();
         $processExecutor->expects($this->at(0))
-            ->method('execute')
-            ->will($this->returnValue(1));
+            ->method('executeAsync')
+            ->will($this->returnValue(\React\Promise\resolve($procMock)));
 
         $zipArchive = $this->getMockBuilder('ZipArchive')->getMock();
         $zipArchive->expects($this->at(0))
@@ -240,7 +268,8 @@ class ZipDownloaderTest extends TestCase
 
         $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, null, $processExecutor);
         $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader);
-        $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $promise = $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $this->wait($promise);
     }
 
     /**
@@ -257,10 +286,21 @@ class ZipDownloaderTest extends TestCase
         $this->setPrivateProperty('hasSystemUnzip', true);
         $this->setPrivateProperty('hasZipArchive', true);
 
+        $procMock = $this->getMockBuilder('Symfony\Component\Process\Process')->disableOriginalConstructor()->getMock();
+        $procMock->expects($this->any())
+            ->method('getExitCode')
+            ->will($this->returnValue(1));
+        $procMock->expects($this->any())
+            ->method('isSuccessful')
+            ->will($this->returnValue(false));
+        $procMock->expects($this->any())
+            ->method('getErrorOutput')
+            ->will($this->returnValue('output'));
+
         $processExecutor = $this->getMockBuilder('Composer\Util\ProcessExecutor')->getMock();
         $processExecutor->expects($this->at(0))
-          ->method('execute')
-          ->will($this->returnValue(1));
+          ->method('executeAsync')
+          ->will($this->returnValue(\React\Promise\resolve($procMock)));
 
         $zipArchive = $this->getMockBuilder('ZipArchive')->getMock();
         $zipArchive->expects($this->at(0))
@@ -272,7 +312,8 @@ class ZipDownloaderTest extends TestCase
 
         $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, null, $processExecutor);
         $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader);
-        $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $promise = $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $this->wait($promise);
     }
 
     public function testWindowsFallbackGood()
@@ -300,7 +341,8 @@ class ZipDownloaderTest extends TestCase
 
         $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, null, $processExecutor);
         $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader);
-        $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $promise = $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $this->wait($promise);
     }
 
     /**
@@ -332,7 +374,26 @@ class ZipDownloaderTest extends TestCase
 
         $downloader = new MockedZipDownloader($this->io, $this->config, $this->httpDownloader, null, null, null, $processExecutor);
         $this->setPrivateProperty('zipArchiveObject', $zipArchive, $downloader);
-        $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $promise = $downloader->extract($this->package, 'testfile.zip', 'vendor/dir');
+        $this->wait($promise);
+    }
+
+    private function wait($promise)
+    {
+        if (null === $promise) {
+            return;
+        }
+
+        $e = null;
+        $promise->then(function () {
+            // noop
+        }, function ($ex) use (&$e) {
+            $e = $ex;
+        });
+
+        if ($e) {
+            throw $e;
+        }
     }
 }
 
@@ -350,6 +411,6 @@ class MockedZipDownloader extends ZipDownloader
 
     public function extract(PackageInterface $package, $file, $path)
     {
-        parent::extract($package, $file, $path);
+        return parent::extract($package, $file, $path);
     }
 }

tests/Composer/Test/Repository/ComposerRepositoryTest.php

@@ -189,16 +189,19 @@ class ComposerRepositoryTest extends TestCase
             ->getMock();
 
         $httpDownloader->expects($this->at(0))
+            ->method('enableAsync');
+
+        $httpDownloader->expects($this->at(1))
             ->method('get')
             ->with($url = 'http://example.org/packages.json')
             ->willReturn(new \Composer\Util\Http\Response(array('url' => $url), 200, array(), json_encode(array('search' => '/search.json?q=%query%&type=%type%'))));
 
-        $httpDownloader->expects($this->at(1))
+        $httpDownloader->expects($this->at(2))
             ->method('get')
             ->with($url = 'http://example.org/search.json?q=foo&type=composer-plugin')
             ->willReturn(new \Composer\Util\Http\Response(array('url' => $url), 200, array(), json_encode($result)));
 
-        $httpDownloader->expects($this->at(2))
+        $httpDownloader->expects($this->at(3))
             ->method('get')
             ->with($url = 'http://example.org/search.json?q=foo&type=library')
             ->willReturn(new \Composer\Util\Http\Response(array('url' => $url), 200, array(), json_encode(array())));
@@ -291,6 +294,9 @@ class ComposerRepositoryTest extends TestCase
             ->getMock();
 
         $httpDownloader->expects($this->at(0))
+            ->method('enableAsync');
+
+        $httpDownloader->expects($this->at(1))
             ->method('get')
             ->with($url = 'http://example.org/packages.json')
             ->willReturn(new \Composer\Util\Http\Response(array('url' => $url), 200, array(), json_encode(array(