From 56805ecafe8597a07b4576b45e5532b9862fd65a Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Wed, 12 Sep 2018 18:58:54 +0200 Subject: [PATCH] Add HttpDownloader to wrap/replace RemoteFilesystem with a new curl multi implementation --- composer.json | 3 +- composer.lock | 46 ++- src/Composer/Downloader/FileDownloader.php | 20 +- src/Composer/Downloader/GzipDownloader.php | 6 +- src/Composer/Downloader/RarDownloader.php | 6 +- src/Composer/Downloader/XzDownloader.php | 6 +- src/Composer/Downloader/ZipDownloader.php | 6 +- src/Composer/Factory.php | 14 +- src/Composer/Plugin/PreFileDownloadEvent.php | 20 +- .../Repository/ComposerRepository.php | 222 +++++++++++--- src/Composer/Repository/RepositoryFactory.php | 6 +- src/Composer/Repository/RepositoryManager.php | 6 +- src/Composer/Util/Http/CurlDownloader.php | 282 ++++++++++++++++++ src/Composer/Util/Http/Response.php | 75 +++++ src/Composer/Util/HttpDownloader.php | 246 +++++++++++++++ src/Composer/Util/RemoteFilesystem.php | 108 +------ src/Composer/Util/StreamContextFactory.php | 105 +++++++ 17 files changed, 986 insertions(+), 191 deletions(-) create mode 100644 src/Composer/Util/Http/CurlDownloader.php create mode 100644 src/Composer/Util/Http/Response.php create mode 100644 src/Composer/Util/HttpDownloader.php diff --git a/composer.json b/composer.json index 41048903b..1b75131bc 100644 --- a/composer.json +++ b/composer.json @@ -34,7 +34,8 @@ "symfony/console": "^2.7 || ^3.0 || ^4.0", "symfony/filesystem": "^2.7 || ^3.0 || ^4.0", "symfony/finder": "^2.7 || ^3.0 || ^4.0", - "symfony/process": "^2.7 || ^3.0 || ^4.0" + "symfony/process": "^2.7 || ^3.0 || ^4.0", + "react/promise": "^1.2" }, "conflict": { "symfony/console": "2.8.38" diff --git a/composer.lock b/composer.lock index 957382bc6..63b8033b9 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e46280c4cfd37bf3ec8be36095feb20e", + "content-hash": "d356b92e869790db1e9d2c0f4b10935e", "packages": [ { "name": "composer/ca-bundle", @@ -342,6 +342,50 @@ ], "time": "2018-11-20T15:27:04+00:00" }, + { + "name": "react/promise", + "version": "v1.2.1", + "source": { + "type": "git", + "url": "https://github.com/reactphp/promise.git", + "reference": "eefff597e67ff66b719f8171480add3c91474a1e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/reactphp/promise/zipball/eefff597e67ff66b719f8171480add3c91474a1e", + "reference": "eefff597e67ff66b719f8171480add3c91474a1e", + "shasum": "" + }, + "require": { + "php": ">=5.3.3" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.1-dev" + } + }, + "autoload": { + "psr-0": { + "React\\Promise": "src/" + }, + "files": [ + "src/React/Promise/functions_include.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jan Sorgalla", + "email": "jsorgalla@gmail.com" + } + ], + "description": "A lightweight implementation of CommonJS Promises/A for PHP", + "time": "2016-03-07T13:46:50+00:00" + }, { "name": "seld/jsonlint", "version": "1.7.1", diff --git a/src/Composer/Downloader/FileDownloader.php b/src/Composer/Downloader/FileDownloader.php index 6596d9c8b..6b1349bb8 100644 --- a/src/Composer/Downloader/FileDownloader.php +++ b/src/Composer/Downloader/FileDownloader.php @@ -24,7 +24,7 @@ use Composer\Plugin\PluginEvents; use Composer\Plugin\PreFileDownloadEvent; use Composer\EventDispatcher\EventDispatcher; use Composer\Util\Filesystem; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\Util\Url as UrlUtil; /** @@ -39,7 +39,7 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface { protected $io; protected $config; - protected $rfs; + protected $httpDownloader; protected $filesystem; protected $cache; protected $outputProgress = true; @@ -52,16 +52,16 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface * @param IOInterface $io The IO instance * @param Config $config The config * @param EventDispatcher $eventDispatcher The event dispatcher - * @param Cache $cache Optional cache instance - * @param RemoteFilesystem $rfs The remote filesystem + * @param Cache $cache Cache instance + * @param HttpDownloader $httpDownloader The remote filesystem * @param Filesystem $filesystem The filesystem */ - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, RemoteFilesystem $rfs = null, Filesystem $filesystem = null) + public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher, Cache $cache, HttpDownloader $httpDownloader, Filesystem $filesystem = null) { $this->io = $io; $this->config = $config; $this->eventDispatcher = $eventDispatcher; - $this->rfs = $rfs ?: Factory::createRemoteFilesystem($this->io, $config); + $this->httpDownloader = $httpDownloader; $this->filesystem = $filesystem ?: new Filesystem(); $this->cache = $cache; @@ -125,13 +125,12 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface $fileName = $this->getFileName($package, $path); $processedUrl = $this->processUrl($package, $url); - $hostname = parse_url($processedUrl, PHP_URL_HOST); - $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->rfs, $processedUrl); + $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $processedUrl); if ($this->eventDispatcher) { $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); } - $rfs = $preFileDownloadEvent->getRemoteFilesystem(); + $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); try { $checksum = $package->getDistSha1Checksum(); @@ -150,7 +149,8 @@ class FileDownloader implements DownloaderInterface, ChangeReportInterface $retries = 3; while ($retries--) { try { - $rfs->copy($hostname, $processedUrl, $fileName, $this->outputProgress, $package->getTransportOptions()); + // TODO handle this->outputProgress + $httpDownloader->copy($processedUrl, $fileName, $package->getTransportOptions()); break; } catch (TransportException $e) { // if we got an http response with a proper code, then requesting again will probably not help, abort diff --git a/src/Composer/Downloader/GzipDownloader.php b/src/Composer/Downloader/GzipDownloader.php index 19e4a45e1..bb86f6267 100644 --- a/src/Composer/Downloader/GzipDownloader.php +++ b/src/Composer/Downloader/GzipDownloader.php @@ -18,7 +18,7 @@ use Composer\EventDispatcher\EventDispatcher; use Composer\Package\PackageInterface; use Composer\Util\Platform; use Composer\Util\ProcessExecutor; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\IO\IOInterface; /** @@ -30,10 +30,10 @@ class GzipDownloader extends ArchiveDownloader { protected $process; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, RemoteFilesystem $rfs = null) + public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null) { $this->process = $process ?: new ProcessExecutor($io); - parent::__construct($io, $config, $eventDispatcher, $cache, $rfs); + parent::__construct($io, $config, $eventDispatcher, $cache, $downloader); } protected function extract($file, $path) diff --git a/src/Composer/Downloader/RarDownloader.php b/src/Composer/Downloader/RarDownloader.php index 40cd09896..c1ed8d60b 100644 --- a/src/Composer/Downloader/RarDownloader.php +++ b/src/Composer/Downloader/RarDownloader.php @@ -18,7 +18,7 @@ use Composer\EventDispatcher\EventDispatcher; use Composer\Util\IniHelper; use Composer\Util\Platform; use Composer\Util\ProcessExecutor; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\IO\IOInterface; use RarArchive; @@ -33,10 +33,10 @@ class RarDownloader extends ArchiveDownloader { protected $process; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, RemoteFilesystem $rfs = null) + public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null) { $this->process = $process ?: new ProcessExecutor($io); - parent::__construct($io, $config, $eventDispatcher, $cache, $rfs); + parent::__construct($io, $config, $eventDispatcher, $cache, $downloader); } protected function extract($file, $path) diff --git a/src/Composer/Downloader/XzDownloader.php b/src/Composer/Downloader/XzDownloader.php index 4a9b854d3..1f5947997 100644 --- a/src/Composer/Downloader/XzDownloader.php +++ b/src/Composer/Downloader/XzDownloader.php @@ -17,7 +17,7 @@ use Composer\Cache; use Composer\EventDispatcher\EventDispatcher; use Composer\Package\PackageInterface; use Composer\Util\ProcessExecutor; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\IO\IOInterface; /** @@ -30,11 +30,11 @@ class XzDownloader extends ArchiveDownloader { protected $process; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, RemoteFilesystem $rfs = null) + public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null) { $this->process = $process ?: new ProcessExecutor($io); - parent::__construct($io, $config, $eventDispatcher, $cache, $rfs); + parent::__construct($io, $config, $eventDispatcher, $cache, $downloader); } protected function extract($file, $path) diff --git a/src/Composer/Downloader/ZipDownloader.php b/src/Composer/Downloader/ZipDownloader.php index 6534db3d8..10518e026 100644 --- a/src/Composer/Downloader/ZipDownloader.php +++ b/src/Composer/Downloader/ZipDownloader.php @@ -19,7 +19,7 @@ use Composer\Package\PackageInterface; use Composer\Util\IniHelper; use Composer\Util\Platform; use Composer\Util\ProcessExecutor; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\IO\IOInterface; use Symfony\Component\Process\ExecutableFinder; use ZipArchive; @@ -36,10 +36,10 @@ class ZipDownloader extends ArchiveDownloader protected $process; private $zipArchiveObject; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, RemoteFilesystem $rfs = null) + public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, Cache $cache = null, ProcessExecutor $process = null, HttpDownloader $downloader = null) { $this->process = $process ?: new ProcessExecutor($io); - parent::__construct($io, $config, $eventDispatcher, $cache, $rfs); + parent::__construct($io, $config, $eventDispatcher, $cache, $downloader); } /** diff --git a/src/Composer/Factory.php b/src/Composer/Factory.php index 1aac934a1..97d507b10 100644 --- a/src/Composer/Factory.php +++ b/src/Composer/Factory.php @@ -23,7 +23,7 @@ use Composer\Repository\WritableRepositoryInterface; use Composer\Util\Filesystem; use Composer\Util\Platform; use Composer\Util\ProcessExecutor; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\Util\Silencer; use Composer\Plugin\PluginEvents; use Composer\EventDispatcher\Event; @@ -325,7 +325,7 @@ class Factory $io->loadConfiguration($config); } - $rfs = self::createRemoteFilesystem($io, $config); + $rfs = self::createHttpDownloader($io, $config); // initialize event dispatcher $dispatcher = new EventDispatcher($composer, $io); @@ -451,7 +451,7 @@ class Factory * @param EventDispatcher $eventDispatcher * @return Downloader\DownloadManager */ - public function createDownloadManager(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, RemoteFilesystem $rfs = null) + public function createDownloadManager(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, HttpDownloader $rfs = null) { $cache = null; if ($config->get('cache-files-ttl') > 0) { @@ -579,10 +579,10 @@ class Factory /** * @param IOInterface $io IO instance * @param Config $config Config instance - * @param array $options Array of options passed directly to RemoteFilesystem constructor - * @return RemoteFilesystem + * @param array $options Array of options passed directly to HttpDownloader constructor + * @return HttpDownloader */ - public static function createRemoteFilesystem(IOInterface $io, Config $config = null, $options = array()) + public static function createHttpDownloader(IOInterface $io, Config $config = null, $options = array()) { static $warned = false; $disableTls = false; @@ -607,7 +607,7 @@ class Factory $remoteFilesystemOptions = array_replace_recursive($remoteFilesystemOptions, $options); } try { - $remoteFilesystem = new RemoteFilesystem($io, $config, $remoteFilesystemOptions, $disableTls); + $remoteFilesystem = new HttpDownloader($io, $config, $remoteFilesystemOptions, $disableTls); } catch (TransportException $e) { if (false !== strpos($e->getMessage(), 'cafile')) { $io->write('Unable to locate a valid CA certificate file. You must set a valid \'cafile\' option.'); diff --git a/src/Composer/Plugin/PreFileDownloadEvent.php b/src/Composer/Plugin/PreFileDownloadEvent.php index 7ae6821ce..076449484 100644 --- a/src/Composer/Plugin/PreFileDownloadEvent.php +++ b/src/Composer/Plugin/PreFileDownloadEvent.php @@ -13,7 +13,7 @@ namespace Composer\Plugin; use Composer\EventDispatcher\Event; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; /** * The pre file download event. @@ -23,7 +23,7 @@ use Composer\Util\RemoteFilesystem; class PreFileDownloadEvent extends Event { /** - * @var RemoteFilesystem + * @var HttpDownloader */ private $rfs; @@ -36,10 +36,10 @@ class PreFileDownloadEvent extends Event * Constructor. * * @param string $name The event name - * @param RemoteFilesystem $rfs + * @param HttpDownloader $rfs * @param string $processedUrl */ - public function __construct($name, RemoteFilesystem $rfs, $processedUrl) + public function __construct($name, HttpDownloader $rfs, $processedUrl) { parent::__construct($name); $this->rfs = $rfs; @@ -47,21 +47,17 @@ class PreFileDownloadEvent extends Event } /** - * Returns the remote filesystem - * - * @return RemoteFilesystem + * @return HttpDownloader */ - public function getRemoteFilesystem() + public function getHttpDownloader() { return $this->rfs; } /** - * Sets the remote filesystem - * - * @param RemoteFilesystem $rfs + * @param HttpDownloader $rfs */ - public function setRemoteFilesystem(RemoteFilesystem $rfs) + public function setHttpDownloader(HttpDownloader $rfs) { $this->rfs = $rfs; } diff --git a/src/Composer/Repository/ComposerRepository.php b/src/Composer/Repository/ComposerRepository.php index 09e2179d8..c4a570c75 100644 --- a/src/Composer/Repository/ComposerRepository.php +++ b/src/Composer/Repository/ComposerRepository.php @@ -21,7 +21,7 @@ use Composer\Cache; use Composer\Config; use Composer\Factory; use Composer\IO\IOInterface; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\Plugin\PluginEvents; use Composer\Plugin\PreFileDownloadEvent; use Composer\EventDispatcher\EventDispatcher; @@ -40,7 +40,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito protected $url; protected $baseUrl; protected $io; - protected $rfs; + protected $httpDownloader; protected $cache; protected $notifyUrl; protected $searchUrl; @@ -60,8 +60,9 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito private $rootData; private $hasPartialPackages; private $partialPackagesByName; + private $versionParser; - public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, RemoteFilesystem $rfs = null) + public function __construct(array $repoConfig, IOInterface $io, Config $config, EventDispatcher $eventDispatcher, HttpDownloader $httpDownloader) { parent::__construct(); if (!preg_match('{^[\w.]+\??://}', $repoConfig['url'])) { @@ -98,12 +99,14 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->baseUrl = rtrim(preg_replace('{(?:/[^/\\\\]+\.json)?(?:[?#].*)?$}', '', $this->url), '/'); $this->io = $io; $this->cache = new Cache($io, $config->get('cache-repo-dir').'/'.preg_replace('{[^a-z0-9.]}i', '-', $this->url), 'a-z0-9.$'); - $this->loader = new ArrayLoader(); - if ($rfs && $this->options) { - $rfs = clone $rfs; - $rfs->setOptions($this->options); - } - $this->rfs = $rfs ?: Factory::createRemoteFilesystem($this->io, $this->config, $this->options); + $this->versionParser = new VersionParser(); + $this->loader = new ArrayLoader($this->versionParser); + if ($httpDownloader && $this->options) { + // TODO solve this somehow - should be sent a request time not on the instance + $httpDownloader = clone $httpDownloader; + $httpDownloader->setOptions($this->options); + } + $this->httpDownloader = $httpDownloader; $this->eventDispatcher = $eventDispatcher; $this->repoConfig = $repoConfig; } @@ -129,8 +132,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $name = strtolower($name); if (!$constraint instanceof ConstraintInterface) { - $versionParser = new VersionParser(); - $constraint = $versionParser->parseConstraints($constraint); + $constraint = $this->versionParser->parseConstraints($constraint); } foreach ($this->getProviderNames() as $providerName) { @@ -161,8 +163,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $name = strtolower($name); if (null !== $constraint && !$constraint instanceof ConstraintInterface) { - $versionParser = new VersionParser(); - $constraint = $versionParser->parseConstraints($constraint); + $constraint = $this->versionParser->parseConstraints($constraint); } $packages = array(); @@ -196,8 +197,10 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito public function loadPackages(array $packageNameMap, $isPackageAcceptableCallable) { + if ($this->lazyProvidersUrl) { + return $this->loadAsyncPackages($packageNameMap, $isPackageAcceptableCallable); + } if (!$this->hasProviders()) { - // TODO build more efficient version of this return parent::loadPackages($packageNameMap, $isPackageAcceptableCallable); } @@ -235,9 +238,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito if ($this->searchUrl && $mode === self::SEARCH_FULLTEXT) { $url = str_replace(array('%query%', '%type%'), array($query, $type), $this->searchUrl); - $hostname = parse_url($url, PHP_URL_HOST) ?: $url; - $json = $this->rfs->getContents($hostname, $url, false); - $search = JsonFile::parseJson($json, $url); + $search = $this->httpDownloader->get($url)->decodeJson(); if (empty($search['results'])) { return array(); @@ -496,6 +497,85 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->configurePackageTransportOptions($package); } + private function loadAsyncPackages(array $packageNames, $isPackageAcceptableCallable) + { + $this->loadRootServerFile(); + + $packages = array(); + $repo = $this; + + $createPackageIfAcceptable = function ($version, $constraint) use (&$packages, $isPackageAcceptableCallable, $repo) { + if (!call_user_func($isPackageAcceptableCallable, strtolower($version['name']), VersionParser::parseStability($version['version']))) { + return; + } + + if (isset($version['version_normalized']) && $constraint && !$constraint->matches(new Constraint('==', $version['version_normalized']))) { + return; + } + + // load acceptable packages in the providers + $package = $this->createPackage($version, 'Composer\Package\CompletePackage'); + $package->setRepository($repo); + + // if there was no version_normalized, then we need to check now for the constraint + if (!$constraint || isset($version['version_normalized']) || $constraint->matches(new Constraint('==', $package->getVersion()))) { + $packages[spl_object_hash($package)] = $package; + if ($package instanceof AliasPackage && !isset($packages[spl_object_hash($package->getAliasOf())])) { + $packages[spl_object_hash($package->getAliasOf())] = $package->getAliasOf(); + } + } + }; + + if ($this->lazyProvidersUrl) { + foreach ($packageNames as $name => $constraint) { + $url = str_replace('%package%', $name, $this->lazyProvidersUrl); + $cacheKey = 'provider-'.strtr($name, '/', '$').'.json'; + + $lastModified = null; + if ($contents = $this->cache->read($cacheKey)) { + $contents = json_decode($contents, true); + $lastModified = isset($contents['last-modified']) ? $contents['last-modified'] : null; + } + + $this->asyncFetchFile($url, $cacheKey, $lastModified) + ->then(function ($response) use (&$packages, $contents, $name, $constraint, $createPackageIfAcceptable) { + if (true === $response) { + $response = $contents; + } + + $uniqKeys = array('version', 'version_normalized', 'source', 'dist', 'time'); + foreach ($response['packages'][$name] as $version) { + if (isset($version['versions'])) { + $baseVersion = $version; + foreach ($uniqKeys as $key) { + unset($baseVersion[$key.'s']); + } + + foreach ($version['versions'] as $index => $dummy) { + $unpackedVersion = $baseVersion; + foreach ($uniqKeys as $key) { + $unpackedVersion[$key] = $version[$key.'s'][$index]; + } + + $createPackageIfAcceptable($unpackedVersion, $constraint); + } + } else { + $createPackageIfAcceptable($version, $constraint); + } + } + }, function ($e) { + // TODO use ->done() above instead with react/promise 2.0 + var_dump('Uncaught Ex', $e->getMessage()); + }); + } + } + + $this->httpDownloader->wait(); + + return $packages; + // RepositorySet should call loadMetadata, getMetadata when all promises resolved, then metadataComplete when done so we can GC the loaded json and whatnot then as needed + } + protected function loadRootServerFile() { if (null !== $this->rootData) { @@ -691,15 +771,13 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $retries = 3; while ($retries--) { try { - $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->rfs, $filename); - if ($this->eventDispatcher) { - $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); - } + $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename); + $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); - $hostname = parse_url($filename, PHP_URL_HOST) ?: $filename; - $rfs = $preFileDownloadEvent->getRemoteFilesystem(); + $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); - $json = $rfs->getContents($hostname, $filename, false); + $response = $httpDownloader->get($filename); + $json = $response->getBody(); if ($sha256 && $sha256 !== hash('sha256', $json)) { // undo downgrade before trying again if http seems to be hijacked or modifying content somehow if ($this->allowSslDowngrade) { @@ -718,7 +796,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito throw new RepositorySecurityException('The contents of '.$filename.' do not match its signature. This could indicate a man-in-the-middle attack or e.g. antivirus software corrupting files. Try running composer again and report this if you think it is a mistake.'); } - $data = JsonFile::parseJson($json, $filename); + $data = $response->decodeJson(); if (!empty($data['warning'])) { $this->io->writeError('Warning from '.$this->url.': '.$data['warning'].''); } @@ -728,7 +806,7 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito if ($cacheKey) { if ($storeLastModifiedTime) { - $lastModifiedDate = $rfs->findHeaderValue($rfs->getLastHeaders(), 'last-modified'); + $lastModifiedDate = $response->getHeader('last-modified'); if ($lastModifiedDate) { $data['last-modified'] = $lastModifiedDate; $json = json_encode($data); @@ -737,8 +815,14 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->cache->write($cacheKey, $json); } + $response->collect(); + break; } catch (\Exception $e) { + if ($e instanceof \LogicException) { + throw $e; + } + if ($e instanceof TransportException && $e->getStatusCode() === 404) { throw $e; } @@ -775,20 +859,18 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $retries = 3; while ($retries--) { try { - $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->rfs, $filename); - if ($this->eventDispatcher) { - $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); - } + $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename); + $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); - $hostname = parse_url($filename, PHP_URL_HOST) ?: $filename; - $rfs = $preFileDownloadEvent->getRemoteFilesystem(); + $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); $options = array('http' => array('header' => array('If-Modified-Since: '.$lastModifiedTime))); - $json = $rfs->getContents($hostname, $filename, false, $options); - if ($json === '' && $rfs->findStatusCode($rfs->getLastHeaders()) === 304) { + $response = $httpDownloader->get($filename, $options); + $json = $response->getBody(); + if ($json === '' && $response->getStatusCode() === 304) { return true; } - $data = JsonFile::parseJson($json, $filename); + $data = $response->decodeJson(); if (!empty($data['warning'])) { $this->io->writeError('Warning from '.$this->url.': '.$data['warning'].''); } @@ -796,7 +878,8 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito $this->io->writeError('Info from '.$this->url.': '.$data['info'].''); } - $lastModifiedDate = $rfs->findHeaderValue($rfs->getLastHeaders(), 'last-modified'); + $lastModifiedDate = $response->getHeader('last-modified'); + $response->collect(); if ($lastModifiedDate) { $data['last-modified'] = $lastModifiedDate; $json = json_encode($data); @@ -805,6 +888,10 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito return $data; } catch (\Exception $e) { + if ($e instanceof \LogicException) { + throw $e; + } + if ($e instanceof TransportException && $e->getStatusCode() === 404) { throw $e; } @@ -825,6 +912,69 @@ class ComposerRepository extends ArrayRepository implements ConfigurableReposito } } + protected function asyncFetchFile($filename, $cacheKey, $lastModifiedTime = null) + { + $retries = 3; + $preFileDownloadEvent = new PreFileDownloadEvent(PluginEvents::PRE_FILE_DOWNLOAD, $this->httpDownloader, $filename); + $this->eventDispatcher->dispatch($preFileDownloadEvent->getName(), $preFileDownloadEvent); + + $httpDownloader = $preFileDownloadEvent->getHttpDownloader(); + $options = $lastModifiedTime ? array('http' => array('header' => array('If-Modified-Since: '.$lastModifiedTime))) : array(); + + $io = $this->io; + $url = $this->url; + $cache = $this->cache; + $degradedMode =& $this->degradedMode; + + $accept = function ($response) use ($io, $url, $cache, $cacheKey) { + $json = $response->getBody(); + if ($json === '' && $response->getStatusCode() === 304) { + return true; + } + + $data = $response->decodeJson(); + if (!empty($data['warning'])) { + $io->writeError('Warning from '.$url.': '.$data['warning'].''); + } + if (!empty($data['info'])) { + $io->writeError('Info from '.$url.': '.$data['info'].''); + } + + $lastModifiedDate = $response->getHeader('last-modified'); + $response->collect(); + if ($lastModifiedDate) { + $data['last-modified'] = $lastModifiedDate; + $json = JsonFile::encode($data, JsonFile::JSON_UNESCAPED_SLASHES | JsonFile::JSON_UNESCAPED_UNICODE); + } + $cache->write($cacheKey, $json); + + return $data; + }; + + $reject = function ($e) use (&$retries, $httpDownloader, $filename, $options, &$reject, $accept, $io, $url, $cache, &$degradedMode) { + var_dump('Caught8', $e->getMessage()); + if ($e instanceof TransportException && $e->getStatusCode() === 404) { + return false; + } + + if (--$retries) { + usleep(100000); + + return $httpDownloader->add($filename, $options)->then($accept, $reject); + } + + if (!$degradedMode) { + $io->writeError(''.$e->getMessage().''); + $io->writeError(''.$url.' could not be fully loaded, package information was loaded from the local cache and may be out of date'); + } + $degradedMode = true; + + return true; + }; + + return $httpDownloader->add($filename, $options)->then($accept, $reject); + } + /** * This initializes the packages key of a partial packages.json that contain some packages inlined + a providers-lazy-url * diff --git a/src/Composer/Repository/RepositoryFactory.php b/src/Composer/Repository/RepositoryFactory.php index ca479a7fd..5737a5359 100644 --- a/src/Composer/Repository/RepositoryFactory.php +++ b/src/Composer/Repository/RepositoryFactory.php @@ -16,7 +16,7 @@ use Composer\Factory; use Composer\IO\IOInterface; use Composer\Config; use Composer\EventDispatcher\EventDispatcher; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; use Composer\Json\JsonFile; /** @@ -108,10 +108,10 @@ class RepositoryFactory * @param IOInterface $io * @param Config $config * @param EventDispatcher $eventDispatcher - * @param RemoteFilesystem $rfs + * @param HttpDownloader $rfs * @return RepositoryManager */ - public static function manager(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, RemoteFilesystem $rfs = null) + public static function manager(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, HttpDownloader $rfs = null) { $rm = new RepositoryManager($io, $config, $eventDispatcher, $rfs); $rm->setRepositoryClass('composer', 'Composer\Repository\ComposerRepository'); diff --git a/src/Composer/Repository/RepositoryManager.php b/src/Composer/Repository/RepositoryManager.php index 87b82d14d..64568514c 100644 --- a/src/Composer/Repository/RepositoryManager.php +++ b/src/Composer/Repository/RepositoryManager.php @@ -16,7 +16,7 @@ use Composer\IO\IOInterface; use Composer\Config; use Composer\EventDispatcher\EventDispatcher; use Composer\Package\PackageInterface; -use Composer\Util\RemoteFilesystem; +use Composer\Util\HttpDownloader; /** * Repositories manager. @@ -35,7 +35,7 @@ class RepositoryManager private $eventDispatcher; private $rfs; - public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, RemoteFilesystem $rfs = null) + public function __construct(IOInterface $io, Config $config, EventDispatcher $eventDispatcher = null, HttpDownloader $rfs = null) { $this->io = $io; $this->config = $config; @@ -127,7 +127,7 @@ class RepositoryManager $reflMethod = new \ReflectionMethod($class, '__construct'); $params = $reflMethod->getParameters(); - if (isset($params[4]) && $params[4]->getClass() && $params[4]->getClass()->getName() === 'Composer\Util\RemoteFilesystem') { + if (isset($params[4]) && $params[4]->getClass() && $params[4]->getClass()->getName() === 'Composer\Util\HttpDownloader') { return new $class($config, $this->io, $this->config, $this->eventDispatcher, $this->rfs); } diff --git a/src/Composer/Util/Http/CurlDownloader.php b/src/Composer/Util/Http/CurlDownloader.php new file mode 100644 index 000000000..846c41883 --- /dev/null +++ b/src/Composer/Util/Http/CurlDownloader.php @@ -0,0 +1,282 @@ + + * Jordi Boggiano + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Composer\Util\Http; + +use Composer\Config; +use Composer\IO\IOInterface; +use Composer\Downloader\TransportException; +use Composer\CaBundle\CaBundle; +use Psr\Log\LoggerInterface; +use React\Promise\Promise; + +/** + * @author Jordi Boggiano + * @author Nicolas Grekas + */ +class CurlDownloader +{ + private $multiHandle; + private $shareHandle; + private $jobs = array(); + private $io; + private $selectTimeout = 5.0; + protected $multiErrors = array( + CURLM_BAD_HANDLE => array('CURLM_BAD_HANDLE', 'The passed-in handle is not a valid CURLM handle.'), + CURLM_BAD_EASY_HANDLE => array('CURLM_BAD_EASY_HANDLE', "An easy handle was not good/valid. It could mean that it isn't an easy handle at all, or possibly that the handle already is in used by this or another multi handle."), + CURLM_OUT_OF_MEMORY => array('CURLM_OUT_OF_MEMORY', 'You are doomed.'), + CURLM_INTERNAL_ERROR => array('CURLM_INTERNAL_ERROR', 'This can only be returned if libcurl bugs. Please report it to us!') + ); + + private static $options = array( + 'http' => array( + 'method' => CURLOPT_CUSTOMREQUEST, + 'content' => CURLOPT_POSTFIELDS, + 'proxy' => CURLOPT_PROXY, + ), + 'ssl' => array( + 'ciphers' => CURLOPT_SSL_CIPHER_LIST, + 'cafile' => CURLOPT_CAINFO, + 'capath' => CURLOPT_CAPATH, + ), + ); + + private static $timeInfo = array( + 'total_time' => true, + 'namelookup_time' => true, + 'connect_time' => true, + 'pretransfer_time' => true, + 'starttransfer_time' => true, + 'redirect_time' => true, + ); + + public function __construct(IOInterface $io, Config $config, array $options = array(), $disableTls = false) + { + $this->io = $io; + + $this->multiHandle = $mh = curl_multi_init(); + if (function_exists('curl_multi_setopt')) { + curl_multi_setopt($mh, CURLMOPT_PIPELINING, /*CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX*/ 3); + if (defined('CURLMOPT_MAX_HOST_CONNECTIONS')) { + curl_multi_setopt($mh, CURLMOPT_MAX_HOST_CONNECTIONS, 8); + } + } + + if (function_exists('curl_share_init')) { + $this->shareHandle = $sh = curl_share_init(); + curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE); + curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); + curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); + } + } + + public function download($resolve, $reject, $origin, $url, $options, $copyTo = null) + { + $ch = curl_init(); + $hd = fopen('php://temp/maxmemory:32768', 'w+b'); + + // TODO auth & other context + // TODO cleanup + + if ($copyTo && !$fd = @fopen($copyTo.'~', 'w+b')) { + // TODO throw here probably? + $copyTo = null; + } + if (!$copyTo) { + $fd = @fopen('php://temp/maxmemory:524288', 'w+b'); + } + + if (!isset($options['http']['header'])) { + $options['http']['header'] = array(); + } + + $headers = array_diff($options['http']['header'], array('Connection: close')); + + // TODO + $degradedMode = false; + if ($degradedMode) { + curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0); + } else { + $headers[] = 'Connection: keep-alive'; + $version = curl_version(); + $features = $version['features']; + if (0 === strpos($url, 'https://') && \defined('CURL_VERSION_HTTP2') && \defined('CURL_HTTP_VERSION_2_0') && (CURL_VERSION_HTTP2 & $features)) { + curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); + } + } + + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + //curl_setopt($ch, CURLOPT_DNS_USE_GLOBAL_CACHE, false); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); + curl_setopt($ch, CURLOPT_TIMEOUT, 10); // TODO increase + curl_setopt($ch, CURLOPT_WRITEHEADER, $hd); + curl_setopt($ch, CURLOPT_FILE, $fd); + if (function_exists('curl_share_init')) { + curl_setopt($ch, CURLOPT_SHARE, $this->shareHandle); + } + + foreach (self::$options as $type => $curlOptions) { + foreach ($curlOptions as $name => $curlOption) { + if (isset($options[$type][$name])) { + curl_setopt($ch, $curlOption, $options[$type][$name]); + } + } + } + + $progress = array_diff_key(curl_getinfo($ch), self::$timeInfo); + + $this->jobs[(int) $ch] = array( + 'progress' => $progress, + 'ch' => $ch, + //'callback' => $params['notification'], + 'file' => $copyTo, + 'hd' => $hd, + 'fd' => $fd, + 'resolve' => $resolve, + 'reject' => $reject, + ); + + $this->io->write('Downloading '.$url, true, IOInterface::DEBUG); + + $this->checkCurlResult(curl_multi_add_handle($this->multiHandle, $ch)); + //$params['notification'](STREAM_NOTIFY_RESOLVE, STREAM_NOTIFY_SEVERITY_INFO, '', 0, 0, 0, false); + } + + public function tick() + { + // TODO check we have active handles before doing this + if (!$this->jobs) { + return; + } + + $active = true; + try { + $this->checkCurlResult(curl_multi_exec($this->multiHandle, $active)); + if (-1 === curl_multi_select($this->multiHandle, $this->selectTimeout)) { + // sleep in case select returns -1 as it can happen on old php versions or some platforms where curl does not manage to do the select + usleep(150); + } + + while ($progress = curl_multi_info_read($this->multiHandle)) { + $h = $progress['handle']; + $i = (int) $h; + if (!isset($this->jobs[$i])) { + continue; + } + $progress = array_diff_key(curl_getinfo($h), self::$timeInfo); + $job = $this->jobs[$i]; + unset($this->jobs[$i]); + curl_multi_remove_handle($this->multiHandle, $h); + $error = curl_error($h); + $errno = curl_errno($h); + curl_close($h); + + try { + //$this->onProgress($h, $job['callback'], $progress, $job['progress']); + if ('' !== $error) { + throw new TransportException(curl_error($h)); + } + + if ($job['file']) { + if (CURLE_OK === $errno) { + fclose($job['fd']); + rename($job['file'].'~', $job['file']); + call_user_func($job['resolve'], true); + } + // TODO otherwise show error? + } else { + rewind($job['hd']); + $headers = explode("\r\n", rtrim(stream_get_contents($job['hd']))); + fclose($job['hd']); + rewind($job['fd']); + $contents = stream_get_contents($job['fd']); + fclose($job['fd']); + $this->io->writeError('['.$progress['http_code'].'] '.$progress['url'], true, IOInterface::DEBUG); + call_user_func($job['resolve'], new Response(array('url' => $progress['url']), $progress['http_code'], $headers, $contents)); + } + } catch (TransportException $e) { + fclose($job['hd']); + fclose($job['fd']); + if ($job['file']) { + @unlink($job['file'].'~'); + } + call_user_func($job['reject'], $e); + } + } + + foreach ($this->jobs as $i => $h) { + if (!isset($this->jobs[$i])) { + continue; + } + $h = $this->jobs[$i]['ch']; + $progress = array_diff_key(curl_getinfo($h), self::$timeInfo); + + if ($this->jobs[$i]['progress'] !== $progress) { + $previousProgress = $this->jobs[$i]['progress']; + $this->jobs[$i]['progress'] = $progress; + try { + //$this->onProgress($h, $this->jobs[$i]['callback'], $progress, $previousProgress); + } catch (TransportException $e) { + var_dump('Caught '.$e->getMessage());die; + unset($this->jobs[$i]); + curl_multi_remove_handle($this->multiHandle, $h); + curl_close($h); + + fclose($job['hd']); + fclose($job['fd']); + if ($job['file']) { + @unlink($job['file'].'~'); + } + call_user_func($job['reject'], $e); + } + } + } + } catch (\Exception $e) { + var_dump('Caught2', get_class($e), $e->getMessage(), $e);die; + } + +// TODO finalize / resolve +// if ($copyTo && !isset($this->exceptions[(int) $ch])) { +// $fd = fopen($copyTo, 'rb'); +// } +// + } + + private function onProgress($ch, callable $notify, array $progress, array $previousProgress) + { + if (300 <= $progress['http_code'] && $progress['http_code'] < 400) { + return; + } + if (!$previousProgress['http_code'] && $progress['http_code'] && $progress['http_code'] < 200 || 400 <= $progress['http_code']) { + $code = 403 === $progress['http_code'] ? STREAM_NOTIFY_AUTH_RESULT : STREAM_NOTIFY_FAILURE; + $notify($code, STREAM_NOTIFY_SEVERITY_ERR, curl_error($ch), $progress['http_code'], 0, 0, false); + } + if ($previousProgress['download_content_length'] < $progress['download_content_length']) { + $notify(STREAM_NOTIFY_FILE_SIZE_IS, STREAM_NOTIFY_SEVERITY_INFO, '', 0, 0, (int) $progress['download_content_length'], false); + } + if ($previousProgress['size_download'] < $progress['size_download']) { + $notify(STREAM_NOTIFY_PROGRESS, STREAM_NOTIFY_SEVERITY_INFO, '', 0, (int) $progress['size_download'], (int) $progress['download_content_length'], false); + } + } + + private function checkCurlResult($code) + { + if ($code != CURLM_OK && $code != CURLM_CALL_MULTI_PERFORM) { + throw new \RuntimeException(isset($this->multiErrors[$code]) + ? "cURL error: {$code} ({$this->multiErrors[$code][0]}): cURL message: {$this->multiErrors[$code][1]}" + : 'Unexpected cURL error: ' . $code + ); + } + } +} diff --git a/src/Composer/Util/Http/Response.php b/src/Composer/Util/Http/Response.php new file mode 100644 index 000000000..ff48fdb40 --- /dev/null +++ b/src/Composer/Util/Http/Response.php @@ -0,0 +1,75 @@ + + * Jordi Boggiano + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Composer\Util\Http; + +use Composer\Json\JsonFile; + +class Response +{ + private $request; + private $code; + private $headers; + private $body; + + public function __construct(array $request, $code, array $headers, $body) + { + $this->request = $request; + $this->code = $code; + $this->headers = $headers; + $this->body = $body; + } + + public function getStatusCode() + { + return $this->code; + } + + public function getHeaders() + { + return $this->headers; + } + + public function getHeader($name) + { + $value = null; + foreach ($this->headers as $header) { + if (preg_match('{^'.$name.':\s*(.+?)\s*$}i', $header, $match)) { + $value = $match[1]; + } elseif (preg_match('{^HTTP/}i', $header)) { + // TODO ideally redirects would be handled in CurlDownloader/RemoteFilesystem and this becomes unnecessary + // + // In case of redirects, http_response_headers contains the headers of all responses + // so we reset the flag when a new response is being parsed as we are only interested in the last response + $value = null; + } + } + + return $value; + } + + + public function getBody() + { + return $this->body; + } + + public function decodeJson() + { + return JsonFile::parseJson($this->body, $this->request['url']); + } + + public function collect() + { + $this->request = $this->code = $this->headers = $this->body = null; + } +} diff --git a/src/Composer/Util/HttpDownloader.php b/src/Composer/Util/HttpDownloader.php new file mode 100644 index 000000000..31c615e0c --- /dev/null +++ b/src/Composer/Util/HttpDownloader.php @@ -0,0 +1,246 @@ + + * Jordi Boggiano + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Composer\Util; + +use Composer\Config; +use Composer\IO\IOInterface; +use Composer\Downloader\TransportException; +use Composer\CaBundle\CaBundle; +use Psr\Log\LoggerInterface; +use React\Promise\Promise; + +/** + * @author Jordi Boggiano + */ +class HttpDownloader +{ + const STATUS_QUEUED = 1; + const STATUS_STARTED = 2; + const STATUS_COMPLETED = 3; + const STATUS_FAILED = 4; + + private $io; + private $config; + private $jobs = array(); + private $index; + private $progress; + private $lastProgress; + private $disableTls = false; + private $curl; + private $rfs; + private $idGen = 0; + + /** + * Constructor. + * + * @param IOInterface $io The IO instance + * @param Config $config The config + * @param array $options The options + * @param bool $disableTls + */ + public function __construct(IOInterface $io, Config $config, array $options = array(), $disableTls = false) + { + $this->io = $io; + + // Setup TLS options + // The cafile option can be set via config.json + if ($disableTls === false) { + $logger = $io instanceof LoggerInterface ? $io : null; + $this->options = StreamContextFactory::getTlsDefaults($options, $logger); + } else { + $this->disableTls = true; + } + + // handle the other externally set options normally. + $this->options = array_replace_recursive($this->options, $options); + $this->config = $config; + + if (extension_loaded('curl')) { + $this->curl = new Http\CurlDownloader($io, $config, $options, $disableTls); + } + + $this->rfs = new RemoteFilesystem($io, $config, $options, $disableTls); + } + + public function get($url, $options = array()) + { + list($job, $promise) = $this->addJob(array('url' => $url, 'options' => $options, 'copyTo' => false), true); + $this->wait($job['id']); + + return $this->getResponse($job['id']); + } + + public function add($url, $options = array()) + { + list($job, $promise) = $this->addJob(array('url' => $url, 'options' => $options, 'copyTo' => false)); + + return $promise; + } + + public function copy($url, $to, $options = array()) + { + list($job, $promise) = $this->addJob(array('url' => $url, 'options' => $options, 'copyTo' => $to), true); + $this->wait($job['id']); + + return $this->getResponse($job['id']); + } + + public function addCopy($url, $to, $options = array()) + { + list($job, $promise) = $this->addJob(array('url' => $url, 'options' => $options, 'copyTo' => $to)); + + return $promise; + } + + private function addJob($request, $sync = false) + { + $job = array( + 'id' => $this->idGen++, + 'status' => self::STATUS_QUEUED, + 'request' => $request, + 'sync' => $sync, + ); + + $curl = $this->curl; + $rfs = $this->rfs; + $io = $this->io; + + $origin = $this->getOrigin($job['request']['url']); + + // TODO only send http/https through curl + if ($curl) { + $resolver = function ($resolve, $reject) use (&$job, $curl, $origin) { + // start job + $url = $job['request']['url']; + $options = $job['request']['options']; + + $job['status'] = HttpDownloader::STATUS_STARTED; + + if ($job['request']['copyTo']) { + $curl->download($resolve, $reject, $origin, $url, $options, $job['request']['copyTo']); + } else { + $curl->download($resolve, $reject, $origin, $url, $options); + } + }; + } else { + $resolver = function ($resolve, $reject) use (&$job, $rfs, $curl, $origin) { + // start job + $url = $job['request']['url']; + $options = $job['request']['options']; + + $job['status'] = HttpDownloader::STATUS_STARTED; + + if ($job['request']['copyTo']) { + if ($curl) { + $result = $curl->download($origin, $url, $options, $job['request']['copyTo']); + } else { + $result = $rfs->copy($origin, $url, $job['request']['copyTo'], false /* TODO progress */, $options); + } + + $resolve($result); + } else { + $body = $rfs->getContents($origin, $url, false /* TODO progress */, $options); + $headers = $rfs->getLastHeaders(); + $response = new Http\Response($job['request'], $rfs->findStatusCode($headers), $headers, $body); + + $resolve($response); + } + }; + } + + $canceler = function () {}; + + $promise = new Promise($resolver, $canceler); + $promise->then(function ($response) use (&$job) { + $job['status'] = HttpDownloader::STATUS_COMPLETED; + $job['response'] = $response; + // TODO look for more jobs to start once we throttle to max X jobs + }, function ($e) use ($io, &$job) { + var_dump(__CLASS__ . __LINE__); + var_dump(gettype($e)); + var_dump($e->getMessage()); + die; + $job['status'] = HttpDownloader::STATUS_FAILED; + $job['exception'] = $e; + }); + $this->jobs[$job['id']] =& $job; + + return array($job, $promise); + } + + public function wait($index = null, $progress = false) + { + while (true) { + if ($this->curl) { + $this->curl->tick(); + } + + if (null !== $index) { + if ($this->jobs[$index]['status'] === self::STATUS_COMPLETED || $this->jobs[$index]['status'] === self::STATUS_FAILED) { + return; + } + } else { + $done = true; + foreach ($this->jobs as $job) { + if (!in_array($job['status'], array(self::STATUS_COMPLETED, self::STATUS_FAILED), true)) { + $done = false; + break; + } elseif (!$job['sync']) { + unset($this->jobs[$job['id']]); + } + } + if ($done) { + return; + } + } + + usleep(1000); + } + } + + private function getResponse($index) + { + if (!isset($this->jobs[$index])) { + throw new \LogicException('Invalid request id'); + } + + if ($this->jobs[$index]['status'] === self::STATUS_FAILED) { + throw $this->jobs[$index]['exception']; + } + + if (!isset($this->jobs[$index]['response'])) { + throw new \LogicException('Response not available yet, call wait() first'); + } + + $resp = $this->jobs[$index]['response']; + + unset($this->jobs[$index]); + + return $resp; + } + + private function getOrigin($url) + { + $origin = parse_url($url, PHP_URL_HOST); + + if ($origin === 'api.github.com') { + return 'github.com'; + } + + if ($origin === 'repo.packagist.org') { + return 'packagist.org'; + } + + return $origin ?: $url; + } +} diff --git a/src/Composer/Util/RemoteFilesystem.php b/src/Composer/Util/RemoteFilesystem.php index ea18a9e30..f4a211acb 100644 --- a/src/Composer/Util/RemoteFilesystem.php +++ b/src/Composer/Util/RemoteFilesystem.php @@ -60,7 +60,8 @@ class RemoteFilesystem // Setup TLS options // The cafile option can be set via config.json if ($disableTls === false) { - $this->options = $this->getTlsDefaults($options); + $logger = $io instanceof LoggerInterface ? $io : null; + $this->options = StreamContextFactory::getTlsDefaults($options, $logger); } else { $this->disableTls = true; } @@ -891,111 +892,6 @@ class RemoteFilesystem return false; } - /** - * @param array $options - * - * @return array - */ - private function getTlsDefaults(array $options) - { - $ciphers = implode(':', array( - 'ECDHE-RSA-AES128-GCM-SHA256', - 'ECDHE-ECDSA-AES128-GCM-SHA256', - 'ECDHE-RSA-AES256-GCM-SHA384', - 'ECDHE-ECDSA-AES256-GCM-SHA384', - 'DHE-RSA-AES128-GCM-SHA256', - 'DHE-DSS-AES128-GCM-SHA256', - 'kEDH+AESGCM', - 'ECDHE-RSA-AES128-SHA256', - 'ECDHE-ECDSA-AES128-SHA256', - 'ECDHE-RSA-AES128-SHA', - 'ECDHE-ECDSA-AES128-SHA', - 'ECDHE-RSA-AES256-SHA384', - 'ECDHE-ECDSA-AES256-SHA384', - 'ECDHE-RSA-AES256-SHA', - 'ECDHE-ECDSA-AES256-SHA', - 'DHE-RSA-AES128-SHA256', - 'DHE-RSA-AES128-SHA', - 'DHE-DSS-AES128-SHA256', - 'DHE-RSA-AES256-SHA256', - 'DHE-DSS-AES256-SHA', - 'DHE-RSA-AES256-SHA', - 'AES128-GCM-SHA256', - 'AES256-GCM-SHA384', - 'AES128-SHA256', - 'AES256-SHA256', - 'AES128-SHA', - 'AES256-SHA', - 'AES', - 'CAMELLIA', - 'DES-CBC3-SHA', - '!aNULL', - '!eNULL', - '!EXPORT', - '!DES', - '!RC4', - '!MD5', - '!PSK', - '!aECDH', - '!EDH-DSS-DES-CBC3-SHA', - '!EDH-RSA-DES-CBC3-SHA', - '!KRB5-DES-CBC3-SHA', - )); - - /** - * CN_match and SNI_server_name are only known once a URL is passed. - * They will be set in the getOptionsForUrl() method which receives a URL. - * - * cafile or capath can be overridden by passing in those options to constructor. - */ - $defaults = array( - 'ssl' => array( - 'ciphers' => $ciphers, - 'verify_peer' => true, - 'verify_depth' => 7, - 'SNI_enabled' => true, - 'capture_peer_cert' => true, - ), - ); - - if (isset($options['ssl'])) { - $defaults['ssl'] = array_replace_recursive($defaults['ssl'], $options['ssl']); - } - - $caBundleLogger = $this->io instanceof LoggerInterface ? $this->io : null; - - /** - * Attempt to find a local cafile or throw an exception if none pre-set - * The user may go download one if this occurs. - */ - if (!isset($defaults['ssl']['cafile']) && !isset($defaults['ssl']['capath'])) { - $result = CaBundle::getSystemCaRootBundlePath($caBundleLogger); - - if (is_dir($result)) { - $defaults['ssl']['capath'] = $result; - } else { - $defaults['ssl']['cafile'] = $result; - } - } - - if (isset($defaults['ssl']['cafile']) && (!is_readable($defaults['ssl']['cafile']) || !CaBundle::validateCaFile($defaults['ssl']['cafile'], $caBundleLogger))) { - throw new TransportException('The configured cafile was not valid or could not be read.'); - } - - if (isset($defaults['ssl']['capath']) && (!is_dir($defaults['ssl']['capath']) || !is_readable($defaults['ssl']['capath']))) { - throw new TransportException('The configured capath was not valid or could not be read.'); - } - - /** - * Disable TLS compression to prevent CRIME attacks where supported. - */ - if (PHP_VERSION_ID >= 50413) { - $defaults['ssl']['disable_compression'] = true; - } - - return $defaults; - } - /** * Fetch certificate common name and fingerprint for validation of SAN. * diff --git a/src/Composer/Util/StreamContextFactory.php b/src/Composer/Util/StreamContextFactory.php index 8dfd6624a..72d12115d 100644 --- a/src/Composer/Util/StreamContextFactory.php +++ b/src/Composer/Util/StreamContextFactory.php @@ -13,6 +13,8 @@ namespace Composer\Util; use Composer\Composer; +use Composer\CaBundle\CaBundle; +use Psr\Log\LoggerInterface; /** * Allows the creation of a basic context supporting http proxy @@ -153,6 +155,109 @@ final class StreamContextFactory return stream_context_create($options, $defaultParams); } + /** + * @param array $options + * + * @return array + */ + public static function getTlsDefaults(array $options, LoggerInterface $logger = null) + { + $ciphers = implode(':', array( + 'ECDHE-RSA-AES128-GCM-SHA256', + 'ECDHE-ECDSA-AES128-GCM-SHA256', + 'ECDHE-RSA-AES256-GCM-SHA384', + 'ECDHE-ECDSA-AES256-GCM-SHA384', + 'DHE-RSA-AES128-GCM-SHA256', + 'DHE-DSS-AES128-GCM-SHA256', + 'kEDH+AESGCM', + 'ECDHE-RSA-AES128-SHA256', + 'ECDHE-ECDSA-AES128-SHA256', + 'ECDHE-RSA-AES128-SHA', + 'ECDHE-ECDSA-AES128-SHA', + 'ECDHE-RSA-AES256-SHA384', + 'ECDHE-ECDSA-AES256-SHA384', + 'ECDHE-RSA-AES256-SHA', + 'ECDHE-ECDSA-AES256-SHA', + 'DHE-RSA-AES128-SHA256', + 'DHE-RSA-AES128-SHA', + 'DHE-DSS-AES128-SHA256', + 'DHE-RSA-AES256-SHA256', + 'DHE-DSS-AES256-SHA', + 'DHE-RSA-AES256-SHA', + 'AES128-GCM-SHA256', + 'AES256-GCM-SHA384', + 'AES128-SHA256', + 'AES256-SHA256', + 'AES128-SHA', + 'AES256-SHA', + 'AES', + 'CAMELLIA', + 'DES-CBC3-SHA', + '!aNULL', + '!eNULL', + '!EXPORT', + '!DES', + '!RC4', + '!MD5', + '!PSK', + '!aECDH', + '!EDH-DSS-DES-CBC3-SHA', + '!EDH-RSA-DES-CBC3-SHA', + '!KRB5-DES-CBC3-SHA', + )); + + /** + * CN_match and SNI_server_name are only known once a URL is passed. + * They will be set in the getOptionsForUrl() method which receives a URL. + * + * cafile or capath can be overridden by passing in those options to constructor. + */ + $defaults = array( + 'ssl' => array( + 'ciphers' => $ciphers, + 'verify_peer' => true, + 'verify_depth' => 7, + 'SNI_enabled' => true, + 'capture_peer_cert' => true, + ), + ); + + if (isset($options['ssl'])) { + $defaults['ssl'] = array_replace_recursive($defaults['ssl'], $options['ssl']); + } + + /** + * Attempt to find a local cafile or throw an exception if none pre-set + * The user may go download one if this occurs. + */ + if (!isset($defaults['ssl']['cafile']) && !isset($defaults['ssl']['capath'])) { + $result = CaBundle::getSystemCaRootBundlePath($logger); + + if (is_dir($result)) { + $defaults['ssl']['capath'] = $result; + } else { + $defaults['ssl']['cafile'] = $result; + } + } + + if (isset($defaults['ssl']['cafile']) && (!is_readable($defaults['ssl']['cafile']) || !CaBundle::validateCaFile($defaults['ssl']['cafile'], $logger))) { + throw new TransportException('The configured cafile was not valid or could not be read.'); + } + + if (isset($defaults['ssl']['capath']) && (!is_dir($defaults['ssl']['capath']) || !is_readable($defaults['ssl']['capath']))) { + throw new TransportException('The configured capath was not valid or could not be read.'); + } + + /** + * Disable TLS compression to prevent CRIME attacks where supported. + */ + if (PHP_VERSION_ID >= 50413) { + $defaults['ssl']['disable_compression'] = true; + } + + return $defaults; + } + /** * A bug in PHP prevents the headers from correctly being sent when a content-type header is present and * NOT at the end of the array