Port/extract most behavior of RemoteFilesystem to CurlDownloader

src/Composer/Composer.php

@@ -32,6 +32,7 @@ class Composer
     const VERSION = '@package_version@';
     const BRANCH_ALIAS_VERSION = '@package_branch_alias_version@';
     const RELEASE_DATE = '@release_date@';
+    const SOURCE_VERSION = '2.0-source';
 
     /**
      * @var Package\RootPackageInterface

src/Composer/Util/AuthHelper.php

@@ -14,6 +14,7 @@ namespace Composer\Util;
 
 use Composer\Config;
 use Composer\IO\IOInterface;
+use Composer\Downloader\TransportException;
 
 /**
  * @author Jordi Boggiano <j.boggiano@seld.be>
@@ -60,4 +61,172 @@ class AuthHelper
             );
         }
     }
+
+
+    public function promptAuthIfNeeded($url, $origin, $httpStatus, $reason = null, $warning = null, $headers = array())
+    {
+        $storeAuth = false;
+        $retry = false;
+
+        if (in_array($origin, $this->config->get('github-domains'), true)) {
+            $gitHubUtil = new GitHub($this->io, $this->config, null);
+            $message = "\n";
+
+            $rateLimited = $gitHubUtil->isRateLimited($headers);
+            if ($rateLimited) {
+                $rateLimit = $gitHubUtil->getRateLimit($headers);
+                if ($this->io->hasAuthentication($origin)) {
+                    $message = 'Review your configured GitHub OAuth token or enter a new one to go over the API rate limit.';
+                } else {
+                    $message = 'Create a GitHub OAuth token to go over the API rate limit.';
+                }
+
+                $message = sprintf(
+                    'GitHub API limit (%d calls/hr) is exhausted, could not fetch '.$url.'. '.$message.' You can also wait until %s for the rate limit to reset.',
+                    $rateLimit['limit'],
+                    $rateLimit['reset']
+                )."\n";
+            } else {
+                $message .= 'Could not fetch '.$url.', please ';
+                if ($this->io->hasAuthentication($origin)) {
+                    $message .= 'review your configured GitHub OAuth token or enter a new one to access private repos';
+                } else {
+                    $message .= 'create a GitHub OAuth token to access private repos';
+                }
+            }
+
+            if (!$gitHubUtil->authorizeOAuth($origin)
+                && (!$this->io->isInteractive() || !$gitHubUtil->authorizeOAuthInteractively($origin, $message))
+            ) {
+                throw new TransportException('Could not authenticate against '.$origin, 401);
+            }
+        } elseif (in_array($origin, $this->config->get('gitlab-domains'), true)) {
+            $message = "\n".'Could not fetch '.$url.', enter your ' . $origin . ' credentials ' .($httpStatus === 401 ? 'to access private repos' : 'to go over the API rate limit');
+            $gitLabUtil = new GitLab($this->io, $this->config, null);
+
+            if ($this->io->hasAuthentication($origin) && ($auth = $this->io->getAuthentication($origin)) && $auth['password'] === 'private-token') {
+                throw new TransportException("Invalid credentials for '" . $url . "', aborting.", $httpStatus);
+            }
+
+            if (!$gitLabUtil->authorizeOAuth($origin)
+                && (!$this->io->isInteractive() || !$gitLabUtil->authorizeOAuthInteractively(parse_url($url, PHP_URL_SCHEME), $origin, $message))
+            ) {
+                throw new TransportException('Could not authenticate against '.$origin, 401);
+            }
+        } elseif ($origin === 'bitbucket.org') {
+            $askForOAuthToken = true;
+            if ($this->io->hasAuthentication($origin)) {
+                $auth = $this->io->getAuthentication($origin);
+                if ($auth['username'] !== 'x-token-auth') {
+                    $bitbucketUtil = new Bitbucket($this->io, $this->config);
+                    $accessToken = $bitbucketUtil->requestToken($origin, $auth['username'], $auth['password']);
+                    if (!empty($accessToken)) {
+                        $this->io->setAuthentication($origin, 'x-token-auth', $accessToken);
+                        $askForOAuthToken = false;
+                    }
+                } else {
+                    throw new TransportException('Could not authenticate against ' . $origin, 401);
+                }
+            }
+
+            if ($askForOAuthToken) {
+                $message = "\n".'Could not fetch ' . $url . ', please create a bitbucket OAuth token to ' . (($httpStatus === 401 || $httpStatus === 403) ? 'access private repos' : 'go over the API rate limit');
+                $bitBucketUtil = new Bitbucket($this->io, $this->config);
+                if (! $bitBucketUtil->authorizeOAuth($origin)
+                    && (! $this->io->isInteractive() || !$bitBucketUtil->authorizeOAuthInteractively($origin, $message))
+                ) {
+                    throw new TransportException('Could not authenticate against ' . $origin, 401);
+                }
+            }
+        } else {
+            // 404s are only handled for github
+            if ($httpStatus === 404) {
+                return;
+            }
+
+            // fail if the console is not interactive
+            if (!$this->io->isInteractive()) {
+                if ($httpStatus === 401) {
+                    $message = "The '" . $url . "' URL required authentication.\nYou must be using the interactive console to authenticate";
+                }
+                if ($httpStatus === 403) {
+                    $message = "The '" . $url . "' URL could not be accessed: " . $reason;
+                }
+
+                throw new TransportException($message, $httpStatus);
+            }
+            // fail if we already have auth
+            if ($this->io->hasAuthentication($origin)) {
+                throw new TransportException("Invalid credentials for '" . $url . "', aborting.", $httpStatus);
+            }
+
+            $this->io->overwriteError('');
+            if ($warning) {
+                $this->io->writeError('    <warning>'.$warning.'</warning>');
+            }
+            $this->io->writeError('    Authentication required (<info>'.parse_url($url, PHP_URL_HOST).'</info>):');
+            $username = $this->io->ask('      Username: ');
+            $password = $this->io->askAndHideAnswer('      Password: ');
+            $this->io->setAuthentication($origin, $username, $password);
+            $storeAuth = $this->config->get('store-auths');
+        }
+
+        $retry = true;
+
+        return array('retry' => $retry, 'storeAuth' => $storeAuth);
+    }
+
+    public function addAuthenticationHeader(array $headers, $origin, $url)
+    {
+        if ($this->io->hasAuthentication($origin)) {
+            $auth = $this->io->getAuthentication($origin);
+            if ('github.com' === $origin && 'x-oauth-basic' === $auth['password']) {
+                $headers[] = 'Authorization: token '.$auth['username'];
+            } elseif (in_array($origin, $this->config->get('gitlab-domains'), true)) {
+                if ($auth['password'] === 'oauth2') {
+                    $headers[] = 'Authorization: Bearer '.$auth['username'];
+                } elseif ($auth['password'] === 'private-token') {
+                    $headers[] = 'PRIVATE-TOKEN: '.$auth['username'];
+                }
+            } elseif (
+                'bitbucket.org' === $origin
+                && $url !== Bitbucket::OAUTH2_ACCESS_TOKEN_URL
+                && 'x-token-auth' === $auth['username']
+            ) {
+                if (!$this->isPublicBitBucketDownload($url)) {
+                    $headers[] = 'Authorization: Bearer ' . $auth['password'];
+                }
+            } else {
+                $authStr = base64_encode($auth['username'] . ':' . $auth['password']);
+                $headers[] = 'Authorization: Basic '.$authStr;
+            }
+        }
+
+        return $headers;
+    }
+
+    /**
+     * @link https://github.com/composer/composer/issues/5584
+     *
+     * @param string $urlToBitBucketFile URL to a file at bitbucket.org.
+     *
+     * @return bool Whether the given URL is a public BitBucket download which requires no authentication.
+     */
+    public function isPublicBitBucketDownload($urlToBitBucketFile)
+    {
+        $domain = parse_url($urlToBitBucketFile, PHP_URL_HOST);
+        if (strpos($domain, 'bitbucket.org') === false) {
+            // Bitbucket downloads are hosted on amazonaws.
+            // We do not need to authenticate there at all
+            return true;
+        }
+
+        $path = parse_url($urlToBitBucketFile, PHP_URL_PATH);
+
+        // Path for a public download follows this pattern /{user}/{repo}/downloads/{whatever}
+        // {@link https://blog.bitbucket.org/2009/04/12/new-feature-downloads/}
+        $pathParts = explode('/', $path);
+
+        return count($pathParts) >= 4 && $pathParts[3] == 'downloads';
+    }
 }

src/Composer/Util/Http/CurlDownloader.php

@@ -16,6 +16,9 @@ use Composer\Config;
 use Composer\IO\IOInterface;
 use Composer\Downloader\TransportException;
 use Composer\CaBundle\CaBundle;
+use Composer\Util\RemoteFilesystem;
+use Composer\Util\StreamContextFactory;
+use Composer\Util\AuthHelper;
 use Psr\Log\LoggerInterface;
 use React\Promise\Promise;
 
@@ -28,8 +31,14 @@ class CurlDownloader
     private $multiHandle;
     private $shareHandle;
     private $jobs = array();
+    /** @var IOInterface */
     private $io;
+    /** @var Config */
+    private $config;
+    /** @var AuthHelper */
+    private $authHelper;
     private $selectTimeout = 5.0;
+    private $maxRedirects = 20;
     protected $multiErrors = array(
         CURLM_BAD_HANDLE      => array('CURLM_BAD_HANDLE', 'The passed-in handle is not a valid CURLM handle.'),
         CURLM_BAD_EASY_HANDLE => array('CURLM_BAD_EASY_HANDLE', "An easy handle was not good/valid. It could mean that it isn't an easy handle at all, or possibly that the handle already is in used by this or another multi handle."),
@@ -42,6 +51,7 @@ class CurlDownloader
             'method' => CURLOPT_CUSTOMREQUEST,
             'content' => CURLOPT_POSTFIELDS,
             'proxy' => CURLOPT_PROXY,
+            'header' => CURLOPT_HTTPHEADER,
         ),
         'ssl' => array(
             'ciphers' => CURLOPT_SSL_CIPHER_LIST,
@@ -62,6 +72,7 @@ class CurlDownloader
     public function __construct(IOInterface $io, Config $config, array $options = array(), $disableTls = false)
     {
         $this->io = $io;
+        $this->config = $config;
 
         $this->multiHandle = $mh = curl_multi_init();
         if (function_exists('curl_multi_setopt')) {
@@ -77,79 +88,112 @@ class CurlDownloader
             curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
             curl_share_setopt($sh, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
         }
+
+        $this->authHelper = new AuthHelper($io, $config);
     }
 
     public function download($resolve, $reject, $origin, $url, $options, $copyTo = null)
     {
-        $ch = curl_init();
-        $hd = fopen('php://temp/maxmemory:32768', 'w+b');
+        return $this->initDownload($resolve, $reject, $origin, $url, $options, $copyTo);
+    }
 
-        // TODO auth & other context
-        // TODO cleanup
+    private function initDownload($resolve, $reject, $origin, $url, $options, $copyTo = null, array $attributes = array())
+    {
+        // TODO allow setting attributes somehow
+        $attributes = array_merge(array(
+            'retryAuthFailure' => true,
+            'redirects' => 1,
+            'storeAuth' => false,
+        ), $attributes);
+
+        $originalOptions = $options;
+
+        // check URL can be accessed (i.e. is not insecure)
+        $this->config->prohibitUrlByConfig($url, $this->io);
+
+        $curlHandle = curl_init();
+        $headerHandle = fopen('php://temp/maxmemory:32768', 'w+b');
+
+        if ($copyTo) {
+            $errorMessage = '';
+            set_error_handler(function ($code, $msg) use (&$errorMessage) {
+                if ($errorMessage) {
+                    $errorMessage .= "\n";
+                }
+                $errorMessage .= preg_replace('{^fopen\(.*?\): }', '', $msg);
+            });
+            $bodyHandle = fopen($copyTo.'~', 'w+b');
+            restore_error_handler();
+            if (!$bodyHandle) {
+                throw new TransportException('The "'.$url.'" file could not be written to '.$copyTo.': '.$errorMessage);
+            }
+        } else {
+            $bodyHandle = @fopen('php://temp/maxmemory:524288', 'w+b');
+        }
 
-        if ($copyTo && !$fd = @fopen($copyTo.'~', 'w+b')) {
-            // TODO throw here probably?
-            $copyTo = null;
+        curl_setopt($curlHandle, CURLOPT_URL, $url);
+        curl_setopt($curlHandle, CURLOPT_FOLLOWLOCATION, true);
+        curl_setopt($curlHandle, CURLOPT_MAXREDIRS, 20);
+        //curl_setopt($curlHandle, CURLOPT_DNS_USE_GLOBAL_CACHE, false);
+        curl_setopt($curlHandle, CURLOPT_CONNECTTIMEOUT, 10);
+        curl_setopt($curlHandle, CURLOPT_TIMEOUT, 10); // TODO increase
+        curl_setopt($curlHandle, CURLOPT_WRITEHEADER, $headerHandle);
+        curl_setopt($curlHandle, CURLOPT_FILE, $bodyHandle);
+        curl_setopt($curlHandle, CURLOPT_ENCODING, "gzip");
+        curl_setopt($curlHandle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS);
+        if (defined('CURLOPT_SSL_FALSESTART')) {
+            curl_setopt($curlHandle, CURLOPT_SSL_FALSESTART, true);
         }
-        if (!$copyTo) {
-            $fd = @fopen('php://temp/maxmemory:524288', 'w+b');
+        if (function_exists('curl_share_init')) {
+            curl_setopt($curlHandle, CURLOPT_SHARE, $this->shareHandle);
         }
 
         if (!isset($options['http']['header'])) {
             $options['http']['header'] = array();
         }
 
-        $headers = array_diff($options['http']['header'], array('Connection: close'));
+        $options['http']['header'] = array_diff($options['http']['header'], array('Connection: close'));
+        $options['http']['header'][] = 'Connection: keep-alive';
 
-        // TODO
-        $degradedMode = false;
-        if ($degradedMode) {
-            curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
-        } else {
-            $headers[] = 'Connection: keep-alive';
-            $version = curl_version();
-            $features = $version['features'];
-            if (0 === strpos($url, 'https://') && \defined('CURL_VERSION_HTTP2') && \defined('CURL_HTTP_VERSION_2_0') && (CURL_VERSION_HTTP2 & $features)) {
-                curl_setopt($ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-            }
+        $version = curl_version();
+        $features = $version['features'];
+        if (0 === strpos($url, 'https://') && \defined('CURL_VERSION_HTTP2') && \defined('CURL_HTTP_VERSION_2_0') && (CURL_VERSION_HTTP2 & $features)) {
+            curl_setopt($curlHandle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
         }
 
-        curl_setopt($ch, CURLOPT_URL, $url);
-        curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
-        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
-        //curl_setopt($ch, CURLOPT_DNS_USE_GLOBAL_CACHE, false);
-        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
-        curl_setopt($ch, CURLOPT_TIMEOUT, 10); // TODO increase
-        curl_setopt($ch, CURLOPT_WRITEHEADER, $hd);
-        curl_setopt($ch, CURLOPT_FILE, $fd);
-        if (function_exists('curl_share_init')) {
-            curl_setopt($ch, CURLOPT_SHARE, $this->shareHandle);
-        }
+        $options['http']['header'] = $this->authHelper->addAuthenticationHeader($options['http']['header'], $origin, $url);
+        $options = StreamContextFactory::initOptions($url, $options);
 
         foreach (self::$options as $type => $curlOptions) {
             foreach ($curlOptions as $name => $curlOption) {
                 if (isset($options[$type][$name])) {
-                    curl_setopt($ch, $curlOption, $options[$type][$name]);
+                    curl_setopt($curlHandle, $curlOption, $options[$type][$name]);
                 }
             }
         }
 
-        $progress = array_diff_key(curl_getinfo($ch), self::$timeInfo);
+        $progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo);
 
-        $this->jobs[(int) $ch] = array(
+        $this->jobs[(int) $curlHandle] = array(
+            'url' => $url,
+            'origin' => $origin,
+            'attributes' => $attributes,
+            'options' => $originalOptions,
             'progress' => $progress,
-            'ch' => $ch,
+            'curlHandle' => $curlHandle,
             //'callback' => $params['notification'],
-            'file' => $copyTo,
-            'hd' => $hd,
-            'fd' => $fd,
+            'filename' => $copyTo,
+            'headerHandle' => $headerHandle,
+            'bodyHandle' => $bodyHandle,
             'resolve' => $resolve,
             'reject' => $reject,
         );
 
-        $this->io->write('Downloading '.$url, true, IOInterface::DEBUG);
+        $usingProxy = !empty($options['http']['proxy']) ? ' using proxy ' . $options['http']['proxy'] : '';
+        $ifModified = false !== strpos(strtolower(implode(',', $options['http']['header'])), 'if-modified-since:') ? ' if modified' : '';
+        $this->io->writeError('Downloading ' . $url . $usingProxy . $ifModified, true, IOInterface::DEBUG);
 
-        $this->checkCurlResult(curl_multi_add_handle($this->multiHandle, $ch));
+        $this->checkCurlResult(curl_multi_add_handle($this->multiHandle, $curlHandle));
         //$params['notification'](STREAM_NOTIFY_RESOLVE, STREAM_NOTIFY_SEVERITY_INFO, '', 0, 0, 0, false);
     }
 
@@ -169,74 +213,114 @@ class CurlDownloader
             }
 
             while ($progress = curl_multi_info_read($this->multiHandle)) {
-                $h = $progress['handle'];
-                $i = (int) $h;
+                $curlHandle = $progress['handle'];
+                $i = (int) $curlHandle;
                 if (!isset($this->jobs[$i])) {
                     continue;
                 }
-                $progress = array_diff_key(curl_getinfo($h), self::$timeInfo);
+                $progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo);
                 $job = $this->jobs[$i];
                 unset($this->jobs[$i]);
-                curl_multi_remove_handle($this->multiHandle, $h);
-                $error = curl_error($h);
-                $errno = curl_errno($h);
-                curl_close($h);
-
+                curl_multi_remove_handle($this->multiHandle, $curlHandle);
+                $error = curl_error($curlHandle);
+                $errno = curl_errno($curlHandle);
+                curl_close($curlHandle);
+
+                $headers = null;
+                $statusCode = null;
+                $response = null;
                 try {
-                    //$this->onProgress($h, $job['callback'], $progress, $job['progress']);
-                    if ('' !== $error) {
-                        throw new TransportException(curl_error($h));
+                    //$this->onProgress($curlHandle, $job['callback'], $progress, $job['progress']);
+                    if (CURLE_OK !== $errno) {
+                        throw new TransportException($error);
                     }
 
-                    if ($job['file']) {
-                        if (CURLE_OK === $errno) {
-                            fclose($job['fd']);
-                            rename($job['file'].'~', $job['file']);
-                            call_user_func($job['resolve'], true);
-                        }
-                        // TODO otherwise show error?
+                    $statusCode = $progress['http_code'];
+                    rewind($job['headerHandle']);
+                    $headers = explode("\r\n", rtrim(stream_get_contents($job['headerHandle'])));
+                    fclose($job['headerHandle']);
+
+                    // prepare response object
+                    if ($job['filename']) {
+                        fclose($job['bodyHandle']);
+                        $response = new Response(array('url' => $progress['url']), $statusCode, $headers, $job['filename'].'~');
                     } else {
-                        rewind($job['hd']);
-                        $headers = explode("\r\n", rtrim(stream_get_contents($job['hd'])));
-                        fclose($job['hd']);
-                        rewind($job['fd']);
-                        $contents = stream_get_contents($job['fd']);
-                        fclose($job['fd']);
-                        $this->io->writeError('['.$progress['http_code'].'] '.$progress['url'], true, IOInterface::DEBUG);
-                        call_user_func($job['resolve'], new Response(array('url' => $progress['url']), $progress['http_code'], $headers, $contents));
+                        rewind($job['bodyHandle']);
+                        $contents = stream_get_contents($job['bodyHandle']);
+                        fclose($job['bodyHandle']);
+                        $response = new Response(array('url' => $progress['url']), $statusCode, $headers, $contents);
+                        $this->io->writeError('['.$statusCode.'] '.$progress['url'], true, IOInterface::DEBUG);
+                    }
+
+                    $response = $this->retryIfAuthNeeded($job, $response);
+
+                    // handle 3xx redirects, 304 Not Modified is excluded
+                    if ($statusCode >= 300 && $statusCode <= 399 && $statusCode !== 304 && $job['redirects'] < $this->maxRedirects) {
+                        // TODO
+                        $response = $this->handleRedirect($job, $response);
+                    }
+
+                    // fail 4xx and 5xx responses and capture the response
+                    if ($statusCode >= 400 && $statusCode <= 599) {
+                        throw $this->failResponse($job, $response, $response->getStatusMessage());
+//                        $this->io->overwriteError("Downloading (<error>failed</error>)", false);
+                    }
+
+                    if ($job['attributes']['storeAuth']) {
+                        $this->authHelper->storeAuth($job['origin'], $job['attributes']['storeAuth']);
+                    }
+
+                    // resolve promise
+                    if ($job['filename']) {
+                        rename($job['filename'].'~', $job['filename']);
+                        call_user_func($job['resolve'], true);
+                    } else {
+                        call_user_func($job['resolve'], $response);
+                    }
+                } catch (\Exception $e) {
+                    if ($e instanceof TransportException && $headers) {
+                        $e->setHeaders($headers);
+                        $e->setStatusCode($statusCode);
                     }
-                } catch (TransportException $e) {
-                    fclose($job['hd']);
-                    fclose($job['fd']);
-                    if ($job['file']) {
-                        @unlink($job['file'].'~');
+                    if ($e instanceof TransportException && $response) {
+                        $e->setResponse($response->getBody());
+                    }
+
+                    if (is_resource($job['headerHandle'])) {
+                        fclose($job['headerHandle']);
+                    }
+                    if (is_resource($job['bodyHandle'])) {
+                        fclose($job['bodyHandle']);
+                    }
+                    if ($job['filename']) {
+                        @unlink($job['filename'].'~');
                     }
                     call_user_func($job['reject'], $e);
                 }
             }
 
-            foreach ($this->jobs as $i => $h) {
+            foreach ($this->jobs as $i => $curlHandle) {
                 if (!isset($this->jobs[$i])) {
                     continue;
                 }
-                $h = $this->jobs[$i]['ch'];
-                $progress = array_diff_key(curl_getinfo($h), self::$timeInfo);
+                $curlHandle = $this->jobs[$i]['curlHandle'];
+                $progress = array_diff_key(curl_getinfo($curlHandle), self::$timeInfo);
 
                 if ($this->jobs[$i]['progress'] !== $progress) {
                     $previousProgress = $this->jobs[$i]['progress'];
                     $this->jobs[$i]['progress'] = $progress;
                     try {
-                        //$this->onProgress($h, $this->jobs[$i]['callback'], $progress, $previousProgress);
+                        //$this->onProgress($curlHandle, $this->jobs[$i]['callback'], $progress, $previousProgress);
                     } catch (TransportException $e) {
                         var_dump('Caught '.$e->getMessage());die;
                         unset($this->jobs[$i]);
-                        curl_multi_remove_handle($this->multiHandle, $h);
-                        curl_close($h);
+                        curl_multi_remove_handle($this->multiHandle, $curlHandle);
+                        curl_close($curlHandle);
 
-                        fclose($job['hd']);
-                        fclose($job['fd']);
-                        if ($job['file']) {
-                            @unlink($job['file'].'~');
+                        fclose($job['headerHandle']);
+                        fclose($job['bodyHandle']);
+                        if ($job['filename']) {
+                            @unlink($job['filename'].'~');
                         }
                         call_user_func($job['reject'], $e);
                     }
@@ -245,22 +329,77 @@ class CurlDownloader
         } catch (\Exception $e) {
             var_dump('Caught2', get_class($e), $e->getMessage(), $e);die;
         }
+    }
+
+    private function retryIfAuthNeeded(array $job, Response $response)
+    {
+        if (in_array($response->getStatusCode(), array(401, 403)) && $job['attributes']['retryAuthFailure']) {
+            $warning = null;
+            if ($response->getHeader('content-type') === 'application/json') {
+                $data = json_decode($response->getBody(), true);
+                if (!empty($data['warning'])) {
+                    $warning = $data['warning'];
+                }
+            }
+
+            $result = $this->authHelper->promptAuthIfNeeded($job['url'], $job['origin'], $response->getStatusCode(), $response->getStatusMessage(), $warning, $response->getHeaders());
+
+            if ($result['retry']) {
+                // TODO retry somehow using $result['storeAuth'] in the attributes
+            }
+        }
 
-// TODO finalize / resolve
-//            if ($copyTo && !isset($this->exceptions[(int) $ch])) {
-//                $fd = fopen($copyTo, 'rb');
-//            }
-//
+        $locationHeader = $response->getHeader('location');
+        $needsAuthRetry = false;
+
+        // check for bitbucket login page asking to authenticate
+        if (
+            $job['origin'] === 'bitbucket.org'
+            && !$this->authHelper->isPublicBitBucketDownload($job['url'])
+            && substr($job['url'], -4) === '.zip'
+            && (!$locationHeader || substr($locationHeader, -4) !== '.zip')
+            && preg_match('{^text/html\b}i', $response->getHeader('content-type'))
+        ) {
+            $needsAuthRetry = 'Bitbucket requires authentication and it was not provided';
+        }
+
+        // check for gitlab 404 when downloading archives
+        if (
+            $response->getStatusCode() === 404
+            && $this->config && in_array($job['origin'], $this->config->get('gitlab-domains'), true)
+            && false !== strpos($job['url'], 'archive.zip')
+        ) {
+            $needsAuthRetry = 'GitLab requires authentication and it was not provided';
+        }
+
+        if ($needsAuthRetry) {
+            if ($job['attributes']['retryAuthFailure']) {
+                $result = $this->authHelper->promptAuthIfNeeded($job['url'], $job['origin'], 401);
+                if ($result['retry']) {
+                    // TODO ...
+                    // TODO return early here to abort failResponse
+                }
+            }
+
+            throw $this->failResponse($job, $response, $needsAuthRetry);
+        }
+
+        return $response;
+    }
+
+    private function failResponse(array $job, Response $response, $errorMessage)
+    {
+        return new TransportException('The "'.$job['url'].'" file could not be downloaded ('.$errorMessage.')', $response->getStatusCode());
     }
 
-    private function onProgress($ch, callable $notify, array $progress, array $previousProgress)
+    private function onProgress($curlHandle, callable $notify, array $progress, array $previousProgress)
     {
         if (300 <= $progress['http_code'] && $progress['http_code'] < 400) {
             return;
         }
         if (!$previousProgress['http_code'] && $progress['http_code'] && $progress['http_code'] < 200 || 400 <= $progress['http_code']) {
             $code = 403 === $progress['http_code'] ? STREAM_NOTIFY_AUTH_RESULT : STREAM_NOTIFY_FAILURE;
-            $notify($code, STREAM_NOTIFY_SEVERITY_ERR, curl_error($ch), $progress['http_code'], 0, 0, false);
+            $notify($code, STREAM_NOTIFY_SEVERITY_ERR, curl_error($curlHandle), $progress['http_code'], 0, 0, false);
         }
         if ($previousProgress['download_content_length'] < $progress['download_content_length']) {
             $notify(STREAM_NOTIFY_FILE_SIZE_IS, STREAM_NOTIFY_SEVERITY_INFO, '', 0, 0, (int) $progress['download_content_length'], false);

src/Composer/Util/Http/Response.php

@@ -27,7 +27,7 @@ class Response
             throw new \LogicException('url key missing from request array');
         }
         $this->request = $request;
-        $this->code = $code;
+        $this->code = (int) $code;
         $this->headers = $headers;
         $this->body = $body;
     }
@@ -37,6 +37,23 @@ class Response
         return $this->code;
     }
 
+    /**
+     * @return string|null
+     */
+    public function getStatusMessage()
+    {
+        $value = null;
+        foreach ($this->headers as $header) {
+            if (preg_match('{^HTTP/\S+ \d+}i', $header)) {
+                // In case of redirects, headers contain the headers of all responses
+                // so we can not return directly and need to keep iterating
+                $value = $header;
+            }
+        }
+
+        return $value;
+    }
+
     public function getHeaders()
     {
         return $this->headers;
@@ -51,7 +68,7 @@ class Response
             } elseif (preg_match('{^HTTP/}i', $header)) {
                 // TODO ideally redirects would be handled in CurlDownloader/RemoteFilesystem and this becomes unnecessary
                 //
-                // In case of redirects, http_response_headers contains the headers of all responses
+                // In case of redirects, headers contains the headers of all responses
                 // so we reset the flag when a new response is being parsed as we are only interested in the last response
                 $value = null;
             }
@@ -60,7 +77,6 @@ class Response
         return $value;
     }
 
-
     public function getBody()
     {
         return $this->body;

src/Composer/Util/HttpDownloader.php

@@ -66,6 +66,7 @@ class HttpDownloader
         $this->options = array_replace_recursive($this->options, $options);
         $this->config = $config;
 
+        // TODO enable curl only on 5.6+ if older versions cause any problem
         if (extension_loaded('curl')) {
             $this->curl = new Http\CurlDownloader($io, $config, $options, $disableTls);
         }
@@ -125,6 +126,11 @@ class HttpDownloader
 
     private function addJob($request, $sync = false)
     {
+        // capture username/password from URL if there is one
+        if (preg_match('{^https?://([^:/]+):([^@/]+)@([^/]+)}i', $request['url'], $match)) {
+            $this->io->setAuthentication($originUrl, rawurldecode($match[1]), rawurldecode($match[2]));
+        }
+
         $job = array(
             'id' => $this->idGen++,
             'status' => self::STATUS_QUEUED,
@@ -138,7 +144,6 @@ class HttpDownloader
 
         $origin = $this->getOrigin($job['request']['url']);
 
-        // TODO experiment with allowing file:// through curl too
         if ($curl && preg_match('{^https?://}i', $job['request']['url'])) {
             $resolver = function ($resolve, $reject) use (&$job, $curl, $origin) {
                 // start job
@@ -183,10 +188,10 @@ class HttpDownloader
             $job['response'] = $response;
             // TODO look for more jobs to start once we throttle to max X jobs
         }, function ($e) use ($io, &$job) {
-            var_dump(__CLASS__ . __LINE__);
-            var_dump(gettype($e));
-            var_dump($e->getMessage());
-            die;
+            // var_dump(__CLASS__ . __LINE__);
+            // var_dump(get_class($e));
+            // var_dump($e->getMessage());
+            // die;
             $job['status'] = HttpDownloader::STATUS_FAILED;
             $job['exception'] = $e;
         });
@@ -248,9 +253,13 @@ class HttpDownloader
 
     private function getOrigin($url)
     {
+        if (0 === strpos($url, 'file://')) {
+            return $url;
+        }
+
         $origin = parse_url($url, PHP_URL_HOST);
 
-        if ($origin === 'api.github.com') {
+        if (strpos($origin, '.github.com') === (strlen($origin) - 11)) {
             return 'github.com';
         }
 
@@ -258,6 +267,20 @@ class HttpDownloader
             return 'packagist.org';
         }
 
+        // Gitlab can be installed in a non-root context (i.e. gitlab.com/foo). When downloading archives the originUrl
+        // is the host without the path, so we look for the registered gitlab-domains matching the host here
+        if (
+            is_array($this->config->get('gitlab-domains'))
+            && false === strpos($origin, '/')
+            && !in_array($origin, $this->config->get('gitlab-domains'))
+        ) {
+            foreach ($this->config->get('gitlab-domains') as $gitlabDomain) {
+                if (0 === strpos($gitlabDomain, $origin)) {
+                    return $gitlabDomain;
+                }
+            }
+        }
+
         return $origin ?: $url;
     }
 }

src/Composer/Util/RemoteFilesystem.php

@@ -41,6 +41,7 @@ class RemoteFilesystem
     private $retryAuthFailure;
     private $lastHeaders;
     private $storeAuth;
+    private $authHelper;
     private $degradedMode = false;
     private $redirects;
     private $maxRedirects = 20;
@@ -53,7 +54,7 @@ class RemoteFilesystem
      * @param array       $options    The options
      * @param bool        $disableTls
      */
-    public function __construct(IOInterface $io, Config $config = null, array $options = array(), $disableTls = false)
+    public function __construct(IOInterface $io, Config $config, array $options = array(), $disableTls = false)
     {
         $this->io = $io;
 
@@ -69,6 +70,7 @@ class RemoteFilesystem
         // handle the other externally set options normally.
         $this->options = array_replace_recursive($this->options, $options);
         $this->config = $config;
+        $this->authHelper = new AuthHelper($io, $config);
     }
 
     /**
@@ -215,27 +217,6 @@ class RemoteFilesystem
      */
     protected function get($originUrl, $fileUrl, $additionalOptions = array(), $fileName = null, $progress = true)
     {
-        if (strpos($originUrl, '.github.com') === (strlen($originUrl) - 11)) {
-            $originUrl = 'github.com';
-        }
-
-        // Gitlab can be installed in a non-root context (i.e. gitlab.com/foo). When downloading archives the originUrl
-        // is the host without the path, so we look for the registered gitlab-domains matching the host here
-        if (
-            $this->config
-            && is_array($this->config->get('gitlab-domains'))
-            && false === strpos($originUrl, '/')
-            && !in_array($originUrl, $this->config->get('gitlab-domains'))
-        ) {
-            foreach ($this->config->get('gitlab-domains') as $gitlabDomain) {
-                if (0 === strpos($gitlabDomain, $originUrl)) {
-                    $originUrl = $gitlabDomain;
-                    break;
-                }
-            }
-            unset($gitlabDomain);
-        }
-
         $this->scheme = parse_url($fileUrl, PHP_URL_SCHEME);
         $this->bytesMax = 0;
         $this->originUrl = $originUrl;
@@ -247,11 +228,6 @@ class RemoteFilesystem
         $this->lastHeaders = array();
         $this->redirects = 1; // The first request counts.
 
-        // capture username/password from URL if there is one
-        if (preg_match('{^https?://([^:/]+):([^@/]+)@([^/]+)}i', $fileUrl, $match)) {
-            $this->io->setAuthentication($originUrl, rawurldecode($match[1]), rawurldecode($match[2]));
-        }
-
         $tempAdditionalOptions = $additionalOptions;
         if (isset($tempAdditionalOptions['retry-auth-failure'])) {
             $this->retryAuthFailure = (bool) $tempAdditionalOptions['retry-auth-failure'];
@@ -272,14 +248,6 @@ class RemoteFilesystem
 
         $origFileUrl = $fileUrl;
 
-        if (isset($options['github-token'])) {
-            // only add the access_token if it is actually a github URL (in case we were redirected to S3)
-            if (preg_match('{^https?://([a-z0-9-]+\.)*github\.com/}', $fileUrl)) {
-                $fileUrl .= (false === strpos($fileUrl, '?') ? '?' : '&') . 'access_token='.$options['github-token'];
-            }
-            unset($options['github-token']);
-        }
-
         if (isset($options['gitlab-token'])) {
             $fileUrl .= (false === strpos($fileUrl, '?') ? '?' : '&') . 'access_token='.$options['gitlab-token'];
             unset($options['gitlab-token']);
@@ -400,7 +368,7 @@ class RemoteFilesystem
 
         // check for bitbucket login page asking to authenticate
         if ($originUrl === 'bitbucket.org'
-            && !$this->isPublicBitBucketDownload($fileUrl)
+            && !$this->authHelper->isPublicBitBucketDownload($fileUrl)
             && substr($fileUrl, -4) === '.zip'
             && (!$locationHeader || substr($locationHeader, -4) !== '.zip')
             && $contentType && preg_match('{^text/html\b}i', $contentType)
@@ -544,8 +512,7 @@ class RemoteFilesystem
             $result = $this->get($this->originUrl, $this->fileUrl, $additionalOptions, $this->fileName, $this->progress);
 
             if ($this->storeAuth && $this->config) {
-                $authHelper = new AuthHelper($this->io, $this->config);
-                $authHelper->storeAuth($this->originUrl, $this->storeAuth);
+                $this->authHelper->storeAuth($this->originUrl, $this->storeAuth);
                 $this->storeAuth = false;
             }
 
@@ -650,111 +617,14 @@ class RemoteFilesystem
 
     protected function promptAuthAndRetry($httpStatus, $reason = null, $warning = null, $headers = array())
     {
-        if ($this->config && in_array($this->originUrl, $this->config->get('github-domains'), true)) {
-            $gitHubUtil = new GitHub($this->io, $this->config, null);
-            $message = "\n";
-
-            $rateLimited = $gitHubUtil->isRateLimited($headers);
-            if ($rateLimited) {
-                $rateLimit = $gitHubUtil->getRateLimit($headers);
-                if ($this->io->hasAuthentication($this->originUrl)) {
-                    $message = 'Review your configured GitHub OAuth token or enter a new one to go over the API rate limit.';
-                } else {
-                    $message = 'Create a GitHub OAuth token to go over the API rate limit.';
-                }
-
-                $message = sprintf(
-                    'GitHub API limit (%d calls/hr) is exhausted, could not fetch '.$this->fileUrl.'. '.$message.' You can also wait until %s for the rate limit to reset.',
-                    $rateLimit['limit'],
-                    $rateLimit['reset']
-                )."\n";
-            } else {
-                $message .= 'Could not fetch '.$this->fileUrl.', please ';
-                if ($this->io->hasAuthentication($this->originUrl)) {
-                    $message .= 'review your configured GitHub OAuth token or enter a new one to access private repos';
-                } else {
-                    $message .= 'create a GitHub OAuth token to access private repos';
-                }
-            }
-
-            if (!$gitHubUtil->authorizeOAuth($this->originUrl)
-                && (!$this->io->isInteractive() || !$gitHubUtil->authorizeOAuthInteractively($this->originUrl, $message))
-            ) {
-                throw new TransportException('Could not authenticate against '.$this->originUrl, 401);
-            }
-        } elseif ($this->config && in_array($this->originUrl, $this->config->get('gitlab-domains'), true)) {
-            $message = "\n".'Could not fetch '.$this->fileUrl.', enter your ' . $this->originUrl . ' credentials ' .($httpStatus === 401 ? 'to access private repos' : 'to go over the API rate limit');
-            $gitLabUtil = new GitLab($this->io, $this->config, null);
-
-            if ($this->io->hasAuthentication($this->originUrl) && ($auth = $this->io->getAuthentication($this->originUrl)) && $auth['password'] === 'private-token') {
-                throw new TransportException("Invalid credentials for '" . $this->fileUrl . "', aborting.", $httpStatus);
-            }
+        $result = $this->authHelper->promptAuthIfNeeded($this->fileUrl, $this->originUrl, $httpStatus, $reason, $warning, $headers);
 
-            if (!$gitLabUtil->authorizeOAuth($this->originUrl)
-                && (!$this->io->isInteractive() || !$gitLabUtil->authorizeOAuthInteractively($this->scheme, $this->originUrl, $message))
-            ) {
-                throw new TransportException('Could not authenticate against '.$this->originUrl, 401);
-            }
-        } elseif ($this->config && $this->originUrl === 'bitbucket.org') {
-            $askForOAuthToken = true;
-            if ($this->io->hasAuthentication($this->originUrl)) {
-                $auth = $this->io->getAuthentication($this->originUrl);
-                if ($auth['username'] !== 'x-token-auth') {
-                    $bitbucketUtil = new Bitbucket($this->io, $this->config);
-                    $accessToken = $bitbucketUtil->requestToken($this->originUrl, $auth['username'], $auth['password']);
-                    if (!empty($accessToken)) {
-                        $this->io->setAuthentication($this->originUrl, 'x-token-auth', $accessToken);
-                        $askForOAuthToken = false;
-                    }
-                } else {
-                    throw new TransportException('Could not authenticate against ' . $this->originUrl, 401);
-                }
-            }
+        $this->storeAuth = $result['storeAuth'];
+        $this->retry = $result['retry'];
 
-            if ($askForOAuthToken) {
-                $message = "\n".'Could not fetch ' . $this->fileUrl . ', please create a bitbucket OAuth token to ' . (($httpStatus === 401 || $httpStatus === 403) ? 'access private repos' : 'go over the API rate limit');
-                $bitBucketUtil = new Bitbucket($this->io, $this->config);
-                if (! $bitBucketUtil->authorizeOAuth($this->originUrl)
-                    && (! $this->io->isInteractive() || !$bitBucketUtil->authorizeOAuthInteractively($this->originUrl, $message))
-                ) {
-                    throw new TransportException('Could not authenticate against ' . $this->originUrl, 401);
-                }
-            }
-        } else {
-            // 404s are only handled for github
-            if ($httpStatus === 404) {
-                return;
-            }
-
-            // fail if the console is not interactive
-            if (!$this->io->isInteractive()) {
-                if ($httpStatus === 401) {
-                    $message = "The '" . $this->fileUrl . "' URL required authentication.\nYou must be using the interactive console to authenticate";
-                }
-                if ($httpStatus === 403) {
-                    $message = "The '" . $this->fileUrl . "' URL could not be accessed: " . $reason;
-                }
-
-                throw new TransportException($message, $httpStatus);
-            }
-            // fail if we already have auth
-            if ($this->io->hasAuthentication($this->originUrl)) {
-                throw new TransportException("Invalid credentials for '" . $this->fileUrl . "', aborting.", $httpStatus);
-            }
-
-            $this->io->overwriteError('');
-            if ($warning) {
-                $this->io->writeError('    <warning>'.$warning.'</warning>');
-            }
-            $this->io->writeError('    Authentication required (<info>'.parse_url($this->fileUrl, PHP_URL_HOST).'</info>):');
-            $username = $this->io->ask('      Username: ');
-            $password = $this->io->askAndHideAnswer('      Password: ');
-            $this->io->setAuthentication($this->originUrl, $username, $password);
-            $this->storeAuth = $this->config->get('store-auths');
+        if ($this->retry) {
+            throw new TransportException('RETRY');
         }
-
-        $this->retry = true;
-        throw new TransportException('RETRY');
     }
 
     protected function getOptionsForUrl($originUrl, $additionalOptions)
@@ -814,27 +684,7 @@ class RemoteFilesystem
             $headers[] = 'Connection: close';
         }
 
-        if ($this->io->hasAuthentication($originUrl)) {
-            $auth = $this->io->getAuthentication($originUrl);
-            if ('github.com' === $originUrl && 'x-oauth-basic' === $auth['password']) {
-                $options['github-token'] = $auth['username'];
-            } elseif ($this->config && in_array($originUrl, $this->config->get('gitlab-domains'), true)) {
-                if ($auth['password'] === 'oauth2') {
-                    $headers[] = 'Authorization: Bearer '.$auth['username'];
-                } elseif ($auth['password'] === 'private-token') {
-                    $headers[] = 'PRIVATE-TOKEN: '.$auth['username'];
-                }
-            } elseif ('bitbucket.org' === $originUrl
-                && $this->fileUrl !== Bitbucket::OAUTH2_ACCESS_TOKEN_URL && 'x-token-auth' === $auth['username']
-            ) {
-                if (!$this->isPublicBitBucketDownload($this->fileUrl)) {
-                    $headers[] = 'Authorization: Bearer ' . $auth['password'];
-                }
-            } else {
-                $authStr = base64_encode($auth['username'] . ':' . $auth['password']);
-                $headers[] = 'Authorization: Basic '.$authStr;
-            }
-        }
+        $headers = $this->authHelper->addAuthenticationHeader($headers, $originUrl, $this->fileUrl);
 
         $options['http']['follow_location'] = 0;
 
@@ -961,29 +811,4 @@ class RemoteFilesystem
 
         return parse_url($url, PHP_URL_HOST).':'.$port;
     }
-
-    /**
-     * @link https://github.com/composer/composer/issues/5584
-     *
-     * @param string $urlToBitBucketFile URL to a file at bitbucket.org.
-     *
-     * @return bool Whether the given URL is a public BitBucket download which requires no authentication.
-     */
-    private function isPublicBitBucketDownload($urlToBitBucketFile)
-    {
-        $domain = parse_url($urlToBitBucketFile, PHP_URL_HOST);
-        if (strpos($domain, 'bitbucket.org') === false) {
-            // Bitbucket downloads are hosted on amazonaws.
-            // We do not need to authenticate there at all
-            return true;
-        }
-
-        $path = parse_url($urlToBitBucketFile, PHP_URL_PATH);
-
-        // Path for a public download follows this pattern /{user}/{repo}/downloads/{whatever}
-        // {@link https://blog.bitbucket.org/2009/04/12/new-feature-downloads/}
-        $pathParts = explode('/', $path);
-
-        return count($pathParts) >= 4 && $pathParts[3] == 'downloads';
-    }
 }

src/Composer/Util/StreamContextFactory.php

@@ -41,6 +41,32 @@ final class StreamContextFactory
             'max_redirects' => 20,
         ));
 
+        $options = array_replace_recursive($options, self::initOptions($url, $defaultOptions));
+        unset($defaultOptions['http']['header']);
+        $options = array_replace_recursive($options, $defaultOptions);
+
+        if (isset($options['http']['header'])) {
+            $options['http']['header'] = self::fixHttpHeaderField($options['http']['header']);
+        }
+
+        return stream_context_create($options, $defaultParams);
+    }
+
+    /**
+     * @param string $url
+     * @param array $options
+     * @return array ['http' => ['header' => [...], 'proxy' => '..', 'request_fulluri' => bool]] formatted as a stream context array
+     */
+    public static function initOptions($url, array $options)
+    {
+        // Make sure the headers are in an array form
+        if (!isset($options['http']['header'])) {
+            $options['http']['header'] = array();
+        }
+        if (is_string($options['http']['header'])) {
+            $options['http']['header'] = explode("\r\n", $options['http']['header']);
+        }
+
         // Handle HTTP_PROXY/http_proxy on CLI only for security reasons
         if ((PHP_SAPI === 'cli' || PHP_SAPI === 'phpdbg') && (!empty($_SERVER['HTTP_PROXY']) || !empty($_SERVER['http_proxy']))) {
             $proxy = parse_url(!empty($_SERVER['http_proxy']) ? $_SERVER['http_proxy'] : $_SERVER['HTTP_PROXY']);
@@ -117,42 +143,36 @@ final class StreamContextFactory
                 }
                 $auth = base64_encode($auth);
 
-                // Preserve headers if already set in default options
-                if (isset($defaultOptions['http']['header'])) {
-                    if (is_string($defaultOptions['http']['header'])) {
-                        $defaultOptions['http']['header'] = array($defaultOptions['http']['header']);
-                    }
-                    $defaultOptions['http']['header'][] = "Proxy-Authorization: Basic {$auth}";
-                } else {
-                    $options['http']['header'] = array("Proxy-Authorization: Basic {$auth}");
-                }
+                $options['http']['header'][] = "Proxy-Authorization: Basic {$auth}";
             }
         }
 
-        $options = array_replace_recursive($options, $defaultOptions);
-
-        if (isset($options['http']['header'])) {
-            $options['http']['header'] = self::fixHttpHeaderField($options['http']['header']);
-        }
-
         if (defined('HHVM_VERSION')) {
             $phpVersion = 'HHVM ' . HHVM_VERSION;
         } else {
             $phpVersion = 'PHP ' . PHP_MAJOR_VERSION . '.' . PHP_MINOR_VERSION . '.' . PHP_RELEASE_VERSION;
         }
 
+        if (extension_loaded('curl')) {
+            $curl = curl_version();
+            $httpVersion = 'curl '.$curl['version'];
+        } else {
+            $httpVersion = 'streams';
+        }
+
         if (!isset($options['http']['header']) || false === stripos(implode('', $options['http']['header']), 'user-agent')) {
             $options['http']['header'][] = sprintf(
-                'User-Agent: Composer/%s (%s; %s; %s%s)',
-                Composer::VERSION === '@package_version@' ? 'source' : Composer::VERSION,
+                'User-Agent: Composer/%s (%s; %s; %s; %s%s)',
+                Composer::VERSION === '@package_version@' ? Composer::SOURCE_VERSION : Composer::VERSION,
                 function_exists('php_uname') ? php_uname('s') : 'Unknown',
                 function_exists('php_uname') ? php_uname('r') : 'Unknown',
                 $phpVersion,
+                $httpVersion,
                 getenv('CI') ? '; CI' : ''
             );
         }
 
-        return stream_context_create($options, $defaultParams);
+        return $options;
     }
 
     /**

tests/Composer/Test/Util/RemoteFilesystemTest.php

@@ -17,6 +17,20 @@ use PHPUnit\Framework\TestCase;
 
 class RemoteFilesystemTest extends TestCase
 {
+    private function getConfigMock()
+    {
+        $config = $this->getMockBuilder('Composer\Config')->getMock();
+        $config->expects($this->any())
+            ->method('get')
+            ->will($this->returnCallback(function ($key) {
+                if ($key === 'github-domains' || $key === 'gitlab-domains') {
+                    return array();
+                }
+            }));
+
+        return $config;
+    }
+
     public function testGetOptionsForUrl()
     {
         $io = $this->getMockBuilder('Composer\IO\IOInterface')->getMock();
@@ -101,7 +115,7 @@ class RemoteFilesystemTest extends TestCase
 
     public function testCallbackGetFileSize()
     {
-        $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock());
+        $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock(), $this->getConfigMock());
         $this->callCallbackGet($fs, STREAM_NOTIFY_FILE_SIZE_IS, 0, '', 0, 0, 20);
         $this->assertAttributeEquals(20, 'bytesMax', $fs);
     }
@@ -114,7 +128,7 @@ class RemoteFilesystemTest extends TestCase
             ->method('overwriteError')
         ;
 
-        $fs = new RemoteFilesystem($io);
+        $fs = new RemoteFilesystem($io, $this->getConfigMock());
         $this->setAttribute($fs, 'bytesMax', 20);
         $this->setAttribute($fs, 'progress', true);
 
@@ -124,7 +138,7 @@ class RemoteFilesystemTest extends TestCase
 
     public function testCallbackGetPassesThrough404()
     {
-        $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock());
+        $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock(), $this->getConfigMock());
 
         $this->assertNull($this->callCallbackGet($fs, STREAM_NOTIFY_FAILURE, 0, 'HTTP/1.1 404 Not Found', 404, 0, 0));
     }
@@ -139,7 +153,7 @@ class RemoteFilesystemTest extends TestCase
             ->method('setAuthentication')
             ->with($this->equalTo('github.com'), $this->equalTo('user'), $this->equalTo('pass'));
 
-        $fs = new RemoteFilesystem($io);
+        $fs = new RemoteFilesystem($io, $this->getConfigMock());
         try {
             $fs->getContents('github.com', 'https://user:pass@github.com/composer/composer/404');
         } catch (\Exception $e) {
@@ -150,14 +164,14 @@ class RemoteFilesystemTest extends TestCase
 
     public function testGetContents()
     {
-        $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock());
+        $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock(), $this->getConfigMock());
 
         $this->assertContains('testGetContents', $fs->getContents('http://example.org', 'file://'.__FILE__));
     }
 
     public function testCopy()
     {
-        $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock());
+        $fs = new RemoteFilesystem($this->getMockBuilder('Composer\IO\IOInterface')->getMock(), $this->getConfigMock());
 
         $file = tempnam(sys_get_temp_dir(), 'c');
         $this->assertTrue($fs->copy('http://example.org', 'file://'.__FILE__, $file));
@@ -218,7 +232,7 @@ class RemoteFilesystemTest extends TestCase
             ->disableOriginalConstructor()
             ->getMock();
 
-        $rfs = new RemoteFilesystem($io);
+        $rfs = new RemoteFilesystem($io, $this->getConfigMock());
         $hostname = parse_url($url, PHP_URL_HOST);
 
         $result = $rfs->getContents($hostname, $url, false);
@@ -240,14 +254,6 @@ class RemoteFilesystemTest extends TestCase
             ->disableOriginalConstructor()
             ->getMock();
 
-        $config = $this
-            ->getMockBuilder('Composer\Config')
-            ->getMock();
-        $config
-            ->method('get')
-            ->withAnyParameters()
-            ->willReturn(array());
-
         $domains = array();
         $io
             ->expects($this->any())
@@ -267,7 +273,7 @@ class RemoteFilesystemTest extends TestCase
                 'password' => '1A0yeK5Po3ZEeiiRiMWLivS0jirLdoGuaSGq9NvESFx1Fsdn493wUDXC8rz_1iKVRTl1GINHEUCsDxGh5lZ=',
             ));
 
-        $rfs = new RemoteFilesystem($io, $config);
+        $rfs = new RemoteFilesystem($io, $this->getConfigMock());
         $hostname = parse_url($url, PHP_URL_HOST);
 
         $result = $rfs->getContents($hostname, $url, false);
@@ -278,7 +284,7 @@ class RemoteFilesystemTest extends TestCase
 
     protected function callGetOptionsForUrl($io, array $args = array(), array $options = array(), $fileUrl = '')
     {
-        $fs = new RemoteFilesystem($io, null, $options);
+        $fs = new RemoteFilesystem($io, $this->getConfigMock(), $options);
         $ref = new \ReflectionMethod($fs, 'getOptionsForUrl');
         $prop = new \ReflectionProperty($fs, 'fileUrl');
         $ref->setAccessible(true);