Respect gitignore, gitattributes and hgignore files in archiving

main
Nils Adermann 11 years ago
parent 64941b0a64
commit deae50392f

@ -0,0 +1,78 @@
<?php
/*
* This file is part of Composer.
*
* (c) Nils Adermann <naderman@naderman.de>
* Jordi Boggiano <j.boggiano@seld.be>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Composer\Package\Archiver;
use Composer\Package\BasePackage;
use Composer\Package\PackageInterface;
use Symfony\Component\Finder;
/**
* A Symfony Finder wrapper which locates files that should go into archives
*
* Handles .gitignore, .gitattributes and .hgignore files as well as composer's
* own exclude rules from composer.json
*
* @author Nils Adermann <naderman@naderman.de>
*/
class ArchivableFilesFinder
{
/**
* @var Symfony\Component\Finder\Finder
*/
protected $finder;
/**
* Initializes the internal Symfony Finder with appropriate filters
*
* @param string $sources Path to source files to be archived
* @param array $excludes Composer's own exclude rules from composer.json
*/
public function __construct($sources, array $excludes)
{
$sources = realpath($sources);
$filters = array(
new HgExcludeFilter($sources),
new GitExcludeFilter($sources),
new ComposerExcludeFilter($sources, $excludes),
);
$this->finder = new Finder\Finder();
$this->finder
->in($sources)
->filter(function (\SplFileInfo $file) use ($sources, $filters) {
$relativePath = preg_replace(
'#^'.preg_quote($sources, '#').'#',
'',
str_replace(PATH_SEPARATOR, '/', $file->getRealPath())
);
$exclude = false;
foreach ($filters as $filter) {
$exclude = $filter->filter($relativePath, $exclude);
}
return !$exclude;
})
->ignoreVCS(true)
->ignoreDotFiles(false);
}
/**
* @return Symfony\Component\Finder\Finder
*/
public function getIterator()
{
return $this->finder->getIterator();
}
}

@ -70,7 +70,7 @@ class ArchiveManager
*
* @return string A filename without an extension
*/
protected function getPackageFilename(PackageInterface $package)
public function getPackageFilename(PackageInterface $package)
{
$nameParts = array(preg_replace('#[^a-z0-9-_.]#i', '-', $package->getName()));

@ -17,6 +17,7 @@ use Composer\Package\PackageInterface;
/**
* @author Till Klampaeckel <till@php.net>
* @author Matthieu Moquet <matthieu@moquet.net>
* @author Nils Adermann <naderman@naderman.de>
*/
interface ArchiverInterface
{
@ -30,7 +31,7 @@ interface ArchiverInterface
*
* @return string The path to the written archive file
*/
public function archive($sources, $target, $format, $excludes = array());
public function archive($sources, $target, $format, array $excludes = array());
/**
* Format supported by the archiver.

@ -0,0 +1,31 @@
<?php
/*
* This file is part of Composer.
*
* (c) Nils Adermann <naderman@naderman.de>
* Jordi Boggiano <j.boggiano@seld.be>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Composer\Package\Archiver;
/**
* An exclude filter which processes composer's own exclude rules
*
* @author Nils Adermann <naderman@naderman.de>
*/
class ComposerExcludeFilter extends ExcludeFilterBase
{
/**
* @param string $sourcePath Directory containing sources to be filtered
* @param array $excludeRules An array of exclude rules from composer.json
*/
public function __construct($sourcePath, array $excludeRules)
{
parent::__construct($sourcePath);
$this->excludePatterns = $this->generatePatterns($excludeRules);
}
}

@ -0,0 +1,141 @@
<?php
/*
* This file is part of Composer.
*
* (c) Nils Adermann <naderman@naderman.de>
* Jordi Boggiano <j.boggiano@seld.be>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Composer\Package\Archiver;
use Symfony\Component\Finder;
/**
* @author Nils Adermann <naderman@naderman.de>
*/
abstract class ExcludeFilterBase
{
/**
* @var string
*/
protected $sourcePath;
/**
* @var array
*/
protected $excludePatterns;
/**
* @param string $sourcePath Directory containing sources to be filtered
*/
public function __construct($sourcePath)
{
$this->sourcePath = $sourcePath;
$this->excludePatterns = array();
}
/**
* Checks the given path against all exclude patterns in this filter
*
* Negated patterns overwrite exclude decisions of previous filters.
*
* @param string $relativePath The file's path relative to the sourcePath
* @param bool $exclude Whether a previous filter wants to exclude this file
*
* @return bool Whether the file should be excluded
*/
public function filter($relativePath, $exclude)
{
foreach ($this->excludePatterns as $patternData) {
list($pattern, $negate, $stripLeadingSlash) = $patternData;
if ($stripLeadingSlash) {
$path = substr($relativePath, 1);
} else {
$path = $relativePath;
}
if (preg_match($pattern, $path)) {
$exclude = !$negate;
}
}
return $exclude;
}
/**
* Processes a file containing exclude rules of different formats per line
*
* @param array $lines A set of lines to be parsed
* @param callback $lineParser The parser to be used on each line
*
* @return array Exclude patterns to be used in filter()
*/
protected function parseLines(array $lines, $lineParser)
{
return array_filter(
array_map(
function ($line) use ($lineParser) {
$line = trim($line);
$commentHash = strpos($line, '#');
if ($commentHash !== false) {
$line = substr($line, 0, $commentHash);
}
if ($line) {
return call_user_func($lineParser, $line);
}
return null;
}, $lines),
function ($pattern) {
return $pattern !== null;
}
);
}
/**
* Generates a set of exclude patterns for filter() from gitignore rules
*
* @param array $rules A list of exclude rules in gitignore syntax
*
* @return array Exclude patterns
*/
protected function generatePatterns($rules)
{
$patterns = array();
foreach ($rules as $rule) {
$patterns[] = $this->generatePattern($rule);
}
return $patterns;
}
/**
* Generates an exclude pattern for filter() from a gitignore rule
*
* @param string An exclude rule in gitignore syntax
*
* @param array An exclude pattern
*/
protected function generatePattern($rule)
{
$negate = false;
$pattern = '#';
if (strlen($rule) && $rule[0] === '!') {
$negate = true;
$rule = substr($rule, 1);
}
if (strlen($rule) && $rule[0] === '/') {
$pattern .= '^/';
$rule = substr($rule, 1);
}
$pattern .= substr(Finder\Glob::toRegex($rule), 2, -2);
return array($pattern . '#', $negate, false);
}
}

@ -0,0 +1,80 @@
<?php
/*
* This file is part of Composer.
*
* (c) Nils Adermann <naderman@naderman.de>
* Jordi Boggiano <j.boggiano@seld.be>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Composer\Package\Archiver;
/**
* An exclude filter that processes gitignore and gitattributes
*
* It respects export-ignore git attributes
*
* @author Nils Adermann <naderman@naderman.de>
*/
class GitExcludeFilter extends ExcludeFilterBase
{
/**
* Parses .gitignore and .gitattributes files if they exist
*
* @param string $sourcePath
*/
public function __construct($sourcePath)
{
parent::__construct($sourcePath);
if (file_exists($sourcePath.'/.gitignore')) {
$this->excludePatterns = $this->parseLines(
file($sourcePath.'/.gitignore'),
array($this, 'parseGitIgnoreLine')
);
}
if (file_exists($sourcePath.'/.gitattributes')) {
$this->excludePatterns = array_merge(
$this->excludePatterns,
$this->parseLines(
file($sourcePath.'/.gitattributes'),
array($this, 'parseGitAttributesLine')
));
}
}
/**
* Callback line parser which process gitignore lines
*
* @param string $line A line from .gitignore
*
* @return array An exclude pattern for filter()
*/
protected function parseGitIgnoreLine($line)
{
return $this->generatePattern($line);
}
/**
* Callback parser which finds export-ignore rules in git attribute lines
*
* @param string $line A line from .gitattributes
*
* @return array An exclude pattern for filter()
*/
protected function parseGitAttributesLine($line)
{
$parts = preg_split('#\s+#', $line);
if (count($parts) != 2) {
return null;
}
if ($parts[1] === 'export-ignore') {
return $this->generatePattern($parts[0]);
}
}
}

@ -0,0 +1,104 @@
<?php
/*
* This file is part of Composer.
*
* (c) Nils Adermann <naderman@naderman.de>
* Jordi Boggiano <j.boggiano@seld.be>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Composer\Package\Archiver;
use Symfony\Component\Finder;
/**
* An exclude filter that processes hgignore files
*
* @author Nils Adermann <naderman@naderman.de>
*/
class HgExcludeFilter extends ExcludeFilterBase
{
const HG_IGNORE_REGEX = 1;
const HG_IGNORE_GLOB = 2;
/**
* Either HG_IGNORE_REGEX or HG_IGNORE_GLOB
* @var integer
*/
protected $patternMode;
/**
* Parses .hgignore file if it exist
*
* @param string $sourcePath
*/
public function __construct($sourcePath)
{
parent::__construct($sourcePath);
$this->patternMode = self::HG_IGNORE_REGEX;
if (file_exists($sourcePath.'/.hgignore')) {
$this->excludePatterns = $this->parseLines(
file($sourcePath.'/.hgignore'),
array($this, 'parseHgIgnoreLine')
);
}
}
/**
* Callback line parser which process hgignore lines
*
* @param string $line A line from .hgignore
*
* @return array An exclude pattern for filter()
*/
public function parseHgIgnoreLine($line)
{
if (preg_match('#^syntax\s*:\s*(glob|regexp)$#', $line, $matches)) {
if ($matches[1] === 'glob') {
$this->patternMode = self::HG_IGNORE_GLOB;
} else {
$this->patternMode = self::HG_IGNORE_REGEX;
}
return null;
}
if ($this->patternMode == self::HG_IGNORE_GLOB) {
return $this->patternFromGlob($line);
} else {
return $this->patternFromRegex($line);
}
}
/**
* Generates an exclude pattern for filter() from a hg glob expression
*
* @param string $line A line from .hgignore in glob mode
*
* @return array An exclude pattern for filter()
*/
protected function patternFromGlob($line)
{
$pattern = '#'.substr(Finder\Glob::toRegex($line), 2, -1).'#';
$pattern = str_replace('[^/]*', '.*', $pattern);
return array($pattern, false, true);
}
/**
* Generates an exclude pattern for filter() from a hg regexp expression
*
* @param string $line A line from .hgignore in regexp mode
*
* @return array An exclude pattern for filter()
*/
public function patternFromRegex($line)
{
// WTF need to escape the delimiter safely
$pattern = '#'.preg_replace('/((?:\\\\\\\\)*)(\\\\?)#/', '\1\2\2\\#', $line).'#';
return array($pattern, false, true);
}
}

@ -15,8 +15,6 @@ namespace Composer\Package\Archiver;
use Composer\Package\BasePackage;
use Composer\Package\PackageInterface;
use Symfony\Component\Finder;
/**
* @author Till Klampaeckel <till@php.net>
* @author Nils Adermann <naderman@naderman.de>
@ -32,31 +30,14 @@ class PharArchiver implements ArchiverInterface
/**
* {@inheritdoc}
*/
public function archive($sources, $target, $format, $excludes = array())
public function archive($sources, $target, $format, array $excludes = array())
{
$sources = realpath($sources);
$excludePatterns = $this->generatePatterns($excludes);
try {
$phar = new \PharData($target, null, null, static::$formats[$format]);
$finder = new Finder\Finder();
$finder
->in($sources)
->filter(function (\SplFileInfo $file) use ($sources, $excludePatterns) {
$relativePath = preg_replace('#^'.preg_quote($sources, '#').'#', '', $file->getRealPath());
$include = true;
foreach ($excludePatterns as $patternData) {
list($pattern, $negate) = $patternData;
if (preg_match($pattern, $relativePath)) {
$include = $negate;
}
}
return $include;
})
->ignoreVCS(true);
$phar->buildFromIterator($finder->getIterator(), $sources);
$files = new ArchivableFilesFinder($sources, $excludes);
$phar->buildFromIterator($files->getIterator(), $sources);
return $target;
} catch (\UnexpectedValueException $e) {
$message = sprintf("Could not create archive '%s' from '%s': %s",
@ -69,35 +50,6 @@ class PharArchiver implements ArchiverInterface
}
}
/**
* Generates a set of PCRE patterns from a set of exclude rules.
*
* @param array $rules A list of exclude rules similar to gitignore syntax
*/
protected function generatePatterns($rules)
{
$patterns = array();
foreach ($rules as $rule) {
$negate = false;
$pattern = '#';
if (strlen($rule) && $rule[0] === '!') {
$negate = true;
$rule = substr($rule, 1);
}
if (strlen($rule) && $rule[0] === '/') {
$pattern .= '^/';
$rule = substr($rule, 1);
}
$pattern .= substr(Finder\Glob::toRegex($rule), 2, -2);
$patterns[] = array($pattern . '#', $negate);
}
return $patterns;
}
/**
* {@inheritdoc}
*/

@ -0,0 +1,206 @@
<?php
/*
* This file is part of Composer.
*
* (c) Nils Adermann <naderman@naderman.de>
* Jordi Boggiano <j.boggiano@seld.be>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Composer\Test\Package\Archiver;
use Composer\Package\Archiver\ArchivableFilesFinder;
use Composer\Util\Filesystem;
use Symfony\Component\Process\Process;
/**
* @author Nils Adermann <naderman@naderman.de>
*/
class ArchivableFilesFinderTest extends \PHPUnit_Framework_TestCase
{
protected $sources;
protected $finder;
protected function setup()
{
$fs = new Filesystem;
$this->sources = sys_get_temp_dir().
'/composer_archiver_test'.uniqid(mt_rand(), true);
$fileTree = array(
'A/prefixA.foo',
'A/prefixB.foo',
'A/prefixC.foo',
'A/prefixD.foo',
'A/prefixE.foo',
'A/prefixF.foo',
'B/sub/prefixA.foo',
'B/sub/prefixB.foo',
'B/sub/prefixC.foo',
'B/sub/prefixD.foo',
'B/sub/prefixE.foo',
'B/sub/prefixF.foo',
'toplevelA.foo',
'toplevelB.foo',
'prefixA.foo',
'prefixB.foo',
'prefixC.foo',
'prefixD.foo',
'prefixE.foo',
'prefixF.foo',
);
foreach ($fileTree as $relativePath) {
$path = $this->sources.'/'.$relativePath;
$fs->ensureDirectoryExists(dirname($path));
file_put_contents($path, '');
}
}
protected function tearDown()
{
$fs = new Filesystem;
$fs->removeDirectory($this->sources);
}
public function testManualExcludes()
{
$excludes = array(
'prefixB.foo',
'!/prefixB.foo',
'/prefixA.foo',
'prefixC.*',
'!*/*/*/prefixC.foo'
);
$this->finder = new ArchivableFilesFinder($this->sources, $excludes);
$this->assertArchivableFiles(array(
'/A/prefixA.foo',
'/A/prefixD.foo',
'/A/prefixE.foo',
'/A/prefixF.foo',
'/B/sub/prefixA.foo',
'/B/sub/prefixC.foo',
'/B/sub/prefixD.foo',
'/B/sub/prefixE.foo',
'/B/sub/prefixF.foo',
'/prefixB.foo',
'/prefixD.foo',
'/prefixE.foo',
'/prefixF.foo',
'/toplevelA.foo',
'/toplevelB.foo',
));
}
public function testGitExcludes()
{
file_put_contents($this->sources.'/.gitignore', implode("\n", array(
'# gitignore rules with comments and blank lines',
'',
'prefixE.foo',
'# and more',
'# comments',
'',
'!/prefixE.foo',
'/prefixD.foo',
'prefixF.*',
'!/*/*/prefixF.foo',
'',
)));
// git does not currently support negative git attributes
file_put_contents($this->sources.'/.gitattributes', implode("\n", array(
'',
'# gitattributes rules with comments and blank lines',
'prefixB.foo export-ignore',
//'!/prefixB.foo export-ignore',
'/prefixA.foo export-ignore',
'prefixC.* export-ignore',
//'!/*/*/prefixC.foo export-ignore'
)));
$this->finder = new ArchivableFilesFinder($this->sources, array());
$this->assertArchivableFiles($this->getArchivedFiles('git init && '.
'git add .git* && '.
'git commit -m "ignore rules" && '.
'git add . && '.
'git commit -m "init" && '.
'git archive --format=zip --prefix=archive/ -o archive.zip HEAD'
));
}
public function testHgExcludes()
{
file_put_contents($this->sources.'/.hgignore', implode("\n", array(
'# hgignore rules with comments, blank lines and syntax changes',
'',
'pre*A.foo',
'prefixE.foo',
'# and more',
'# comments',
'',
'^prefixD.foo',
'syntax: glob',
'prefixF.*',
'B/*',
)));
$this->finder = new ArchivableFilesFinder($this->sources, array());
$expectedFiles = $this->getArchivedFiles('hg init && '.
'hg add && '.
'hg commit -m "init" && '.
'hg archive archive.zip'
);
array_shift($expectedFiles); // remove .hg_archival.txt
$this->assertArchivableFiles($expectedFiles);
}
protected function getArchivableFiles()
{
$files = array();
foreach ($this->finder->getIterator() as $file) {
if (!$file->isDir()) {
$files[] = preg_replace('#^'.preg_quote($this->sources, '#').'#', '', $file->getRealPath());
}
}
sort($files);
return $files;
}
protected function getArchivedFiles($command)
{
$process = new Process($command, $this->sources);
$process->run();
$archive = new \PharData($this->sources.'/archive.zip');
$iterator = new \RecursiveIteratorIterator($archive);
$files = array();
foreach ($iterator as $file) {
$files[] = preg_replace('#^phar://'.preg_quote($this->sources, '#').'/archive\.zip/archive#', '', $file);
}
unlink($this->sources.'/archive.zip');
return $files;
}
protected function assertArchivableFiles($expectedFiles)
{
$actualFiles = $this->getArchivableFiles();
$this->assertEquals($expectedFiles, $actualFiles);
}
}

@ -51,8 +51,6 @@ class ArchiveManagerTest extends ArchiverTest
$package = $this->setupPackage();
// The package is source from git,
// so it should `git archive --format tar`
$this->manager->archive($package, 'tar', $this->targetDir);
$target = $this->getTargetName($package, 'tar');
@ -63,7 +61,7 @@ class ArchiveManagerTest extends ArchiverTest
protected function getTargetName(PackageInterface $package, $format)
{
$packageName = preg_replace('#[^a-z0-9-_.]#i', '-', $package->getPrettyString());
$packageName = $this->manager->getPackageFilename($package);
$target = $this->targetDir.'/'.$packageName.'.'.$format;
return $target;

@ -0,0 +1,42 @@
<?php
/*
* This file is part of Composer.
*
* (c) Nils Adermann <naderman@naderman.de>
* Jordi Boggiano <j.boggiano@seld.be>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Composer\Test\Package\Archiver;
use Composer\Package\Archiver\HgExcludeFilter;
/**
* @author Nils Adermann <naderman@naderman.de>
*/
class HgExcludeFilterTest extends \PHPUnit_Framework_TestCase
{
/**
* @dataProvider patterns
*/
public function testPatternEscape($ignore, $expected)
{
$filter = new HgExcludeFilter('/');
$this->assertEquals($expected, $filter->patternFromRegex($ignore));
}
public function patterns()
{
return array(
array('.#', array('#.\\##', false, true)),
array('.\\#', array('#.\\\\\\##', false, true)),
array('\\.#', array('#\\.\\##', false, true)),
array('\\\\.\\\\\\\\#', array('#\\\\.\\\\\\\\\\##', false, true)),
array('.\\\\\\\\\\#', array('#.\\\\\\\\\\\\\\##', false, true)),
);
}
}

@ -29,7 +29,7 @@ class PharArchiverTest extends ArchiverTest
// Test archive
$archiver = new PharArchiver();
$archiver->archive($package->getSourceUrl(), $target, 'tar', null, array('foo/bar', 'baz', '!/foo/bar/baz'));
$archiver->archive($package->getSourceUrl(), $target, 'tar', array('foo/bar', 'baz', '!/foo/bar/baz'));
$this->assertFileExists($target);
unlink($target);

Loading…
Cancel
Save