diff --git a/bin/update-spdx-licenses b/bin/update-spdx-licenses old mode 100644 new mode 100755 index e509ded93..f6b6f9209 --- a/bin/update-spdx-licenses +++ b/bin/update-spdx-licenses @@ -5,5 +5,6 @@ require __DIR__ . '/../src/bootstrap.php'; use Composer\Util\SpdxLicensesUpdater; -$licenses = new SpdxLicensesUpdater; -$licenses->update(); +$updater = new SpdxLicensesUpdater; +$updater->dumpLicenses(__DIR__ . '/../res/spdx-licenses.json'); +$updater->dumpExceptions(__DIR__ . '/../res/spdx-exceptions.json'); diff --git a/res/spdx-exceptions.json b/res/spdx-exceptions.json new file mode 100644 index 000000000..15491d2a5 --- /dev/null +++ b/res/spdx-exceptions.json @@ -0,0 +1,29 @@ +{ + "Autoconf-exception-2.0": [ + "Autoconf exception 2.0" + ], + "Autoconf-exception-3.0": [ + "Autoconf exception 3.0" + ], + "Bison-exception-2.2": [ + "Bison exception 2.2" + ], + "Classpath-exception-2.0": [ + "Classpath exception 2.0" + ], + "eCos-exception-2.0": [ + "eCos exception 2.0" + ], + "Font-exception-2.0": [ + "Font exception 2.0" + ], + "GCC-exception-2.0": [ + "GCC Runtime Library exception 2.0" + ], + "GCC-exception-3.1": [ + "GCC Runtime Library exception 3.1" + ], + "WxWindows-exception-3.1": [ + "WxWindows Library Exception 3.1" + ] +} \ No newline at end of file diff --git a/res/spdx-licenses.json b/res/spdx-licenses.json index 3e1d93c35..7cb654966 100644 --- a/res/spdx-licenses.json +++ b/res/spdx-licenses.json @@ -431,10 +431,6 @@ "Eclipse Public License 1.0", true ], - "eCos-2.0": [ - "eCos license version 2.0", - false - ], "ECL-1.0": [ "Educational Community License v1.0", true @@ -499,6 +495,10 @@ "Frameworx Open License 1.0", true ], + "FreeImage": [ + "FreeImage Public License v1.0", + false + ], "FTL": [ "Freetype Project License", false @@ -543,78 +543,26 @@ "GNU General Public License v1.0 only", false ], - "GPL-1.0+": [ - "GNU General Public License v1.0 or later", - false - ], "GPL-2.0": [ "GNU General Public License v2.0 only", true ], - "GPL-2.0+": [ - "GNU General Public License v2.0 or later", - true - ], - "GPL-2.0-with-autoconf-exception": [ - "GNU General Public License v2.0 w/Autoconf exception", - true - ], - "GPL-2.0-with-bison-exception": [ - "GNU General Public License v2.0 w/Bison exception", - true - ], - "GPL-2.0-with-classpath-exception": [ - "GNU General Public License v2.0 w/Classpath exception", - true - ], - "GPL-2.0-with-font-exception": [ - "GNU General Public License v2.0 w/Font exception", - true - ], - "GPL-2.0-with-GCC-exception": [ - "GNU General Public License v2.0 w/GCC Runtime Library exception", - true - ], "GPL-3.0": [ "GNU General Public License v3.0 only", true ], - "GPL-3.0+": [ - "GNU General Public License v3.0 or later", - true - ], - "GPL-3.0-with-autoconf-exception": [ - "GNU General Public License v3.0 w/Autoconf exception", - true - ], - "GPL-3.0-with-GCC-exception": [ - "GNU General Public License v3.0 w/GCC Runtime Library exception", - true - ], "LGPL-2.1": [ "GNU Lesser General Public License v2.1 only", true ], - "LGPL-2.1+": [ - "GNU Lesser General Public License v2.1 or later", - true - ], "LGPL-3.0": [ "GNU Lesser General Public License v3.0 only", true ], - "LGPL-3.0+": [ - "GNU Lesser General Public License v3.0 or later", - true - ], "LGPL-2.0": [ "GNU Library General Public License v2 only", true ], - "LGPL-2.0+": [ - "GNU Library General Public License v2 or later", - true - ], "gnuplot": [ "gnuplot License", false @@ -927,6 +875,10 @@ "Open LDAP Public License v2.7", false ], + "OLDAP-2.8": [ + "Open LDAP Public License v2.8", + false + ], "OML": [ "Open Market License", false @@ -955,10 +907,6 @@ "Open Software License 3.0", true ], - "OLDAP-2.8": [ - "OpenLDAP Public License v2.8", - false - ], "OpenSSL": [ "OpenSSL License", false @@ -1079,10 +1027,6 @@ "Standard ML of New Jersey License", false ], - "StandardML-NJ": [ - "Standard ML of New Jersey License", - false - ], "SugarCRM-1.1.3": [ "SugarCRM Public License v1.1.3", false @@ -1144,17 +1088,17 @@ true ], "W3C": [ - "W3C Software Notice and License", + "W3C Software Notice and License (2002-12-31)", true ], + "W3C-19980720": [ + "W3C Software Notice and License (1998-07-20)", + false + ], "Wsuipa": [ "Wsuipa License", false ], - "WXwindows": [ - "wxWindows Library License", - true - ], "Xnet": [ "X.Net License", true @@ -1203,6 +1147,10 @@ "Zimbra Public License v1.3", false ], + "Zimbra-1.4": [ + "Zimbra Public License v1.4", + false + ], "Zlib": [ "zlib License", true @@ -1222,5 +1170,9 @@ "ZPL-2.1": [ "Zope Public License 2.1", false + ], + "ICU": [ + "ICU License", + false ] } \ No newline at end of file diff --git a/src/Composer/Util/SpdxLicense.php b/src/Composer/Util/SpdxLicense.php index 650b918e4..b9ffdc70d 100644 --- a/src/Composer/Util/SpdxLicense.php +++ b/src/Composer/Util/SpdxLicense.php @@ -12,8 +12,6 @@ namespace Composer\Util; -use Composer\Json\JsonFile; - /** * Supports composer array and SPDX tag notation for disjunctive/conjunctive * licenses. @@ -22,53 +20,60 @@ use Composer\Json\JsonFile; */ class SpdxLicense { - /** - * @var array - */ + /** @var array */ private $licenses; + /** @var array */ + private $exceptions; + public function __construct() { $this->loadLicenses(); + $this->loadExceptions(); } - private function loadLicenses() + /** + * Returns license metadata by license identifier. + * + * @param string $identifier + * + * @return array|null + */ + public function getLicenseByIdentifier($identifier) { - if (is_array($this->licenses)) { - return $this->licenses; + if (!isset($this->licenses[$identifier])) { + return; } - $jsonFile = new JsonFile(__DIR__ . '/../../../res/spdx-licenses.json'); - $this->licenses = $jsonFile->read(); + $license = $this->licenses[$identifier]; + $license[] = 'http://spdx.org/licenses/' . $identifier . '.html#licenseText'; - return $this->licenses; + return $license; } /** - * Returns license metadata by license identifier. + * Returns license exception metadata by license exception identifier. * * @param string $identifier * * @return array|null */ - public function getLicenseByIdentifier($identifier) + public function getExceptionByIdentifier($identifier) { - if (!isset($this->licenses[$identifier])) { + if (!isset($this->exceptions[$identifier])) { return; } - $license = $this->licenses[$identifier]; - - // add URL for the license text (it's not included in the json) - $license[2] = 'http://spdx.org/licenses/' . $identifier . '#licenseText'; + $license = $this->exceptions[$identifier]; + $license[] = 'http://spdx.org/licenses/' . $identifier . '.html#licenseExceptionText'; return $license; } /** - * Returns the short identifier of a license by full name. + * Returns the short identifier of a license (exception) by full name. * - * @param string $identifier + * @param string $name * * @return string */ @@ -79,11 +84,19 @@ class SpdxLicense return $identifier; } } + + foreach ($this->exceptions as $identifier => $licenseData) { + if ($licenseData[0] === $name) { // key 0 = fullname + return $identifier; + } + } } /** * Returns the OSI Approved status for a license by identifier. * + * @param string $identifier + * * @return bool */ public function isOsiApprovedByIdentifier($identifier) @@ -105,6 +118,20 @@ class SpdxLicense return in_array($identifier, $identifiers); } + /** + * Check, if the identifier for a exception is valid. + * + * @param string $identifier + * + * @return bool + */ + private function isValidExceptionIdentifier($identifier) + { + $identifiers = array_keys($this->exceptions); + + return in_array($identifier, $identifiers); + } + /** * @param array|string $license * @@ -118,18 +145,49 @@ class SpdxLicense if ($count !== count(array_filter($license, 'is_string'))) { throw new \InvalidArgumentException('Array of strings expected.'); } - $license = $count > 1 ? '('.implode(' or ', $license).')' : (string) reset($license); + $license = $count > 1 ? '('.implode(' OR ', $license).')' : (string) reset($license); } if (!is_string($license)) { throw new \InvalidArgumentException(sprintf( - 'Array or String expected, %s given.', gettype($license) + 'Array or String expected, %s given.', + gettype($license) )); } return $this->isValidLicenseString($license); } + /** + * @return array + */ + private function loadLicenses() + { + if (is_array($this->licenses)) { + return $this->licenses; + } + + $jsonFile = file_get_contents(__DIR__ . '/../../../res/spdx-licenses.json'); + $this->licenses = json_decode($jsonFile, true); + + return $this->licenses; + } + + /** + * @return array + */ + private function loadExceptions() + { + if (is_array($this->exceptions)) { + return $this->exceptions; + } + + $jsonFile = file_get_contents(__DIR__ . '/../../../res/spdx-exceptions.json'); + $this->exceptions = json_decode($jsonFile, true); + + return $this->exceptions; + } + /** * @param string $license * @@ -141,10 +199,11 @@ class SpdxLicense $tokens = array( 'po' => '\(', 'pc' => '\)', - 'op' => '(?:or|and)', + 'op' => '(?:or|OR|and|AND)', + 'wi' => '(?:with|WITH)', 'lix' => '(?:NONE|NOASSERTION)', 'lir' => 'LicenseRef-\d+', - 'lic' => '[-+_.a-zA-Z0-9]{3,}', + 'lic' => '[-_.a-zA-Z0-9]{3,}\+?', 'ws' => '\s+', '_' => '.', ); @@ -171,44 +230,58 @@ class SpdxLicense return array($name, $matches[0][0]); } - throw new \RuntimeException('At least the last pattern needs to match, but it did not (dot-match-all is missing?).'); + throw new \RuntimeException( + 'At least the last pattern needs to match, but it did not (dot-match-all is missing?).' + ); }; $open = 0; - $require = 1; + $with = false; + $require = true; $lastop = null; while (list($token, $string) = $next()) { switch ($token) { case 'po': - if ($open || !$require) { + if ($open || !$require || $with) { return false; } $open = 1; break; case 'pc': - if ($open !== 1 || $require || !$lastop) { + if ($open !== 1 || $require || !$lastop || $with) { return false; } $open = 2; break; case 'op': - if ($require || !$open) { + if ($require || !$open || $with) { return false; } $lastop || $lastop = $string; if ($lastop !== $string) { return false; } - $require = 1; + $require = true; + break; + case 'wi': + $with = true; break; case 'lix': - if ($open) { + if ($open || $with) { return false; } goto lir; case 'lic': - if (!$this->isValidLicenseIdentifier($string)) { + if ($with && $this->isValidExceptionIdentifier($string)) { + $require = true; + $with = false; + goto lir; + } + if ($with) { + return false; + } + if (!$this->isValidLicenseIdentifier(rtrim($string, '+'))) { return false; } // Fall-through intended @@ -217,7 +290,7 @@ class SpdxLicense if (!$require) { return false; } - $require = 0; + $require = false; break; case 'ws': break; @@ -228,6 +301,6 @@ class SpdxLicense } } - return !($open % 2 || $require); + return !($open % 2 || $require || $with); } } diff --git a/src/Composer/Util/SpdxLicensesUpdater.php b/src/Composer/Util/SpdxLicensesUpdater.php index efcc30065..457f5b389 100644 --- a/src/Composer/Util/SpdxLicensesUpdater.php +++ b/src/Composer/Util/SpdxLicensesUpdater.php @@ -12,8 +12,6 @@ namespace Composer\Util; -use Composer\Json\JsonFormatter; - /** * The SPDX Licenses Updater scrapes licenses from the spdx website * and updates the "res/spdx-licenses.json" file accordingly. @@ -22,21 +20,57 @@ use Composer\Json\JsonFormatter; */ class SpdxLicensesUpdater { - private $licensesUrl = 'http://www.spdx.org/licenses/'; + /** + * @param string $file + * @param string $url + */ + public function dumpLicenses($file, $url = 'http://www.spdx.org/licenses/') + { + $options = 0; - public function update() + if (defined('JSON_PRETTY_PRINT')) { + $options |= JSON_PRETTY_PRINT; + } + + if (defined('JSON_UNESCAPED_SLASHES')) { + $options |= JSON_UNESCAPED_SLASHES; + } + + $licenses = json_encode($this->getLicenses($url), $options); + file_put_contents($file, $licenses); + } + + /** + * @param string $file + * @param string $url + */ + public function dumpExceptions($file, $url = 'http://www.spdx.org/licenses/exceptions-index.html') { - $json = json_encode($this->getLicenses(), true); - $prettyJson = JsonFormatter::format($json, true, true); - file_put_contents(__DIR__ . '/../../../res/spdx-licenses.json', $prettyJson); + $options = 0; + + if (defined('JSON_PRETTY_PRINT')) { + $options |= JSON_PRETTY_PRINT; + } + + if (defined('JSON_UNESCAPED_SLASHES')) { + $options |= JSON_UNESCAPED_SLASHES; + } + + $exceptions = json_encode($this->getExceptions($url), $options); + file_put_contents($file, $exceptions); } - private function getLicenses() + /** + * @param string $url + * + * @return array + */ + private function getLicenses($url) { $licenses = array(); $dom = new \DOMDocument; - $dom->loadHTMLFile($this->licensesUrl); + @$dom->loadHTMLFile($url); $xPath = new \DOMXPath($dom); $trs = $xPath->query('//table//tbody//tr'); @@ -45,8 +79,8 @@ class SpdxLicensesUpdater foreach ($trs as $tr) { $tds = $tr->getElementsByTagName('td'); // get the columns in this row - if ($tds->length < 4) { - throw new \Exception('Obtaining the license table failed. Wrong table format. Found less than 4 cells in a row.'); + if ($tds->length !== 4) { + continue; } if (trim($tds->item(3)->nodeValue) == 'License Text') { @@ -56,7 +90,7 @@ class SpdxLicensesUpdater // The license URL is not scraped intentionally to keep json file size low. // It's build when requested, see SpdxLicense->getLicenseByIdentifier(). - //$licenseURL = = $tds->item(3)->getAttribute('href'); + //$licenseURL = $tds->item(3)->getAttribute('href'); $licenses += array($identifier => array($fullname, $osiApproved)); } @@ -64,4 +98,42 @@ class SpdxLicensesUpdater return $licenses; } + + /** + * @param string $url + * + * @return array + */ + private function getExceptions($url) + { + $exceptions = array(); + + $dom = new \DOMDocument; + @$dom->loadHTMLFile($url); + + $xPath = new \DOMXPath($dom); + $trs = $xPath->query('//table//tbody//tr'); + + // iterate over each row in the table + foreach ($trs as $tr) { + $tds = $tr->getElementsByTagName('td'); // get the columns in this row + + if ($tds->length !== 3) { + continue; + } + + if (trim($tds->item(2)->nodeValue) == 'License Exception Text') { + $fullname = trim($tds->item(0)->nodeValue); + $identifier = trim($tds->item(1)->nodeValue); + + // The license URL is not scraped intentionally to keep json file size low. + // It's build when requested, see SpdxLicense->getLicenseExceptionByIdentifier(). + //$licenseURL = $tds->item(2)->getAttribute('href'); + + $exceptions += array($identifier => array($fullname)); + } + } + + return $exceptions; + } } diff --git a/tests/Composer/Test/Util/SpdxLicenseTest.php b/tests/Composer/Test/Util/SpdxLicenseTest.php index d4665954e..ef6e7d45d 100644 --- a/tests/Composer/Test/Util/SpdxLicenseTest.php +++ b/tests/Composer/Test/Util/SpdxLicenseTest.php @@ -27,12 +27,18 @@ class SpdxLicenseTest extends TestCase $valid = array_merge( array( "MIT", + "MIT+", "NONE", "NOASSERTION", "LicenseRef-3", array("LGPL-2.0", "GPL-3.0+"), "(LGPL-2.0 or GPL-3.0+)", + "(LGPL-2.0 OR GPL-3.0+)", "(EUDatagrid and GPL-3.0+)", + "(EUDatagrid AND GPL-3.0+)", + "GPL-2.0 with Autoconf-exception-2.0", + "GPL-2.0 WITH Autoconf-exception-2.0", + "GPL-2.0+ WITH Autoconf-exception-2.0", ), $identifiers ); @@ -52,7 +58,10 @@ class SpdxLicenseTest extends TestCase array("The system pwns you"), array("()"), array("(MIT)"), + array("(MIT"), + array("MIT)"), array("MIT NONE"), + array("MIT AND NONE"), array("MIT (MIT and MIT)"), array("(MIT and MIT) MIT"), array(array("LGPL-2.0", "The system pwns you")), @@ -64,6 +73,10 @@ class SpdxLicenseTest extends TestCase array("(MIT Or MIT)"), array("(NONE or MIT)"), array("(NOASSERTION or MIT)"), + array("Autoconf-exception-2.0 WITH MIT"), + array("MIT WITH"), + array("MIT OR"), + array("MIT AND"), ); }