Source code
Revision control
Copy as Markdown
// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#include "scanner.h"
#include "logging.h"
#include <QtCore/qdir.h>
#include <QtCore/qhash.h>
#include <QtCore/qjsonarray.h>
#include <QtCore/qjsondocument.h>
#include <QtCore/qjsonobject.h>
#include <QtCore/qtextstream.h>
#include <QtCore/qvariant.h>
#include <iostream>
using namespace Qt::Literals::StringLiterals;
namespace Scanner {
static void missingPropertyWarning(const QString &filePath, const QString &property)
{
std::cerr << qPrintable(tr("File %1: Missing mandatory property '%2'.").arg(
QDir::toNativeSeparators(filePath), property)) << std::endl;
}
bool validatePackage(Package &p, Checks checks, LogLevel logLevel)
{
const auto &filePath = p.filePath;
bool validPackage = true;
if (filePath.isEmpty()) {
std::cerr << qPrintable(tr("The origin file of package '%1' was not recorded.")
.arg(p.id));
return false;
}
if (p.qtParts.isEmpty())
p.qtParts << u"libs"_s;
if (p.name.isEmpty()) {
if (p.id.startsWith("chromium-"_L1)) // Ignore invalid README.chromium files
return false;
if (logLevel != SilentLog)
missingPropertyWarning(filePath, u"Name"_s);
validPackage = false;
}
if (p.id.isEmpty()) {
if (logLevel != SilentLog)
missingPropertyWarning(filePath, u"Id"_s);
validPackage = false;
} else if (!p.id.isLower() || p.id.contains(' '_L1)) {
if (logLevel != SilentLog)
std::cerr << qPrintable(tr("File %1: Value of 'Id' must be in lowercase and without spaces.")
.arg(QDir::toNativeSeparators(filePath))) << std::endl;
validPackage = false;
}
if (p.license.isEmpty()) {
if (logLevel != SilentLog)
missingPropertyWarning(filePath, u"License"_s);
validPackage = false;
}
if (!p.copyright.isEmpty() && !p.copyrightFile.isEmpty()) {
if (logLevel != SilentLog) {
std::cerr << qPrintable(tr("File %1: Properties 'Copyright' and 'CopyrightFile' are "
"mutually exclusive.")
.arg(QDir::toNativeSeparators(filePath)))
<< std::endl;
}
validPackage = false;
}
if (p.securityCritical && p.downloadLocation.isEmpty()) {
if (logLevel != SilentLog)
missingPropertyWarning(filePath, u"DownloadLocation"_s);
validPackage = false;
}
for (const QString &part : std::as_const(p.qtParts)) {
if (part != "examples"_L1 && part != "tests"_L1
&& part != "tools"_L1 && part != "libs"_L1) {
if (logLevel != SilentLog) {
std::cerr << qPrintable(tr("File %1: Property 'QtPart' contains unknown element "
"'%2'. Valid entries are 'examples', 'tests', 'tools' "
"and 'libs'.").arg(
QDir::toNativeSeparators(filePath), part))
<< std::endl;
}
validPackage = false;
}
}
if (p.origin == Package::Origin::Provisioned && !p.files.isEmpty()) {
std::cerr << qPrintable(tr("File %1: Do not set property 'Files' if property "
"'Origin' is set to 'Provisioned'.")
.arg(QDir::toNativeSeparators(filePath)))
<< std::endl;
validPackage = false;
}
if (!(checks & Check::Paths))
return validPackage;
const QDir dir = p.path;
if (!dir.exists()) {
std::cerr << qPrintable(
tr("File %1: Directory '%2' does not exist.")
.arg(QDir::toNativeSeparators(filePath), QDir::toNativeSeparators(p.path)))
<< std::endl;
validPackage = false;
} else {
for (const QString &file : std::as_const(p.files)) {
if (!dir.exists(file)) {
if (logLevel != SilentLog) {
std::cerr << qPrintable(
tr("File %1: Path '%2' does not exist in directory '%3'.")
.arg(QDir::toNativeSeparators(filePath),
QDir::toNativeSeparators(file),
QDir::toNativeSeparators(p.path)))
<< std::endl;
}
validPackage = false;
}
}
}
return validPackage;
}
static std::optional<QStringList> toStringList(const QJsonValue &value)
{
if (!value.isArray())
return std::nullopt;
QStringList result;
for (const auto &iter : value.toArray()) {
if (iter.type() != QJsonValue::String)
return std::nullopt;
result.push_back(iter.toString());
}
return result;
}
static std::optional<QString> arrayToMultiLineString(const QJsonValue &value)
{
if (!value.isArray())
return std::nullopt;
QString result;
for (const auto &iter : value.toArray()) {
if (iter.type() != QJsonValue::String)
return std::nullopt;
result.append(iter.toString());
result.append(QLatin1StringView("\n"));
}
return result;
}
// Extracts SPDX license ids from a SPDX license expression.
// For "(BSD-3-Clause AND BeerWare)" this function returns { "BSD-3-Clause", "BeerWare" }.
static QStringList extractLicenseIdsFromSPDXExpression(QString expression)
{
const QStringList spdxOperators = {
u"AND"_s,
u"OR"_s,
u"WITH"_s
};
// Replace parentheses with spaces. We're not interested in grouping.
const QRegularExpression parensRegex(u"[()]"_s);
expression.replace(parensRegex, u" "_s);
// Split the string at space boundaries to extract tokens.
QStringList result;
for (const QString &token : expression.split(QLatin1Char(' '), Qt::SkipEmptyParts)) {
if (spdxOperators.contains(token))
continue;
// Remove the unary + operator, if present.
if (token.endsWith(QLatin1Char('+')))
result.append(token.mid(0, token.size() - 1));
else
result.append(token);
}
return result;
}
// Starting at packageDir, look for a LICENSES subdirectory in the directory hierarchy upwards.
// Return a default-constructed QString if the directory was not found.
static QString locateLicensesDir(const QString &packageDir)
{
static const QString licensesSubDir = u"LICENSES"_s;
QDir dir(packageDir);
while (true) {
if (!dir.exists())
break;
if (dir.cd(licensesSubDir))
return dir.path();
if (dir.isRoot() || !dir.cdUp())
break;
}
return {};
}
// Locates the license files that belong to the licenses mentioned in LicenseId and stores them in
// the specified package object.
static bool autoDetectLicenseFiles(Package &p)
{
const QString licensesDirPath = locateLicensesDir(p.path);
const QStringList licenseIds = extractLicenseIdsFromSPDXExpression(p.licenseId);
bool success = true;
QDir licensesDir(licensesDirPath);
QDir licensesDirLocal = p.path;
for (const QString &id : licenseIds) {
QString fileName = id + u".txt";
QString fileNameLocal = u"LICENSE." + id + u".txt";
if (licensesDirLocal.exists(fileNameLocal)) {
p.licenseFiles.append(licensesDirLocal.filePath(fileNameLocal));
} else if (licensesDir.exists(fileName)) {
p.licenseFiles.append(licensesDir.filePath(fileName));
} else {
std::cerr << qPrintable(tr("Missing expected license file:")) << std::endl;
std::cerr << qPrintable(QDir::toNativeSeparators(licensesDirLocal.filePath(fileNameLocal)))
<< std::endl;
if (!licensesDirPath.isEmpty()) {
std::cerr << qPrintable(tr("or\n %1").arg(
QDir::toNativeSeparators(licensesDir.filePath(fileName))))
<< std::endl;
}
success = false;
}
}
return success;
}
// Tries to interpret a json value either as a string or an array of strings, and assigns the
// result to outList. Returns true on success, false on failure. On failure, it also conditionally
// prints an error.
static bool handleStringOrStringArrayJsonKey(QStringList &outList, const QString &key,
QJsonValueConstRef jsonValue, const QString &filePath,
LogLevel logLevel)
{
if (jsonValue.isArray()) {
auto maybeStringList = toStringList(jsonValue);
if (maybeStringList)
outList = maybeStringList.value();
} else if (jsonValue.isString()) {
outList.append(jsonValue.toString());
} else {
if (logLevel != SilentLog) {
std::cerr << qPrintable(tr("File %1: Expected JSON array of strings or "
"string as value of %2.").arg(
QDir::toNativeSeparators(filePath), key))
<< std::endl;
}
return false;
}
return true;
}
// Transforms a JSON object into a Package object
static std::optional<Package> readPackage(const QJsonObject &object, const QString &filePath,
LogLevel logLevel)
{
Package p;
bool validPackage = true;
const QString directory = QFileInfo(filePath).absolutePath();
p.filePath = QFileInfo(filePath).absoluteFilePath();
p.path = directory;
for (auto iter = object.constBegin(); iter != object.constEnd(); ++iter) {
const QString key = iter.key();
if (!iter.value().isString() && key != "QtParts"_L1 && key != "SecurityCritical"_L1
&& key != "Files"_L1 && key != "LicenseFiles"_L1 && key != "Comment"_L1
&& key != "Copyright"_L1 && key != "CPE"_L1 && key != "PURL"_L1) {
if (logLevel != SilentLog)
std::cerr << qPrintable(tr("File %1: Expected JSON string as value of %2.").arg(
QDir::toNativeSeparators(filePath), key)) << std::endl;
validPackage = false;
continue;
}
const QString value = iter.value().toString();
if (key == "Name"_L1) {
p.name = value;
} else if (key == "Path"_L1) {
p.path = QDir(directory).absoluteFilePath(value);
} else if (key == "Files"_L1) {
QJsonValueConstRef jsonValue = iter.value();
if (jsonValue.isArray()) {
auto maybeStringList = toStringList(jsonValue);
if (maybeStringList)
p.files = maybeStringList.value();
} else if (jsonValue.isString()) {
// Legacy format: multiple values separated by space in one string.
p.files = value.simplified().split(QLatin1Char(' '), Qt::SkipEmptyParts);
} else {
if (logLevel != SilentLog) {
std::cerr << qPrintable(tr("File %1: Expected JSON array of strings as value "
"of Files.").arg(QDir::toNativeSeparators(filePath)))
<< std::endl;
validPackage = false;
continue;
}
}
} else if (key == "Comment"_L1) {
// Accepted purely to record details of potential interest doing
// updates in future. Value is an arbitrary object. Any number of
// Comment entries may be present: JSON doesn't require names to be
// unique, albeit some linters may kvetch.
} else if (key == "Id"_L1) {
p.id = value;
} else if (key == "Origin"_L1) {
if (value == "InSource"_L1) {
p.origin = Package::Origin::InSource;
} else if (value == "Provisioned"_L1) {
p.origin = Package::Origin::Provisioned;
} else {
std::cerr << qPrintable(tr("File %1: Expected either 'InSource' or 'Provisioned'"
" as value of 'Origin'.")
.arg(QDir::toNativeSeparators(filePath)))
<< std::endl;
validPackage = false;
continue;
}
} else if (key == "Homepage"_L1) {
p.homepage = value;
} else if (key == "Version"_L1) {
p.version = value;
} else if (key == "DownloadLocation"_L1) {
p.downloadLocation = value;
} else if (key == "License"_L1) {
p.license = value;
} else if (key == "LicenseId"_L1) {
p.licenseId = value;
} else if (key == "LicenseFile"_L1) {
p.licenseFiles = QStringList(QDir(directory).absoluteFilePath(value));
} else if (key == "LicenseFiles"_L1) {
auto strings = toStringList(iter.value());
if (!strings) {
if (logLevel != SilentLog)
std::cerr << qPrintable(tr("File %1: Expected JSON array of strings in %2.")
.arg(QDir::toNativeSeparators(filePath), key))
<< std::endl;
validPackage = false;
continue;
}
const QDir dir(directory);
for (const auto &iter : std::as_const(strings.value()))
p.licenseFiles.push_back(dir.absoluteFilePath(iter));
} else if (key == "Copyright"_L1) {
QJsonValueConstRef jsonValue = iter.value();
if (jsonValue.isArray()) {
// Array joined with new lines
auto maybeString = arrayToMultiLineString(jsonValue);
if (maybeString)
p.copyright = maybeString.value();
} else if (jsonValue.isString()) {
// Legacy format: multiple values separated by space in one string.
p.copyright = value;
} else {
if (logLevel != SilentLog) {
std::cerr << qPrintable(tr("File %1: Expected JSON array of strings or "
"string as value of %2.").arg(
QDir::toNativeSeparators(filePath), key)) << std::endl;
validPackage = false;
continue;
}
}
} else if (key == "CPE"_L1) {
const QJsonValueConstRef jsonValue = iter.value();
if (!handleStringOrStringArrayJsonKey(p.cpeList, key, jsonValue, filePath, logLevel)) {
validPackage = false;
continue;
}
} else if (key == "PURL"_L1) {
const QJsonValueConstRef jsonValue = iter.value();
if (!handleStringOrStringArrayJsonKey(p.purlList, key, jsonValue, filePath, logLevel)) {
validPackage = false;
continue;
}
} else if (key == "CopyrightFile"_L1) {
p.copyrightFile = QDir(directory).absoluteFilePath(value);
} else if (key == "PackageComment"_L1) {
p.packageComment = value;
} else if (key == "QDocModule"_L1) {
p.qdocModule = value;
} else if (key == "Description"_L1) {
p.description = value;
} else if (key == "QtUsage"_L1) {
p.qtUsage = value;
} else if (key == "SecurityCritical"_L1) {
if (!iter.value().isBool()) {
std::cerr << qPrintable(tr("File %1: Expected JSON boolean in %2.")
.arg(QDir::toNativeSeparators(filePath), key))
<< std::endl;
validPackage = false;
continue;
}
p.securityCritical = iter.value().toBool();
} else if (key == "QtParts"_L1) {
auto parts = toStringList(iter.value());
if (!parts) {
if (logLevel != SilentLog) {
std::cerr << qPrintable(tr("File %1: Expected JSON array of strings in %2.")
.arg(QDir::toNativeSeparators(filePath), key))
<< std::endl;
}
validPackage = false;
continue;
}
p.qtParts = parts.value();
} else {
if (logLevel != SilentLog) {
std::cerr << qPrintable(tr("File %1: Unknown key %2.").arg(
QDir::toNativeSeparators(filePath), key)) << std::endl;
}
validPackage = false;
}
}
// Replace $<VERSION> and $<VERSION_DASHED> in string values
{
const QString versionVar = u"$<VERSION>"_s;
const QString versionDashedVar = u"$<VERSION_DASHED>"_s;
auto replaceInString = [&](QString &s) {
if (s.contains(versionVar) || s.contains(versionDashedVar)) {
if (p.version.isEmpty()) {
if (logLevel != SilentLog) {
std::cerr << qPrintable(
tr("File %1: $<VERSION> used but 'Version' is not set.")
.arg(QDir::toNativeSeparators(filePath)))
<< std::endl;
}
validPackage = false;
return;
}
s.replace(versionVar, p.version);
s.replace(versionDashedVar,
QString(p.version).replace(u'.', u'-'));
}
};
auto replaceInList = [&](QStringList &list) {
for (QString &s : list)
replaceInString(s);
};
replaceInString(p.name);
replaceInString(p.homepage);
replaceInString(p.downloadLocation);
replaceInString(p.description);
replaceInString(p.qtUsage);
replaceInString(p.packageComment);
replaceInList(p.cpeList);
replaceInList(p.purlList);
}
if (!p.copyrightFile.isEmpty()) {
QFile file(p.copyrightFile);
if (!file.open(QIODevice::ReadOnly)) {
std::cerr << qPrintable(tr("File %1: Cannot open 'CopyrightFile' %2.\n")
.arg(QDir::toNativeSeparators(filePath),
QDir::toNativeSeparators(p.copyrightFile)));
validPackage = false;
}
p.copyrightFileContents = QString::fromUtf8(file.readAll());
}
if (p.licenseFiles.isEmpty() && !autoDetectLicenseFiles(p))
return std::nullopt;
for (const QString &licenseFile : std::as_const(p.licenseFiles)) {
QFile file(licenseFile);
if (!file.open(QIODevice::ReadOnly)) {
if (logLevel != SilentLog) {
std::cerr << qPrintable(tr("File %1: Cannot open 'LicenseFile' %2.\n")
.arg(QDir::toNativeSeparators(filePath),
QDir::toNativeSeparators(licenseFile)));
}
validPackage = false;
}
p.licenseFilesContents << QString::fromUtf8(file.readAll()).trimmed();
}
if (!validPackage)
return std::nullopt;
return p;
}
// Parses a package's details from a README.chromium file
static Package parseChromiumFile(QFile &file, const QString &filePath, LogLevel logLevel)
{
const QString directory = QFileInfo(filePath).absolutePath();
// Parse the fields in the file
QHash<QString, QString> fields;
QTextStream in(&file);
while (!in.atEnd()) {
QString line = in.readLine().trimmed();
QStringList parts = line.split(u":"_s);
if (parts.size() < 2)
continue;
QString key = parts.at(0);
parts.removeFirst();
QString value = parts.join(QString()).trimmed();
fields[key] = value;
if (line == "Description:"_L1) { // special field : should handle multi-lines values
while (!in.atEnd()) {
QString line = in.readLine().trimmed();
if (line.startsWith("Local Modifications:"_L1)) // Don't include this part
break;
fields[key] += line + u"\n"_s;
}
break;
}
}
// Construct the Package object
Package p;
QString shortName = fields.contains("Short Name"_L1)
? fields["Short Name"_L1]
: fields["Name"_L1];
QString version = fields[u"Version"_s];
p.filePath = QFileInfo(filePath).absoluteFilePath();
p.id = u"chromium-"_s + shortName.toLower().replace(QChar::Space, u"-"_s);
p.name = fields[u"Name"_s];
if (version != QLatin1Char('0')) // "0" : not applicable
p.version = version;
p.license = fields[u"License"_s];
p.homepage = fields[u"URL"_s];
p.qdocModule = u"qtwebengine"_s;
p.qtUsage = u"Used in Qt WebEngine"_s;
p.description = fields[u"Description"_s].trimmed();
p.path = directory;
QString licenseFile = fields[u"License File"_s];
if (licenseFile != QString() && licenseFile != "NOT_SHIPPED"_L1) {
p.licenseFiles = QStringList(QDir(directory).absoluteFilePath(licenseFile));
} else {
// Look for a LICENSE or COPYING file as a fallback
QDir dir = directory;
dir.setNameFilters({ u"LICENSE"_s, u"COPYING"_s });
dir.setFilter(QDir::Files | QDir::NoDotAndDotDot);
const QFileInfoList entries = dir.entryInfoList();
if (!entries.empty())
p.licenseFiles = QStringList(entries.at(0).absoluteFilePath());
}
// let's ignore warnings regarding Chromium files for now
Q_UNUSED(validatePackage(p, {}, logLevel));
return p;
}
struct CursorPosition
{
int line = -1;
int column = -1;
};
static CursorPosition mapFromOffset(const QByteArray &content, int offset)
{
CursorPosition pos{ 1, 1 };
for (int i = 0; i < content.size(); ++i) {
if (i == offset)
return pos;
if (content[i] == '\n') {
pos.line++;
pos.column = 1;
} else {
pos.column++;
}
}
return CursorPosition();
}
std::optional<QList<Package>> readFile(const QString &filePath, LogLevel logLevel)
{
QList<Package> packages;
bool errorsFound = false;
if (logLevel == VerboseLog) {
std::cerr << qPrintable(tr("Reading file %1...").arg(
QDir::toNativeSeparators(filePath))) << std::endl;
}
QFile file(filePath);
if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
if (logLevel != SilentLog)
std::cerr << qPrintable(tr("Could not open file %1.").arg(
QDir::toNativeSeparators(file.fileName()))) << std::endl;
return std::nullopt;
}
if (filePath.endsWith(".json"_L1)) {
QJsonParseError jsonParseError;
const QByteArray content = file.readAll();
const QJsonDocument document = QJsonDocument::fromJson(content, &jsonParseError);
if (document.isNull()) {
if (logLevel != SilentLog) {
const CursorPosition pos = mapFromOffset(content, jsonParseError.offset);
std::cerr << qPrintable(tr("Could not parse file %1: %2 at line %3, column %4")
.arg(QDir::toNativeSeparators(file.fileName()),
jsonParseError.errorString(),
QString::number(pos.line),
QString::number(pos.column)))
<< std::endl;
}
return std::nullopt;
}
if (document.isObject()) {
std::optional<Package> p =
readPackage(document.object(), file.fileName(), logLevel);
if (p) {
packages << *p;
} else {
errorsFound = true;
}
} else if (document.isArray()) {
QJsonArray array = document.array();
for (int i = 0, size = array.size(); i < size; ++i) {
QJsonValue value = array.at(i);
if (value.isObject()) {
std::optional<Package> p =
readPackage(value.toObject(), file.fileName(), logLevel);
if (p) {
packages << *p;
} else {
errorsFound = true;
}
} else {
if (logLevel != SilentLog) {
std::cerr << qPrintable(tr("File %1: Expecting JSON object in array.")
.arg(QDir::toNativeSeparators(file.fileName())))
<< std::endl;
}
errorsFound = true;
}
}
} else {
if (logLevel != SilentLog) {
std::cerr << qPrintable(tr("File %1: Expecting JSON object in array.").arg(
QDir::toNativeSeparators(file.fileName()))) << std::endl;
}
errorsFound = true;
}
} else if (filePath.endsWith(".chromium"_L1)) {
Package chromiumPackage = parseChromiumFile(file, filePath, logLevel);
if (!chromiumPackage.name.isEmpty()) // Skip invalid README.chromium files
packages << chromiumPackage;
} else {
if (logLevel != SilentLog) {
std::cerr << qPrintable(tr("File %1: Unsupported file type.")
.arg(QDir::toNativeSeparators(file.fileName())))
<< std::endl;
}
errorsFound = true;
}
if (errorsFound)
return std::nullopt;
return packages;
}
std::optional<QList<Package>> scanDirectory(const QString &directory, InputFormats inputFormats,
LogLevel logLevel)
{
QDir dir(directory);
QList<Package> packages;
bool errorsFound = false;
QStringList nameFilters = QStringList();
if (inputFormats & InputFormat::QtAttributions)
nameFilters << u"qt_attribution.json"_s;
if (inputFormats & InputFormat::ChromiumAttributions)
nameFilters << u"README.chromium"_s;
if (qEnvironmentVariableIsSet("QT_ATTRIBUTIONSSCANNER_TEST"))
nameFilters << u"qt_attribution_test.json"_s << u"README_test.chromium"_s;
dir.setNameFilters(nameFilters);
dir.setFilter(QDir::AllDirs | QDir::NoDotAndDotDot | QDir::Files);
const QFileInfoList entries = dir.entryInfoList();
for (const QFileInfo &info : entries) {
if (info.isDir()) {
std::optional<QList<Package>> ps =
scanDirectory(info.filePath(), inputFormats, logLevel);
if (!ps)
errorsFound = true;
else
packages += *ps;
} else {
std::optional p = readFile(info.filePath(), logLevel);
if (!p)
errorsFound = true;
else
packages += *p;
}
}
if (errorsFound)
return std::nullopt;
return packages;
}
} // namespace Scanner