Revision control
Copy as Markdown
// Copyright (C) 2002-2007 Detlev Offenbach <detlev@die-offenbachs.de>
// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#include <translator.h>
#include "trparser.h"
#include "metastrings.h"
#include <QtCore/qhash.h>
#include <QtCore/qlist.h>
#include <QtCore/qstring.h>
#include <QtCore/qtextstream.h>
#include <QtCore/qstack.h>
#include <cctype>
#include <cerrno>
#include <cstdio>
#include <cstring>
using namespace Qt::StringLiterals;
QT_BEGIN_NAMESPACE
class PythonParser
{
enum Token {
Tok_Eof,
Tok_class,
Tok_def,
Tok_return,
Tok_tr,
Tok_trUtf8,
Tok_translate,
Tok_Ident,
Tok_Dot,
Tok_String,
Tok_LeftParen,
Tok_RightParen,
Tok_Comma,
Tok_None,
Tok_Integer
};
enum class StringType { NoString, String, FormatString, RawString };
public:
PythonParser(Translator &translator, const QString &fileName, bool &error, ConversionData &cd)
: tor(translator), m_cd(cd)
{
#ifdef Q_CC_MSVC
const auto *fileNameC = reinterpret_cast<const wchar_t *>(fileName.utf16());
error = _wfopen_s(&yyInFile, fileNameC, L"r") != 0;
#else
const QByteArray fileNameC = QFile::encodeName(fileName);
yyInFile = std::fopen(fileNameC.constData(), "r");
error = yyInFile == nullptr;
#endif
if (!error)
startTokenizer(fileName);
}
/*
Accomplishes a very easy task: It finds all strings inside a tr() or translate()
call, and possibly finds out the context of the call. It supports
three cases:
(1) the context is specified, as in FunnyDialog.tr("Hello") or
translate("FunnyDialog", "Hello");
(2) the call appears within an inlined function;
(3) the call appears within a function defined outside the class definition.
*/
void parse(const QByteArray &initialContext = {}, const QByteArray &defaultContext = {})
{
QByteArray context;
QByteArray text;
QByteArray comment;
QByteArray prefix;
yyTok = getToken();
while (yyTok != Tok_Eof) {
switch (yyTok) {
case Tok_class: {
if (yyIndentationSize < 0 && yyContinuousSpaceCount > 0)
yyIndentationSize = yyContinuousSpaceCount; // First indented "class"
const int indent =
yyIndentationSize > 0 ? yyContinuousSpaceCount / yyIndentationSize : 0;
while (!yyContextStack.isEmpty() && yyContextStack.top().second >= indent)
yyContextStack.pop();
yyTok = getToken();
yyContextStack.push({ yyIdent, indent });
yyTok = getToken();
} break;
case Tok_def:
if (yyIndentationSize < 0 && yyContinuousSpaceCount > 0)
yyIndentationSize = yyContinuousSpaceCount; // First indented "def"
if (!yyContextStack.isEmpty()) {
// Pop classes if the function is further outdented than the class on the top
// (end of a nested class).
const int classIndent = yyIndentationSize > 0
? yyContinuousSpaceCount / yyIndentationSize - 1
: 0;
while (!yyContextStack.isEmpty() && yyContextStack.top().second > classIndent)
yyContextStack.pop();
}
yyTok = getToken();
break;
case Tok_tr:
case Tok_trUtf8: {
yyTok = getToken();
const int lineNo = yyCurLineNo;
if (match(Tok_LeftParen) && matchString(&text)) {
comment.clear();
bool plural = false;
MetaStrings metaBackup = std::move(metaStrings);
if (match(Tok_RightParen)) {
// There is no comment or plural arguments.
} else if (match(Tok_Comma) && matchStringOrNone(&comment)) {
// There is a comment argument.
if (match(Tok_RightParen)) {
// There is no plural argument.
} else if (match(Tok_Comma)) {
// There is a plural argument.
plural = true;
}
}
if (prefix.isEmpty())
context = defaultContext;
else if (prefix == "self")
context = yyContextStack.isEmpty() ? initialContext
: yyContextStack.top().first;
else
context = prefix;
prefix.clear();
TranslatorMessage message(QString::fromUtf8(context), QString::fromUtf8(text),
QString::fromUtf8(comment), {}, yyFileName, lineNo,
{}, TranslatorMessage::Unfinished, plural);
setMessageParameters(&message, metaBackup);
tor.extend(message, m_cd);
}
} break;
case Tok_translate: {
bool plural{};
const int lineNo = yyCurLineNo;
MetaStrings metaBackup = std::move(metaStrings);
if (parseTranslate(&text, &context, &comment, &plural)) {
TranslatorMessage message(QString::fromUtf8(context), QString::fromUtf8(text),
QString::fromUtf8(comment), {}, yyFileName, lineNo,
{}, TranslatorMessage::Unfinished, plural);
setMessageParameters(&message, metaBackup);
tor.extend(message, m_cd);
} else {
metaStrings = std::move(metaBackup);
}
} break;
case Tok_Ident:
if (!prefix.isEmpty())
prefix += '.';
prefix += yyIdent;
yyTok = getToken();
if (yyTok != Tok_Dot)
prefix.clear();
break;
default:
yyTok = getToken();
}
}
if (yyParenDepth != 0) {
qWarning("%s: Unbalanced parentheses in Python code", qPrintable(yyFileName));
}
}
~PythonParser() { std::fclose(yyInFile); }
private:
QHash<QByteArray, Token> fillTokens()
{
QHash<QByteArray, Token> tokens = { { "None", Tok_None }, { "class", Tok_class },
{ "def", Tok_def }, { "return", Tok_return },
{ "__tr", Tok_tr }, // Legacy?
{ "__trUtf8", Tok_trUtf8 } };
const auto &nameMap = trFunctionAliasManager.nameToTrFunctionMap();
for (auto it = nameMap.cbegin(), end = nameMap.cend(); it != end; ++it) {
switch (it.value()) {
case TrFunctionAliasManager::Function_tr:
case TrFunctionAliasManager::Function_QT_TR_NOOP:
tokens.insert(it.key().toUtf8(), Tok_tr);
break;
case TrFunctionAliasManager::Function_trUtf8:
tokens.insert(it.key().toUtf8(), Tok_trUtf8);
break;
case TrFunctionAliasManager::Function_translate:
case TrFunctionAliasManager::Function_QT_TRANSLATE_NOOP:
// QTranslator::findMessage() has the same parameters as QApplication::translate().
case TrFunctionAliasManager::Function_findMessage:
tokens.insert(it.key().toUtf8(), Tok_translate);
break;
default:
break;
}
}
return tokens;
}
QHash<QByteArray, Token> &getTokens()
{
static QHash<QByteArray, Token> tokens = fillTokens();
return tokens;
}
int getChar()
{
int c;
if (buf < 0) {
c = getc(yyInFile);
} else {
c = buf;
buf = -1;
}
if (c == '\n') {
yyCurLineNo++;
yyCountingIndentation = true;
yyContinuousSpaceCount = 0;
} else if (yyCountingIndentation && (c == 32 || c == 9)) {
yyContinuousSpaceCount++;
} else {
yyCountingIndentation = false;
}
return c;
}
int peekChar()
{
int c = getc(yyInFile);
buf = c;
return c;
}
void startTokenizer(const QString &fileName)
{
yyInPos = 0;
buf = -1;
yyFileName = fileName;
yyCh = getChar();
yyParenDepth = 0;
yyCurLineNo = 1;
yyIndentationSize = -1;
yyContinuousSpaceCount = 0;
yyContextStack.clear();
}
bool parseStringEscape(int quoteChar, StringType stringType)
{
static const char tab[] = "abfnrtv";
static const char backTab[] = "\a\b\f\n\r\t\v";
yyCh = getChar();
if (yyCh == EOF)
return false;
if (stringType == StringType::RawString) {
if (yyCh != quoteChar) // Only quotes can be escaped in raw strings
yyString[yyStringLen++] = '\\';
yyString[yyStringLen++] = yyCh;
yyCh = getChar();
return true;
}
if (yyCh == 'x' || yyCh == 'u' || yyCh == 'U') {
qsizetype maxSize = 2; // \x
if (yyCh == 'u')
maxSize = 4;
else if (yyCh == 'U')
maxSize = 8;
QByteArray hex;
yyCh = getChar();
if (yyCh == EOF)
return false;
while (maxSize-- && std::isxdigit(yyCh)) {
hex += char(yyCh);
yyCh = getChar();
if (yyCh == EOF)
return false;
}
uint n;
#ifdef Q_CC_MSVC
sscanf_s(hex.constData(), "%x", &n);
#else
std::sscanf(hex.constData(), "%x", &n);
#endif
QByteArray hexChar = QString(QChar(n)).toUtf8();
if (yyStringLen < sizeof(yyString) - hexChar.size())
for (char c : std::as_const(hexChar))
yyString[yyStringLen++] = c;
return true;
}
if (yyCh >= '0' && yyCh < '8') {
QByteArray oct;
int n = 0;
do {
oct += char(yyCh);
++n;
yyCh = getChar();
if (yyCh == EOF)
return false;
} while (yyCh >= '0' && yyCh < '8' && n < 3);
#ifdef Q_CC_MSVC
sscanf_s(oct.constData(), "%o", &n);
#else
std::sscanf(oct.constData(), "%o", &n);
#endif
if (yyStringLen < sizeof(yyString) - 1)
yyString[yyStringLen++] = char(n);
return true;
}
const char *p = std::strchr(tab, yyCh);
if (yyStringLen < sizeof(yyString) - 1) {
yyString[yyStringLen++] = p == nullptr ? char(yyCh) : backTab[p - tab];
}
yyCh = getChar();
return true;
}
Token parseString(StringType stringType = StringType::NoString)
{
int quoteChar = yyCh;
bool tripleQuote = false;
bool singleQuote = true;
bool in = false;
yyCh = getChar();
while (yyCh != EOF) {
if (singleQuote && (yyCh == '\n' || (in && yyCh == quoteChar)))
break;
if (yyCh == quoteChar) {
if (peekChar() == quoteChar) {
yyCh = getChar();
if (!tripleQuote) {
tripleQuote = true;
singleQuote = false;
in = true;
yyCh = getChar();
} else {
yyCh = getChar();
if (yyCh == quoteChar) {
tripleQuote = false;
break;
}
}
} else if (tripleQuote) {
if (yyStringLen < sizeof(yyString) - 1)
yyString[yyStringLen++] = char(yyCh);
yyCh = getChar();
continue;
} else {
break;
}
} else {
in = true;
}
if (yyCh == '\\') {
if (!parseStringEscape(quoteChar, stringType))
return Tok_Eof;
} else {
char *yStart = yyString + yyStringLen;
char *yp = yStart;
while (yyCh != EOF && (tripleQuote || yyCh != '\n') && yyCh != quoteChar
&& yyCh != '\\') {
*yp++ = char(yyCh);
yyCh = getChar();
}
yyStringLen += yp - yStart;
}
}
yyString[yyStringLen] = '\0';
if (yyCh != quoteChar) {
printf("%c\n", yyCh);
qWarning("%s:%d: Unterminated string", qPrintable(yyFileName), yyLineNo);
}
if (yyCh == EOF)
return Tok_Eof;
yyCh = getChar();
return Tok_String;
}
QByteArray readLine()
{
QByteArray result;
while (true) {
yyCh = getChar();
if (yyCh == EOF || yyCh == '\n')
break;
result.append(char(yyCh));
}
return result;
}
Token getToken(StringType stringType = StringType::NoString)
{
yyIdent.clear();
yyStringLen = 0;
while (yyCh != EOF) {
yyLineNo = yyCurLineNo;
if (std::isalpha(yyCh) || yyCh == '_') {
do {
yyIdent.append(char(yyCh));
yyCh = getChar();
} while (std::isalnum(yyCh) || yyCh == '_');
return getTokens().value(yyIdent, Tok_Ident);
}
switch (yyCh) {
case '#': {
auto comment = QString::fromUtf8(readLine());
if (!metaStrings.parse(comment)) {
qWarning() << qPrintable(yyFileName) << ':' << yyLineNo << ": "
<< metaStrings.popError().toStdString();
break;
}
if (metaStrings.magicComment()) {
auto [context, comment] = *metaStrings.magicComment();
TranslatorMessage msg(transcode(context), QString(), transcode(comment),
QString(), yyFileName, yyCurLineNo, QStringList(),
TranslatorMessage::Finished, false);
msg.setExtraComment(transcode(metaStrings.extracomment().simplified()));
tor.append(msg);
tor.setExtras(metaStrings.extra());
metaStrings.clear();
}
break;
}
case '"':
case '\'':
return parseString(stringType);
case '(':
yyParenDepth++;
yyCh = getChar();
return Tok_LeftParen;
case ')':
yyParenDepth--;
yyCh = getChar();
return Tok_RightParen;
case ',':
yyCh = getChar();
return Tok_Comma;
case '.':
yyCh = getChar();
return Tok_Dot;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': {
QByteArray ba;
ba += char(yyCh);
yyCh = getChar();
const bool hex = yyCh == 'x';
if (hex) {
ba += char(yyCh);
yyCh = getChar();
}
while ((hex ? std::isxdigit(yyCh) : std::isdigit(yyCh))) {
ba += char(yyCh);
yyCh = getChar();
}
bool ok;
auto v = ba.toLongLong(&ok);
Q_UNUSED(v);
if (ok)
return Tok_Integer;
break;
}
default:
yyCh = getChar();
}
}
return Tok_Eof;
}
bool match(Token t)
{
const bool matches = (yyTok == t);
if (matches)
yyTok = getToken();
return matches;
}
bool matchStringStart()
{
if (yyTok == Tok_String)
return true;
// Check for f"bla{var}" and raw strings r"bla".
if (yyTok == Tok_Ident && yyIdent.size() == 1) {
switch (yyIdent.at(0)) {
case 'r':
yyTok = getToken(StringType::RawString);
return yyTok == Tok_String;
case 'f':
yyTok = getToken(StringType::FormatString);
return yyTok == Tok_String;
}
}
return false;
}
bool matchString(QByteArray *s)
{
s->clear();
bool ok = false;
while (matchStringStart()) {
*s += yyString;
yyTok = getToken();
ok = true;
}
return ok;
}
bool matchStringOrNone(QByteArray *s)
{
bool matches = matchString(s);
if (!matches)
matches = match(Tok_None);
return matches;
}
/*
* Skip any expression that can return a number, which can be
* 1. Literal number (e.g. '11')
* 2. simple identifier (e.g. 'm_count')
* 3. simple function call (e.g. 'size()')
* 4. function call on an object (e.g. 'list.size()')
* * Other cases:
* size(2,4)
* list().size()
* list(a,b).size(2,4)
* etc. (modeled after cpp CppParser::skipExpression()).
*/
bool skipExpression()
{
if (match(Tok_Integer))
return true;
int parenlevel = 0;
while (parenlevel >= 0) {
yyTok = getToken();
if (yyTok == Tok_RightParen)
--parenlevel;
else if (yyTok == Tok_LeftParen)
++parenlevel;
else if (yyTok == Tok_Eof)
return false;
}
return true;
}
bool parseTranslate(QByteArray *text, QByteArray *context, QByteArray *comment, bool *plural)
{
text->clear();
context->clear();
comment->clear();
*plural = false;
yyTok = getToken();
if (!match(Tok_LeftParen) || !matchString(context) || !match(Tok_Comma)
|| !matchString(text)) {
return false;
}
if (match(Tok_RightParen))
return true;
// not a comma or a right paren, illegal syntax
if (!match(Tok_Comma))
return false;
// python accepts trailing commas within parenthesis, so allow a comma with nothing after
if (match(Tok_RightParen))
return true;
// check for comment
if (!matchStringOrNone(comment))
return false; // not a comment, or a trailing comma... something is wrong
if (match(Tok_RightParen))
return true;
// not a comma or a right paren, illegal syntax
if (!match(Tok_Comma))
return false;
// python accepts trailing commas within parenthesis, so allow a comma with nothing after
if (match(Tok_RightParen))
return true;
// Must be a plural expression
if (!skipExpression())
return false;
*plural = true;
// Ignore any trailing comma here
match(Tok_Comma);
// This must be the end, or there are too many parameters
if (match(Tok_RightParen))
return true;
return false;
}
void setMessageParameters(TranslatorMessage *message, const MetaStrings &meta)
{
// PYSIDE-2863: parseTranslate() can read past the message
// and capture extraComments intended for the next message.
// Use only extraComments for the current message.
message->setExtraComment(transcode(meta.extracomment().simplified()));
message->setId(meta.msgid());
message->setExtras(meta.extra());
if (!meta.label().isEmpty())
m_cd.appendError("%1:%2: labels cannot be used with text-based translation. "
"Ignoring\n"_L1.arg(yyFileName)
.arg(yyLineNo));
}
QString yyFileName;
Token yyTok{};
int yyCh{};
QByteArray yyIdent;
char yyString[65536];
size_t yyStringLen{};
int yyParenDepth{};
int yyLineNo = 1;
int yyCurLineNo{};
// the file to read from (if reading from a file)
FILE *yyInFile;
// the string to read from and current position in the string (otherwise)
int yyInPos{};
int buf{};
int yyIndentationSize{};
int yyContinuousSpaceCount{};
bool yyCountingIndentation = false;
// (Context, indentation level) pair.
using ContextPair = QPair<QByteArray, int>;
// Stack of (Context, indentation level) pairs.
using ContextStack = QStack<ContextPair>;
ContextStack yyContextStack;
MetaStrings metaStrings;
Translator &tor;
ConversionData &m_cd;
};
bool loadPython(Translator &translator, const QString &fileName, ConversionData &cd)
{
bool error = false;
PythonParser parser(translator, fileName, error, cd);
if (error) {
cd.appendError(QStringLiteral("Cannot open %1").arg(fileName));
return false;
}
parser.parse();
return true;
}
QT_END_NAMESPACE