python.cpp - mozsearch

++ // Copyright (C) 2002-2007 Detlev Offenbach <detlev@die-offenbachs.de>

++ // Copyright (C) 2021 The Qt Company Ltd.

++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0

++

++ #include <translator.h>

++ #include "trparser.h"

++ #include "metastrings.h"

++

++ #include <QtCore/qhash.h>

++ #include <QtCore/qlist.h>

++ #include <QtCore/qstring.h>

++ #include <QtCore/qtextstream.h>

++ #include <QtCore/qstack.h>

++

++ #include <cctype>

++ #include <cerrno>

++ #include <cstdio>

++ #include <cstring>

++

++ using namespace Qt::StringLiterals;

++

++ QT_BEGIN_NAMESPACE

++

++ class PythonParser

++ {

++

++     enum Token {

++         Tok_Eof,

++         Tok_class,

++         Tok_def,

++         Tok_return,

++         Tok_tr,

++         Tok_trUtf8,

++         Tok_translate,

++         Tok_Ident,

++         Tok_Dot,

++         Tok_String,

++         Tok_LeftParen,

++         Tok_RightParen,

++         Tok_Comma,

++         Tok_None,

++         Tok_Integer

++     };

++

++     enum class StringType { NoString, String, FormatString, RawString };

++

++ public:

++     PythonParser(Translator &translator, const QString &fileName, bool &error, ConversionData &cd)

++         : tor(translator), m_cd(cd)

++     {

++ #ifdef Q_CC_MSVC

++         const auto *fileNameC = reinterpret_cast<const wchar_t *>(fileName.utf16());

++         error = _wfopen_s(&yyInFile, fileNameC, L"r") != 0;

++ #else

++         const QByteArray fileNameC = QFile::encodeName(fileName);

++         yyInFile = std::fopen(fileNameC.constData(), "r");

++         error = yyInFile == nullptr;

++ #endif

++         if (!error)

++             startTokenizer(fileName);

++     }

++

++     /*

++       Accomplishes a very easy task: It finds all strings inside a tr() or translate()

++       call, and possibly finds out the context of the call. It supports

++       three cases:

++       (1) the context is specified, as in FunnyDialog.tr("Hello") or

++          translate("FunnyDialog", "Hello");

++       (2) the call appears within an inlined function;

++       (3) the call appears within a function defined outside the class definition.

++     */

++     void parse(const QByteArray &initialContext = {}, const QByteArray &defaultContext = {})

++     {

++         QByteArray context;

++         QByteArray text;

++         QByteArray comment;

++         QByteArray prefix;

++

++         yyTok = getToken();

++         while (yyTok != Tok_Eof) {

++

++             switch (yyTok) {

++             case Tok_class: {

++                 if (yyIndentationSize < 0 && yyContinuousSpaceCount > 0)

++                     yyIndentationSize = yyContinuousSpaceCount; // First indented "class"

++                 const int indent =

++                         yyIndentationSize > 0 ? yyContinuousSpaceCount / yyIndentationSize : 0;

++                 while (!yyContextStack.isEmpty() && yyContextStack.top().second >= indent)

++                     yyContextStack.pop();

++                 yyTok = getToken();

++                 yyContextStack.push({ yyIdent, indent });

++                 yyTok = getToken();

++             } break;

++             case Tok_def:

++                 if (yyIndentationSize < 0 && yyContinuousSpaceCount > 0)

++                     yyIndentationSize = yyContinuousSpaceCount; // First indented "def"

++                 if (!yyContextStack.isEmpty()) {

++                     // Pop classes if the function is further outdented than the class on the top

++                     // (end of a nested class).

++                     const int classIndent = yyIndentationSize > 0

++                             ? yyContinuousSpaceCount / yyIndentationSize - 1

++                             : 0;

++                     while (!yyContextStack.isEmpty() && yyContextStack.top().second > classIndent)

++                         yyContextStack.pop();

++                 }

++                 yyTok = getToken();

++                 break;

++             case Tok_tr:

++             case Tok_trUtf8: {

++                 yyTok = getToken();

++                 const int lineNo = yyCurLineNo;

++                 if (match(Tok_LeftParen) && matchString(&text)) {

++                     comment.clear();

++                     bool plural = false;

++

++                     MetaStrings metaBackup = std::move(metaStrings);

++

++                     if (match(Tok_RightParen)) {

++                         // There is no comment or plural arguments.

++                     } else if (match(Tok_Comma) && matchStringOrNone(&comment)) {

++                         // There is a comment argument.

++                         if (match(Tok_RightParen)) {

++                             // There is no plural argument.

++                         } else if (match(Tok_Comma)) {

++                             // There is a plural argument.

++                             plural = true;

++                         }

++                     }

++

++                     if (prefix.isEmpty())

++                         context = defaultContext;

++                     else if (prefix == "self")

++                         context = yyContextStack.isEmpty() ? initialContext

++                                                            : yyContextStack.top().first;

++                     else

++                         context = prefix;

++

++                     prefix.clear();

++                     TranslatorMessage message(QString::fromUtf8(context), QString::fromUtf8(text),

++                                               QString::fromUtf8(comment), {}, yyFileName, lineNo,

++                                               {}, TranslatorMessage::Unfinished, plural);

++                     setMessageParameters(&message, metaBackup);

++                     tor.extend(message, m_cd);

++                 }

++             } break;

++             case Tok_translate: {

++                 bool plural{};

++                 const int lineNo = yyCurLineNo;

++                 MetaStrings metaBackup = std::move(metaStrings);

++                 if (parseTranslate(&text, &context, &comment, &plural)) {

++                     TranslatorMessage message(QString::fromUtf8(context), QString::fromUtf8(text),

++                                               QString::fromUtf8(comment), {}, yyFileName, lineNo,

++                                               {}, TranslatorMessage::Unfinished, plural);

++                     setMessageParameters(&message, metaBackup);

++                     tor.extend(message, m_cd);

++                 } else {

++                     metaStrings = std::move(metaBackup);

++                 }

++             } break;

++             case Tok_Ident:

++                 if (!prefix.isEmpty())

++                     prefix += '.';

++                 prefix += yyIdent;

++                 yyTok = getToken();

++                 if (yyTok != Tok_Dot)

++                     prefix.clear();

++                 break;

++             default:

++                 yyTok = getToken();

++             }

++         }

++

++         if (yyParenDepth != 0) {

++             qWarning("%s: Unbalanced parentheses in Python code", qPrintable(yyFileName));

++         }

++     }

++

++     ~PythonParser() { std::fclose(yyInFile); }

++

++ private:

++     QHash<QByteArray, Token> fillTokens()

++     {

++         QHash<QByteArray, Token> tokens = { { "None", Tok_None },      { "class", Tok_class },

++                                             { "def", Tok_def },        { "return", Tok_return },

++                                             { "__tr", Tok_tr }, // Legacy?

++                                             { "__trUtf8", Tok_trUtf8 } };

++

++         const auto &nameMap = trFunctionAliasManager.nameToTrFunctionMap();

++         for (auto it = nameMap.cbegin(), end = nameMap.cend(); it != end; ++it) {

++             switch (it.value()) {

++             case TrFunctionAliasManager::Function_tr:

++             case TrFunctionAliasManager::Function_QT_TR_NOOP:

++                 tokens.insert(it.key().toUtf8(), Tok_tr);

++                 break;

++             case TrFunctionAliasManager::Function_trUtf8:

++                 tokens.insert(it.key().toUtf8(), Tok_trUtf8);

++                 break;

++             case TrFunctionAliasManager::Function_translate:

++             case TrFunctionAliasManager::Function_QT_TRANSLATE_NOOP:

++             // QTranslator::findMessage() has the same parameters as QApplication::translate().

++             case TrFunctionAliasManager::Function_findMessage:

++                 tokens.insert(it.key().toUtf8(), Tok_translate);

++                 break;

++             default:

++                 break;

++             }

++         }

++         return tokens;

++     }

++

++     QHash<QByteArray, Token> &getTokens()

++     {

++         static QHash<QByteArray, Token> tokens = fillTokens();

++         return tokens;

++     }

++

++     int getChar()

++     {

++         int c;

++

++         if (buf < 0) {

++             c = getc(yyInFile);

++         } else {

++             c = buf;

++             buf = -1;

++         }

++         if (c == '\n') {

++             yyCurLineNo++;

++             yyCountingIndentation = true;

++             yyContinuousSpaceCount = 0;

++         } else if (yyCountingIndentation && (c == 32 || c == 9)) {

++             yyContinuousSpaceCount++;

++         } else {

++             yyCountingIndentation = false;

++         }

++         return c;

++     }

++

++     int peekChar()

++     {

++         int c = getc(yyInFile);

++         buf = c;

++         return c;

++     }

++

++     void startTokenizer(const QString &fileName)

++     {

++         yyInPos = 0;

++         buf = -1;

++

++         yyFileName = fileName;

++         yyCh = getChar();

++         yyParenDepth = 0;

++         yyCurLineNo = 1;

++

++         yyIndentationSize = -1;

++         yyContinuousSpaceCount = 0;

++         yyContextStack.clear();

++     }

++

++     bool parseStringEscape(int quoteChar, StringType stringType)

++     {

++         static const char tab[] = "abfnrtv";

++         static const char backTab[] = "\a\b\f\n\r\t\v";

++

++         yyCh = getChar();

++         if (yyCh == EOF)

++             return false;

++

++         if (stringType == StringType::RawString) {

++             if (yyCh != quoteChar) // Only quotes can be escaped in raw strings

++                 yyString[yyStringLen++] = '\\';

++             yyString[yyStringLen++] = yyCh;

++             yyCh = getChar();

++             return true;

++         }

++

++         if (yyCh == 'x' || yyCh == 'u' || yyCh == 'U') {

++             qsizetype maxSize = 2; // \x

++             if (yyCh == 'u')

++                 maxSize = 4;

++             else if (yyCh == 'U')

++                 maxSize = 8;

++

++             QByteArray hex;

++             yyCh = getChar();

++             if (yyCh == EOF)

++                 return false;

++

++             while (maxSize-- && std::isxdigit(yyCh)) {

++                 hex += char(yyCh);

++                 yyCh = getChar();

++                 if (yyCh == EOF)

++                     return false;

++             }

++             uint n;

++ #ifdef Q_CC_MSVC

++             sscanf_s(hex.constData(), "%x", &n);

++ #else

++             std::sscanf(hex.constData(), "%x", &n);

++ #endif

++

++             QByteArray hexChar = QString(QChar(n)).toUtf8();

++             if (yyStringLen < sizeof(yyString) - hexChar.size())

++                 for (char c : std::as_const(hexChar))

++                     yyString[yyStringLen++] = c;

++             return true;

++         }

++

++         if (yyCh >= '0' && yyCh < '8') {

++             QByteArray oct;

++             int n = 0;

++             do {

++                 oct += char(yyCh);

++                 ++n;

++                 yyCh = getChar();

++                 if (yyCh == EOF)

++                     return false;

++             } while (yyCh >= '0' && yyCh < '8' && n < 3);

++ #ifdef Q_CC_MSVC

++             sscanf_s(oct.constData(), "%o", &n);

++ #else

++             std::sscanf(oct.constData(), "%o", &n);

++ #endif

++             if (yyStringLen < sizeof(yyString) - 1)

++                 yyString[yyStringLen++] = char(n);

++             return true;

++         }

++

++         const char *p = std::strchr(tab, yyCh);

++         if (yyStringLen < sizeof(yyString) - 1) {

++             yyString[yyStringLen++] = p == nullptr ? char(yyCh) : backTab[p - tab];

++         }

++         yyCh = getChar();

++         return true;

++     }

++

++     Token parseString(StringType stringType = StringType::NoString)

++     {

++         int quoteChar = yyCh;

++         bool tripleQuote = false;

++         bool singleQuote = true;

++         bool in = false;

++

++         yyCh = getChar();

++

++         while (yyCh != EOF) {

++             if (singleQuote && (yyCh == '\n' || (in && yyCh == quoteChar)))

++                 break;

++

++             if (yyCh == quoteChar) {

++                 if (peekChar() == quoteChar) {

++                     yyCh = getChar();

++                     if (!tripleQuote) {

++                         tripleQuote = true;

++                         singleQuote = false;

++                         in = true;

++                         yyCh = getChar();

++                     } else {

++                         yyCh = getChar();

++                         if (yyCh == quoteChar) {

++                             tripleQuote = false;

++                             break;

++                         }

++                     }

++                 } else if (tripleQuote) {

++                     if (yyStringLen < sizeof(yyString) - 1)

++                         yyString[yyStringLen++] = char(yyCh);

++                     yyCh = getChar();

++                     continue;

++                 } else {

++                     break;

++                 }

++             } else {

++                 in = true;

++             }

++

++             if (yyCh == '\\') {

++                 if (!parseStringEscape(quoteChar, stringType))

++                     return Tok_Eof;

++             } else {

++                 char *yStart = yyString + yyStringLen;

++                 char *yp = yStart;

++                 while (yyCh != EOF && (tripleQuote || yyCh != '\n') && yyCh != quoteChar

++                        && yyCh != '\\') {

++                     *yp++ = char(yyCh);

++                     yyCh = getChar();

++                 }

++                 yyStringLen += yp - yStart;

++             }

++         }

++         yyString[yyStringLen] = '\0';

++

++         if (yyCh != quoteChar) {

++             printf("%c\n", yyCh);

++

++             qWarning("%s:%d: Unterminated string", qPrintable(yyFileName), yyLineNo);

++         }

++

++         if (yyCh == EOF)

++             return Tok_Eof;

++         yyCh = getChar();

++         return Tok_String;

++     }

++

++     QByteArray readLine()

++     {

++         QByteArray result;

++         while (true) {

++             yyCh = getChar();

++             if (yyCh == EOF || yyCh == '\n')

++                 break;

++             result.append(char(yyCh));

++         }

++         return result;

++     }

++

++     Token getToken(StringType stringType = StringType::NoString)

++     {

++         yyIdent.clear();

++         yyStringLen = 0;

++         while (yyCh != EOF) {

++             yyLineNo = yyCurLineNo;

++

++             if (std::isalpha(yyCh) || yyCh == '_') {

++                 do {

++                     yyIdent.append(char(yyCh));

++                     yyCh = getChar();

++                 } while (std::isalnum(yyCh) || yyCh == '_');

++

++                 return getTokens().value(yyIdent, Tok_Ident);

++             }

++             switch (yyCh) {

++             case '#': {

++                 auto comment = QString::fromUtf8(readLine());

++                 if (!metaStrings.parse(comment)) {

++                     qWarning() << qPrintable(yyFileName) << ':' << yyLineNo << ": "

++                                << metaStrings.popError().toStdString();

++                     break;

++                 }

++                 if (metaStrings.magicComment()) {

++                     auto [context, comment] = *metaStrings.magicComment();

++                     TranslatorMessage msg(transcode(context), QString(), transcode(comment),

++                                           QString(), yyFileName, yyCurLineNo, QStringList(),

++                                           TranslatorMessage::Finished, false);

++                     msg.setExtraComment(transcode(metaStrings.extracomment().simplified()));

++                     tor.append(msg);

++                     tor.setExtras(metaStrings.extra());

++                     metaStrings.clear();

++                 }

++                 break;

++             }

++             case '"':

++             case '\'':

++                 return parseString(stringType);

++             case '(':

++                 yyParenDepth++;

++                 yyCh = getChar();

++                 return Tok_LeftParen;

++             case ')':

++                 yyParenDepth--;

++                 yyCh = getChar();

++                 return Tok_RightParen;

++             case ',':

++                 yyCh = getChar();

++                 return Tok_Comma;

++             case '.':

++                 yyCh = getChar();

++                 return Tok_Dot;

++             case '0':

++             case '1':

++             case '2':

++             case '3':

++             case '4':

++             case '5':

++             case '6':

++             case '7':

++             case '8':

++             case '9': {

++                 QByteArray ba;

++                 ba += char(yyCh);

++                 yyCh = getChar();

++                 const bool hex = yyCh == 'x';

++                 if (hex) {

++                     ba += char(yyCh);

++                     yyCh = getChar();

++                 }

++                 while ((hex ? std::isxdigit(yyCh) : std::isdigit(yyCh))) {

++                     ba += char(yyCh);

++                     yyCh = getChar();

++                 }

++                 bool ok;

++                 auto v = ba.toLongLong(&ok);

++                 Q_UNUSED(v);

++                 if (ok)

++                     return Tok_Integer;

++                 break;

++             }

++             default:

++                 yyCh = getChar();

++             }

++         }

++         return Tok_Eof;

++     }

++

++     bool match(Token t)

++     {

++         const bool matches = (yyTok == t);

++         if (matches)

++             yyTok = getToken();

++         return matches;

++     }

++

++     bool matchStringStart()

++     {

++         if (yyTok == Tok_String)

++             return true;

++         // Check for f"bla{var}" and raw strings r"bla".

++         if (yyTok == Tok_Ident && yyIdent.size() == 1) {

++             switch (yyIdent.at(0)) {

++             case 'r':

++                 yyTok = getToken(StringType::RawString);

++                 return yyTok == Tok_String;

++             case 'f':

++                 yyTok = getToken(StringType::FormatString);

++                 return yyTok == Tok_String;

++             }

++         }

++         return false;

++     }

++

++     bool matchString(QByteArray *s)

++     {

++         s->clear();

++         bool ok = false;

++         while (matchStringStart()) {

++             *s += yyString;

++             yyTok = getToken();

++             ok = true;

++         }

++         return ok;

++     }

++

++     bool matchStringOrNone(QByteArray *s)

++     {

++         bool matches = matchString(s);

++

++         if (!matches)

++             matches = match(Tok_None);

++

++         return matches;

++     }

++

++     /*

++      * Skip any expression that can return a number, which can be

++      * 1. Literal number (e.g. '11')

++      * 2. simple identifier (e.g. 'm_count')

++      * 3. simple function call (e.g. 'size()')

++      * 4. function call on an object (e.g. 'list.size()')

++      *      * Other cases:

++      * size(2,4)

++      * list().size()

++      * list(a,b).size(2,4)

++      * etc. (modeled after cpp CppParser::skipExpression()).

++      */

++     bool skipExpression()

++     {

++         if (match(Tok_Integer))

++             return true;

++

++         int parenlevel = 0;

++         while (parenlevel >= 0) {

++             yyTok = getToken();

++             if (yyTok == Tok_RightParen)

++                 --parenlevel;

++             else if (yyTok == Tok_LeftParen)

++                 ++parenlevel;

++             else if (yyTok == Tok_Eof)

++                 return false;

++         }

++         return true;

++     }

++

++     bool parseTranslate(QByteArray *text, QByteArray *context, QByteArray *comment, bool *plural)

++     {

++         text->clear();

++         context->clear();

++         comment->clear();

++         *plural = false;

++

++         yyTok = getToken();

++         if (!match(Tok_LeftParen) || !matchString(context) || !match(Tok_Comma)

++             || !matchString(text)) {

++             return false;

++         }

++

++         if (match(Tok_RightParen))

++             return true;

++

++         // not a comma or a right paren, illegal syntax

++         if (!match(Tok_Comma))

++             return false;

++

++         // python accepts trailing commas within parenthesis, so allow a comma with nothing after

++         if (match(Tok_RightParen))

++             return true;

++

++         // check for comment

++         if (!matchStringOrNone(comment))

++             return false; // not a comment, or a trailing comma... something is wrong

++

++         if (match(Tok_RightParen))

++             return true;

++

++         // not a comma or a right paren, illegal syntax

++         if (!match(Tok_Comma))

++             return false;

++

++         // python accepts trailing commas within parenthesis, so allow a comma with nothing after

++         if (match(Tok_RightParen))

++             return true;

++

++         // Must be a plural expression

++         if (!skipExpression())

++             return false;

++

++         *plural = true;

++

++         // Ignore any trailing comma here

++         match(Tok_Comma);

++

++         // This must be the end, or there are too many parameters

++         if (match(Tok_RightParen))

++             return true;

++

++         return false;

++     }

++

++     void setMessageParameters(TranslatorMessage *message, const MetaStrings &meta)

++     {

++         // PYSIDE-2863: parseTranslate() can read past the message

++         // and capture extraComments intended for the next message.

++         // Use only extraComments for the current message.

++

++         message->setExtraComment(transcode(meta.extracomment().simplified()));

++         message->setId(meta.msgid());

++         message->setExtras(meta.extra());

++         if (!meta.label().isEmpty())

++             m_cd.appendError("%1:%2: labels cannot be used with text-based translation. "

++                              "Ignoring\n"_L1.arg(yyFileName)

++                                      .arg(yyLineNo));

++     }

++

++     QString yyFileName;

++     Token yyTok{};

++     int yyCh{};

++     QByteArray yyIdent;

++     char yyString[65536];

++     size_t yyStringLen{};

++     int yyParenDepth{};

++     int yyLineNo = 1;

++     int yyCurLineNo{};

++     // the file to read from (if reading from a file)

++     FILE *yyInFile;

++     // the string to read from and current position in the string (otherwise)

++     int yyInPos{};

++     int buf{};

++     int yyIndentationSize{};

++     int yyContinuousSpaceCount{};

++     bool yyCountingIndentation = false;

++     // (Context, indentation level) pair.

++     using ContextPair = QPair<QByteArray, int>;

++     // Stack of (Context, indentation level) pairs.

++     using ContextStack = QStack<ContextPair>;

++     ContextStack yyContextStack;

++     MetaStrings metaStrings;

++     Translator &tor;

++     ConversionData &m_cd;

++ };

++

++ bool loadPython(Translator &translator, const QString &fileName, ConversionData &cd)

++ {

++

++     bool error = false;

++     PythonParser parser(translator, fileName, error, cd);

++     if (error) {

++         cd.appendError(QStringLiteral("Cannot open %1").arg(fileName));

++         return false;

++     }

++

++     parser.parse();

++     return true;

++ }

++

++ QT_END_NAMESPACE

Revision control