Revision control
++ // Copyright (C) 2025 The Qt Company Ltd.
++ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
++ // Qt-Security score:significant reason:default
++
++ #ifndef IOPROCESSOR_P_H
++ #define IOPROCESSOR_P_H
++
++ //
++ // W A R N I N G
++ // -------------
++ //
++ // This file is not part of the Qt API. It exists purely as an
++ // implementation detail. This header file may change from version to
++ // version without notice, or even be removed.
++ //
++ // We mean it.
++ //
++
++ #include <QtCore/private/qtcoreglobal_p.h>
++
++ #include <QtCore/qstring.h>
++ #include <QtCore/qspan.h>
++ #include <QtCore/qhash.h>
++ #include <QtCore/qfiledevice.h>
++ #include <QtCore/qloggingcategory.h>
++ #include <QtCore/qdeadlinetimer.h>
++
++ #ifdef Q_OS_LINUX
++ # include <QtCore/qsocketnotifier.h>
++ struct io_uring_sqe;
++ struct io_uring_cqe;
++ #elif defined(Q_OS_WIN)
++ # include <QtCore/qwineventnotifier.h>
++ # include <qt_windows.h>
++ # include <ioringapi.h>
++ #endif
++
++ #include <algorithm>
++ #include <filesystem>
++ #include <QtCore/qxpfunctional.h>
++ #include <variant>
++ #include <optional>
++ #include <type_traits>
++
++ /*
++ This file defines an interface for the backend of QRandomAccessFile.
++ The backends themselves are implemented in platform-specific files, such as
++ ioring_linux.cpp, ioring_win.cpp, etc.
++ And has a lower-level interface than the public interface will have, but the
++ separation hopefully makes it easier to implement the ioring backends, test
++ them, and tweak them without the higher-level interface needing to see
++ changes, and to make it possible to tweak the higher-level interface without
++ needing to touch the (somewhat similar) ioring backends.
++
++ Most of the interface is just an enum QIORing::Operation + the
++ QIORingRequest template class, which is specialized for each operation so it
++ carries just the relevant data for that operation. And a small mechanism to
++ store the request in a generic manner so they can be used in the
++ implementation files at the cost of some overhead.
++
++ There will be absolutely zero binary compatibility guarantees for this
++ interface.
++ */
++
++ QT_BEGIN_NAMESPACE
++
++ Q_DECLARE_LOGGING_CATEGORY(lcQIORing);
++
++ namespace QtPrivate {
++ Q_NAMESPACE
++
++ #define FOREACH_IO_OPERATION(OP) \
++ OP(Open) \
++ OP(Close) \
++ OP(Read) \
++ OP(Write) \
++ OP(VectoredRead) \
++ OP(VectoredWrite) \
++ OP(Flush) \
++ OP(Stat) \
++ OP(Cancel) \
++ /**/
++ #define DEFINE_ENTRY(OP) OP,
++
++ // clang-format off
++ enum class Operation : quint8 {
++ FOREACH_IO_OPERATION(DEFINE_ENTRY)
++
++ NumOperations,
++ };
++ // clang-format on
++ Q_ENUM_NS(Operation);
++ #undef DEFINE_ENTRY
++
++ #ifdef Q_OS_WIN
++ struct IORingApiTable;
++ #endif
++
++ #ifdef Q_OS_LINUX
++ extern Q_CORE_EXPORT std::atomic<qsizetype> testMaxReadWriteLen;
++ #endif
++ }; // namespace QtPrivate
++
++ template <QtPrivate::Operation Op>
++ struct QIORingRequest;
++
++ class QIORing final
++ {
++ class GenericRequestType;
++ struct RequestHandleTag; // Just used as an opaque pointer
++ public:
++ static constexpr quint32 DefaultSubmissionQueueSize = 128;
++ static constexpr quint32 DefaultCompletionQueueSize = DefaultSubmissionQueueSize * 2;
++ using Operation = QtPrivate::Operation;
++ using RequestHandle = RequestHandleTag *;
++
++ Q_CORE_EXPORT
++ explicit QIORing(quint32 submissionQueueSize = DefaultSubmissionQueueSize,
++ quint32 completionQueueSize = DefaultCompletionQueueSize);
++ Q_CORE_EXPORT
++ ~QIORing();
++ Q_DISABLE_COPY_MOVE(QIORing)
++
++ Q_CORE_EXPORT
++ static QIORing *sharedInstance();
++ bool ensureInitialized() { return initializeIORing(); }
++
++ Q_CORE_EXPORT
++ static bool supportsOperation(Operation op);
++ template <Operation Op>
++ QIORing::RequestHandle queueRequest(QIORingRequest<Op> &&request)
++ {
++ Q_ASSERT(supportsOperation(Op));
++ auto &r = pendingRequests.emplace_back(std::move(request));
++ addrItMap.emplace(&r, std::prev(pendingRequests.end()));
++ if (queueRequestInternal(r) == QueuedRequestStatus::CompletedImmediately)
++ return nullptr; // Return an invalid handle, to avoid ABA with following requests
++ return reinterpret_cast<RequestHandle>(&r);
++ }
++ Q_CORE_EXPORT
++ void submitRequests();
++ Q_CORE_EXPORT
++ bool waitForRequest(RequestHandle handle, QDeadlineTimer deadline = QDeadlineTimer::Forever);
++
++ quint32 submissionQueueSize() const noexcept { return sqEntries; }
++ quint32 completionQueueSize() const noexcept { return cqEntries; }
++
++ private:
++ std::list<GenericRequestType> pendingRequests;
++ using PendingRequestsIterator = decltype(pendingRequests.begin());
++ QHash<void *, PendingRequestsIterator> addrItMap;
++ std::optional<PendingRequestsIterator> lastUnqueuedIterator;
++ quint32 sqEntries = 0;
++ quint32 cqEntries = 0;
++ quint32 inFlightRequests = 0;
++ quint32 unstagedRequests = 0;
++ bool stagePending = false;
++ bool preparingRequests = false;
++ qsizetype ongoingSplitOperations = 0;
++
++ Q_CORE_EXPORT
++ bool initializeIORing();
++
++ enum class QueuedRequestStatus : bool {
++ Pending = false,
++ CompletedImmediately = true,
++ };
++ Q_CORE_EXPORT
++ QueuedRequestStatus queueRequestInternal(GenericRequestType &request);
++ void prepareRequests();
++ void completionReady();
++ bool waitForCompletions(QDeadlineTimer deadline);
++
++ template <typename Fun>
++ static auto invokeOnOp(GenericRequestType &req, Fun fn);
++
++ template <Operation Op>
++ static void setFileErrorResult(QIORingRequest<Op> &req, QFileDevice::FileError error)
++ {
++ req.result.template emplace<QFileDevice::FileError>(error);
++ }
++ static void setFileErrorResult(GenericRequestType &req, QFileDevice::FileError error);
++ static void finishRequestWithError(GenericRequestType &req, QFileDevice::FileError error);
++ static bool verifyFd(GenericRequestType &req);
++
++ enum RequestPrepResult : quint8 {
++ Ok,
++ QueueFull,
++ Defer,
++ RequestCompleted,
++ };
++ enum class ReadWriteStatus : bool {
++ MoreToDo,
++ Finished,
++ };
++ #ifdef Q_OS_LINUX
++ using NativeResultType = qint32;
++ static constexpr bool isResultFailure(NativeResultType result) { return result < 0; }
++
++ std::optional<QSocketNotifier> notifier;
++ // io_uring 'sq', 'sqe', and 'cqe' pointers:
++ void *submissionQueue = nullptr;
++ io_uring_sqe *submissionQueueEntries = nullptr;
++ const io_uring_cqe *completionQueueEntries = nullptr;
++
++ // Some pointers for working with the ring-buffer.
++ // The pointers to const are controlled by the kernel.
++ const quint32 *sqHead = nullptr;
++ quint32 *sqTail = nullptr;
++ const quint32 *sqIndexMask = nullptr;
++ quint32 *sqIndexArray = nullptr;
++ quint32 *cqHead = nullptr;
++ const quint32 *cqTail = nullptr;
++ const quint32 *cqIndexMask = nullptr;
++ // Because we want the flush to act as a barrier operation we need to track
++ // if there is one currently in progress. With kernel 6.16+ this seems to be
++ // fixed, but since we support older kernels we implement this deferring
++ // ourselves.
++ bool flushInProgress = false;
++
++ int io_uringFd = -1;
++ int eventDescriptor = -1;
++ [[nodiscard]]
++ RequestPrepResult prepareRequest(io_uring_sqe *sqe, GenericRequestType &request);
++
++ template <typename SpanOfBytes>
++ auto getVectoredOpAddressAndSize(QIORing::GenericRequestType &request,
++ QSpan<SpanOfBytes> spans);
++ #elif defined(Q_OS_WIN)
++ // We use UINT32 because that's the type used for size parameters in their API.
++ static constexpr qsizetype MaxReadWriteLen = std::numeric_limits<UINT32>::max();
++ using NativeResultType = HRESULT;
++ static constexpr bool isResultFailure(NativeResultType result)
++ {
++ return FAILED(result) && result != HRESULT_FROM_WIN32(ERROR_HANDLE_EOF);
++ }
++
++ std::optional<QWinEventNotifier> notifier;
++ HIORING ioRingHandle = nullptr;
++ HANDLE eventHandle = INVALID_HANDLE_VALUE;
++ const QtPrivate::IORingApiTable *apiTable;
++
++ bool initialized = false;
++ bool queueWasFull = false;
++
++ [[nodiscard]]
++ RequestPrepResult prepareRequest(GenericRequestType &request);
++ #endif
++ static QFileDevice::FileError mapFileError(NativeResultType result,
++ QFileDevice::FileError defaultValue);
++ using SetResultFn = qxp::function_ref<qint64(qint64)>;
++ static ReadWriteStatus handleReadCompletion(size_t value, QSpan<std::byte> *destinations,
++ void *voidExtra, SetResultFn setResult);
++ template <Operation Op>
++ ReadWriteStatus handleReadCompletion(NativeResultType result, size_t value,
++ GenericRequestType *request);
++ static ReadWriteStatus handleWriteCompletion(size_t value,
++ const QSpan<const std::byte> *sources,
++ void *voidExtra, SetResultFn setResult);
++ template <Operation Op>
++ ReadWriteStatus handleWriteCompletion(NativeResultType result, size_t value,
++ GenericRequestType *request);
++ void finalizeReadWriteCompletion(GenericRequestType *request, ReadWriteStatus rwstatus);
++ };
++
++ struct QIORingRequestEmptyBase
++ {
++ };
++
++ template <QtPrivate::Operation Op>
++ struct QIORingResult;
++ template <QtPrivate::Operation Op>
++ struct QIORingRequest;
++
++ // @todo: q23::expected once emplace() returns a reference
++ template <QtPrivate::Operation Op>
++ using ExpectedResultType = std::variant<std::monostate, QIORingResult<Op>, QFileDevice::FileError>;
++
++ struct QIORingRequestOffsetFdBase : QIORingRequestEmptyBase
++ {
++ qintptr fd;
++ quint64 offset;
++ };
++
++ template <QtPrivate::Operation Op, typename Base = QIORingRequestOffsetFdBase>
++ struct QIORingRequestBase : Base
++ {
++ ExpectedResultType<Op> result; // To be filled in by the backend
++ QtPrivate::SlotObjUniquePtr callback;
++ template <typename Func>
++ Q_ALWAYS_INLINE void setCallback(Func &&func)
++ {
++ using Prototype = void (*)(const QIORingRequest<Op> &);
++ callback.reset(QtPrivate::makeCallableObject<Prototype>(std::forward<Func>(func)));
++ }
++ };
++
++ template <>
++ struct QIORingResult<QtPrivate::Operation::Open>
++ {
++ // On Windows this is a HANDLE
++ qintptr fd;
++ };
++ template <>
++ struct QIORingRequest<QtPrivate::Operation::Open> final
++ : QIORingRequestBase<QtPrivate::Operation::Open, QIORingRequestEmptyBase>
++ {
++ std::filesystem::path path;
++ QFileDevice::OpenMode flags;
++ };
++ template <>
++ struct QIORingResult<QtPrivate::Operation::Close>
++ {
++ };
++ template <>
++ struct QIORingRequest<QtPrivate::Operation::Close> final
++ : QIORingRequestBase<QtPrivate::Operation::Close, QIORingRequestEmptyBase>
++ {
++ // On Windows this is a HANDLE
++ qintptr fd;
++ };
++
++ template <>
++ struct QIORingResult<QtPrivate::Operation::Write>
++ {
++ qint64 bytesWritten;
++ };
++ template <>
++ struct QIORingRequest<QtPrivate::Operation::Write> final
++ : QIORingRequestBase<QtPrivate::Operation::Write>
++ {
++ QSpan<const std::byte> source;
++ };
++ template <>
++ struct QIORingResult<QtPrivate::Operation::VectoredWrite> final
++ : QIORingResult<QtPrivate::Operation::Write>
++ {
++ };
++ template <>
++ struct QIORingRequest<QtPrivate::Operation::VectoredWrite> final
++ : QIORingRequestBase<QtPrivate::Operation::VectoredWrite>
++ {
++ QSpan<const QSpan<const std::byte>> sources;
++ };
++
++ template <>
++ struct QIORingResult<QtPrivate::Operation::Read>
++ {
++ qint64 bytesRead;
++ };
++ template <>
++ struct QIORingRequest<QtPrivate::Operation::Read> final
++ : QIORingRequestBase<QtPrivate::Operation::Read>
++ {
++ QSpan<std::byte> destination;
++ };
++
++ template <>
++ struct QIORingResult<QtPrivate::Operation::VectoredRead> final
++ : QIORingResult<QtPrivate::Operation::Read>
++ {
++ };
++ template <>
++ struct QIORingRequest<QtPrivate::Operation::VectoredRead> final
++ : QIORingRequestBase<QtPrivate::Operation::VectoredRead>
++ {
++ QSpan<QSpan<std::byte>> destinations;
++ };
++
++ template <>
++ struct QIORingResult<QtPrivate::Operation::Flush> final
++ {
++ // No value in the result, just a success or failure
++ };
++ template <>
++ struct QIORingRequest<QtPrivate::Operation::Flush> final : QIORingRequestBase<QtPrivate::Operation::Flush, QIORingRequestEmptyBase>
++ {
++ // On Windows this is a HANDLE
++ qintptr fd;
++ };
++
++ template <>
++ struct QIORingResult<QtPrivate::Operation::Stat> final
++ {
++ quint64 size;
++ };
++ template <>
++ struct QIORingRequest<QtPrivate::Operation::Stat> final
++ : QIORingRequestBase<QtPrivate::Operation::Stat, QIORingRequestEmptyBase>
++ {
++ // On Windows this is a HANDLE
++ qintptr fd;
++ };
++
++ // This is not inheriting the QIORingRequestBase because it doesn't have a result,
++ // whether it was successful or not is indicated by whether the operation
++ // it was cancelling was successful or not.
++ template <>
++ struct QIORingRequest<QtPrivate::Operation::Cancel> final : QIORingRequestEmptyBase
++ {
++ QIORing::RequestHandle handle;
++ QtPrivate::SlotObjUniquePtr callback;
++ template <typename Func>
++ Q_ALWAYS_INLINE void setCallback(Func &&func)
++ {
++ using Op = QtPrivate::Operation;
++ using Prototype = void (*)(const QIORingRequest<Op::Cancel> &);
++ callback.reset(QtPrivate::makeCallableObject<Prototype>(std::forward<Func>(func)));
++ }
++ };
++
++ template <QIORing::Operation Op>
++ Q_ALWAYS_INLINE void invokeCallback(const QIORingRequest<Op> &request)
++ {
++ if (!request.callback)
++ return;
++ void *args[2] = { nullptr, const_cast<QIORingRequest<Op> *>(&request) };
++ request.callback->call(nullptr, args);
++ }
++
++ class QIORing::GenericRequestType
++ {
++ friend class QIORing;
++
++ #define POPULATE_VARIANT(Op) \
++ QIORingRequest<Operation::Op>, \
++ /**/
++
++ std::variant<
++ FOREACH_IO_OPERATION(POPULATE_VARIANT)
++ std::monostate
++ > taggedUnion;
++
++ #undef POPULATE_VARIANT
++
++ void *extraData = nullptr;
++ bool queued = false;
++
++ template <Operation Op>
++ Q_ALWAYS_INLINE void initializeStorage(QIORingRequest<Op> &&t) noexcept
++ {
++ static_assert(Op < Operation::NumOperations);
++ taggedUnion.emplace<QIORingRequest<Op>>(std::move(t));
++ }
++
++ Q_CORE_EXPORT
++ static void cleanupExtra(Operation op, void *extra);
++ template <typename T>
++ T *getOrInitializeExtra()
++ {
++ if (!extraData)
++ extraData = new T();
++ return static_cast<T *>(extraData);
++ }
++ template <typename T>
++ T *getExtra() const
++ {
++ return static_cast<T *>(extraData);
++ }
++ void reset() noexcept
++ {
++ Operation op = operation();
++ taggedUnion.emplace<std::monostate>();
++ if (extraData)
++ cleanupExtra(op, std::exchange(extraData, nullptr));
++ }
++
++ public:
++ template <Operation Op>
++ explicit GenericRequestType(QIORingRequest<Op> &&t) noexcept
++ {
++ initializeStorage(std::move(t));
++ }
++ ~GenericRequestType() noexcept
++ {
++ reset();
++ }
++ Q_DISABLE_COPY_MOVE(GenericRequestType)
++ // We have to provide equality operators. Since copying is disabled, we just check for equality
++ // based on the address in memory. Two requests could be constructed to be equal, but we don't
++ // actually care because the order in which they are added to the queue may also matter.
++ friend bool operator==(const GenericRequestType &l, const GenericRequestType &r) noexcept
++ {
++ return std::addressof(l) == std::addressof(r);
++ }
++ friend bool operator!=(const GenericRequestType &l, const GenericRequestType &r) noexcept
++ {
++ return !(l == r);
++ }
++
++ Operation operation() const { return Operation(taggedUnion.index()); }
++ template <Operation Op>
++ QIORingRequest<Op> *requestData()
++ {
++ if (operation() == Op)
++ return std::get_if<QIORingRequest<Op>>(&taggedUnion);
++ Q_ASSERT("Wrong operation requested, see operation()");
++ return nullptr;
++ }
++ template <Operation Op>
++ QIORingRequest<Op> takeRequestData()
++ {
++ if (operation() == Op)
++ return std::move(*std::get_if<QIORingRequest<Op>>(&taggedUnion));
++ Q_ASSERT("Wrong operation requested, see operation()");
++ return {};
++ }
++ bool wasQueued() const { return queued; }
++ void setQueued(bool status) { queued = status; }
++ };
++
++ template <typename Fun>
++ auto QIORing::invokeOnOp(GenericRequestType &req, Fun fn)
++ {
++ #define INVOKE_ON_OP(Op) \
++ case QIORing::Operation::Op: \
++ fn(req.template requestData<Operation::Op>()); \
++ return; \
++ /**/
++
++ switch (req.operation()) {
++ FOREACH_IO_OPERATION(INVOKE_ON_OP)
++ case QIORing::Operation::NumOperations:
++ break;
++ }
++
++ Q_UNREACHABLE();
++ #undef INVOKE_ON_OP
++ }
++
++ template <QIORing::Operation Op>
++ QIORing::ReadWriteStatus QIORing::handleReadCompletion(NativeResultType result, size_t value,
++ GenericRequestType *request)
++ {
++ static_assert(Op == Operation::Read || Op == Operation::VectoredRead);
++ QIORingRequest<Op> *readRequest = request->requestData<Op>();
++ Q_ASSERT(readRequest);
++
++ if (isResultFailure(result)) { // error
++ QFileDevice::FileError fileError = mapFileError(result, QFileDevice::ReadError);
++ QIORing::setFileErrorResult(*readRequest, fileError);
++ return ReadWriteStatus::Finished;
++ }
++
++ auto setResult = [readRequest](qint64 bytesRead) {
++ auto &readResult = [&readRequest]() -> QIORingResult<Op> & {
++ if (auto *result = std::get_if<QIORingResult<Op>>(&readRequest->result))
++ return *result;
++ return readRequest->result.template emplace<QIORingResult<Op>>();
++ }();
++ readResult.bytesRead += bytesRead;
++ return readResult.bytesRead;
++ };
++
++ auto *destinations = [&readRequest]() {
++ if constexpr (Op == Operation::Read)
++ return &readRequest->destination;
++ else
++ return &readRequest->destinations[0];
++ }();
++
++ QIORing::ReadWriteStatus rwstatus = handleReadCompletion(value, destinations,
++ request->getExtra<void>(), setResult);
++ finalizeReadWriteCompletion(request, rwstatus);
++ return rwstatus;
++ }
++
++ template <QIORing::Operation Op>
++ QIORing::ReadWriteStatus QIORing::handleWriteCompletion(NativeResultType result, size_t value,
++ GenericRequestType *request)
++ {
++ static_assert(Op == Operation::Write || Op == Operation::VectoredWrite);
++ QIORingRequest<Op> *writeRequest = request->requestData<Op>();
++ Q_ASSERT(writeRequest);
++
++ if (isResultFailure(result)) { // error
++ QFileDevice::FileError fileError = mapFileError(result, QFileDevice::WriteError);
++ QIORing::setFileErrorResult(*writeRequest, fileError);
++ return ReadWriteStatus::Finished;
++ }
++
++ auto setResult = [writeRequest](qint64 bytesWritten) {
++ auto &writeResult = [&writeRequest]() -> QIORingResult<Op> & {
++ if (auto *result = std::get_if<QIORingResult<Op>>(&writeRequest->result))
++ return *result;
++ return writeRequest->result.template emplace<QIORingResult<Op>>();
++ }();
++ writeResult.bytesWritten += bytesWritten;
++ return writeResult.bytesWritten;
++ };
++ auto *sources = [&writeRequest]() {
++ if constexpr (Op == Operation::Write)
++ return &writeRequest->source;
++ else
++ return &writeRequest->sources[0];
++ }();
++ QIORing::ReadWriteStatus rwstatus = handleWriteCompletion(value, sources,
++ request->getExtra<void>(), setResult);
++ finalizeReadWriteCompletion(request, rwstatus);
++ return rwstatus;
++ }
++
++ namespace QtPrivate {
++ // The 'extra' struct for Read/Write operations that must be split up
++ struct ReadWriteExtra
++ {
++ qint64 totalProcessed = 0;
++ qsizetype spanIndex = 0;
++ qsizetype spanOffset = 0;
++ qsizetype numSpans = 1;
++ };
++ } // namespace QtPrivate
++
++ QT_END_NAMESPACE
++
++ #endif // IOPROCESSOR_P_H