Revision control
Copy as Markdown
// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
// Qt-Security score:critical reason:network-protocol
//#define QNETWORKACCESSHTTPBACKEND_DEBUG
#include "qnetworkreplyhttpimpl_p.h"
#include "qnetworkaccessmanager_p.h"
#include "qnetworkaccesscache_p.h"
#include "qabstractnetworkcache.h"
#include "qnetworkrequest.h"
#include "qnetworkreply.h"
#include "qnetworkrequest_p.h"
#include "qnetworkcookie.h"
#include "qnetworkcookie_p.h"
#include "QtCore/qdatetime.h"
#include "QtCore/qelapsedtimer.h"
#include "QtNetwork/qsslconfiguration.h"
#include "qhttpthreaddelegate_p.h"
#include "qhsts_p.h"
#include "qthread.h"
#include "QtCore/qcoreapplication.h"
#include <QtCore/private/qthread_p.h>
#include <QtCore/private/qtools_p.h>
#include "qnetworkcookiejar.h"
#include "qnetworkreplyimpl_p.h"
#include <string.h> // for strchr
QT_BEGIN_NAMESPACE
using namespace Qt::StringLiterals;
using namespace QtMiscUtils;
using namespace std::chrono_literals;
class QNetworkProxy;
static inline QByteArray rangeName() { return "Range"_ba; }
static inline QByteArray cacheControlName() { return "Cache-Control"_ba; }
static constexpr QByteArrayView bytesEqualPrefix() noexcept { return "bytes="; }
// ### merge with nextField in cookiejar.cpp
static QHash<QByteArray, QByteArray> parseHttpOptionHeader(QByteArrayView header)
{
// The HTTP header is of the form:
// header = #1(directives)
// directives = token | value-directive
// value-directive = token "=" (token | quoted-string)
QHash<QByteArray, QByteArray> result;
qsizetype pos = 0;
while (true) {
// skip spaces
pos = nextNonWhitespace(header, pos);
if (pos == header.size())
return result; // end of parsing
// pos points to a non-whitespace
const qsizetype comma = header.indexOf(',', pos);
const qsizetype equal = header.indexOf('=', pos);
if (comma == pos || equal == pos)
// huh? Broken header.
return result;
// The key name is delimited by either a comma, an equal sign or the end
// of the header, whichever comes first
qsizetype end = comma;
if (end == -1)
end = header.size();
if (equal != -1 && end > equal)
end = equal; // equal sign comes before comma/end
const auto key = header.sliced(pos, end - pos).trimmed();
pos = end + 1;
if (size_t(equal) < size_t(comma)) {
// case: token "=" (token | quoted-string)
// skip spaces
pos = nextNonWhitespace(header, pos);
if (pos == header.size())
// huh? Broken header
return result;
QByteArray value;
value.reserve(header.size() - pos);
if (header.at(pos) == '"') {
// case: quoted-string
// quoted-string = ( <"> *(qdtext | quoted-pair ) <"> )
// qdtext = <any TEXT except <">>
// quoted-pair = "\" CHAR
++pos;
while (pos < header.size()) {
char c = header.at(pos);
if (c == '"') {
// end of quoted text
break;
} else if (c == '\\') {
++pos;
if (pos >= header.size())
// broken header
return result;
c = header.at(pos);
}
value += c;
++pos;
}
} else {
const auto isSeparator = [](char c) {
static const char separators[] = "()<>@,;:\\\"/[]?={}";
return isLWS(c) || strchr(separators, c) != nullptr;
};
// case: token
while (pos < header.size()) {
char c = header.at(pos);
if (isSeparator(c))
break;
value += c;
++pos;
}
}
result.insert(key.toByteArray().toLower(), value);
// find the comma now:
if (qsizetype comma = header.indexOf(',', pos); comma == -1)
return result; // end of parsing
else
pos = comma + 1;
} else {
// case: token
// key is already set
result.insert(key.toByteArray().toLower(), QByteArray());
}
}
}
// If the user specified CustomOperation we try to remap the operation to a known
// operation. This is useful because we treat the operations differently,
// ie for caching or redirection
static auto remapCustom(QNetworkAccessManager::Operation operation, const QNetworkRequest &req)
{
if (operation == QNetworkAccessManager::CustomOperation) {
const QByteArray customVerb = req.attribute(QNetworkRequest::CustomVerbAttribute)
.toByteArray();
if (customVerb.compare("get", Qt::CaseInsensitive) == 0)
return QNetworkAccessManager::GetOperation;
if (customVerb.compare("head", Qt::CaseInsensitive) == 0)
return QNetworkAccessManager::HeadOperation;
if (customVerb.compare("delete", Qt::CaseInsensitive) == 0)
return QNetworkAccessManager::DeleteOperation;
if (customVerb.compare("put", Qt::CaseInsensitive) == 0)
return QNetworkAccessManager::PutOperation;
if (customVerb.compare("post", Qt::CaseInsensitive) == 0)
return QNetworkAccessManager::PostOperation;
}
return operation;
}
QNetworkReplyHttpImpl::QNetworkReplyHttpImpl(QNetworkAccessManager* const manager,
const QNetworkRequest& request,
QNetworkAccessManager::Operation& operation,
QIODevice* outgoingData)
: QNetworkReply(*new QNetworkReplyHttpImplPrivate, manager)
{
Q_D(QNetworkReplyHttpImpl);
Q_ASSERT(manager);
d->manager = manager;
d->managerPrivate = manager->d_func();
d->request = request;
d->originalRequest = request;
d->operation = remapCustom(operation, request);
d->outgoingData = outgoingData;
d->url = request.url();
#ifndef QT_NO_SSL
if (request.url().scheme() == "https"_L1)
d->sslConfiguration.reset(new QSslConfiguration(request.sslConfiguration()));
#endif
QObjectPrivate::connect(this, &QNetworkReplyHttpImpl::redirectAllowed, d,
&QNetworkReplyHttpImplPrivate::followRedirect, Qt::QueuedConnection);
// FIXME Later maybe set to Unbuffered, especially if it is zerocopy or from cache?
QIODevice::open(QIODevice::ReadOnly);
// Internal code that does a HTTP reply for the synchronous Ajax
// in Qt WebKit.
QVariant synchronousHttpAttribute = request.attribute(
static_cast<QNetworkRequest::Attribute>(QNetworkRequest::SynchronousRequestAttribute));
if (synchronousHttpAttribute.isValid()) {
d->synchronous = synchronousHttpAttribute.toBool();
if (d->synchronous && outgoingData) {
// The synchronous HTTP is a corner case, we will put all upload data in one big QByteArray in the outgoingDataBuffer.
// Yes, this is not the most efficient thing to do, but on the other hand synchronous XHR needs to die anyway.
d->outgoingDataBuffer = std::make_shared<QRingBuffer>();
qint64 previousDataSize = 0;
do {
previousDataSize = d->outgoingDataBuffer->size();
d->outgoingDataBuffer->append(d->outgoingData->readAll());
} while (d->outgoingDataBuffer->size() != previousDataSize);
d->_q_startOperation();
return;
}
}
if (outgoingData) {
// there is data to be uploaded, e.g. HTTP POST.
if (!d->outgoingData->isSequential()) {
// fixed size non-sequential (random-access)
// just start the operation
QMetaObject::invokeMethod(this, "_q_startOperation", Qt::QueuedConnection);
// FIXME make direct call?
} else {
bool bufferingDisallowed =
request.attribute(QNetworkRequest::DoNotBufferUploadDataAttribute,
false).toBool();
if (bufferingDisallowed) {
// if a valid content-length header for the request was supplied, we can disable buffering
// if not, we will buffer anyway
const auto sizeOpt = QNetworkHeadersPrivate::toInt(
request.headers().value(QHttpHeaders::WellKnownHeader::ContentLength));
if (sizeOpt) {
QMetaObject::invokeMethod(this, "_q_startOperation", Qt::QueuedConnection);
// FIXME make direct call?
} else {
d->state = d->Buffering;
QMetaObject::invokeMethod(this, "_q_bufferOutgoingData", Qt::QueuedConnection);
}
} else {
// _q_startOperation will be called when the buffering has finished.
d->state = d->Buffering;
QMetaObject::invokeMethod(this, "_q_bufferOutgoingData", Qt::QueuedConnection);
}
}
} else {
// No outgoing data (POST, ..)
d->_q_startOperation();
}
}
QNetworkReplyHttpImpl::~QNetworkReplyHttpImpl()
{
// This will do nothing if the request was already finished or aborted
emit abortHttpRequest();
}
void QNetworkReplyHttpImpl::close()
{
Q_D(QNetworkReplyHttpImpl);
if (d->state == QNetworkReplyPrivate::Aborted ||
d->state == QNetworkReplyPrivate::Finished)
return;
// According to the documentation close only stops the download
// by closing we can ignore the download part and continue uploading.
QNetworkReply::close();
// call finished which will emit signals
// FIXME shouldn't this be emitted Queued?
d->error(OperationCanceledError, tr("Operation canceled"));
d->finished();
}
void QNetworkReplyHttpImpl::abort()
{
abortImpl(QNetworkReply::OperationCanceledError);
}
void QNetworkReplyHttpImpl::abortImpl(QNetworkReply::NetworkError error)
{
Q_D(QNetworkReplyHttpImpl);
// FIXME
if (d->state == QNetworkReplyPrivate::Finished || d->state == QNetworkReplyPrivate::Aborted)
return;
QNetworkReply::close();
if (d->state != QNetworkReplyPrivate::Finished) {
// call finished which will emit signals
// FIXME shouldn't this be emitted Queued?
d->error(error,
error == TimeoutError ? tr("Operation timed out") : tr("Operation canceled"));
d->finished();
}
d->state = QNetworkReplyPrivate::Aborted;
emit abortHttpRequest();
}
qint64 QNetworkReplyHttpImpl::bytesAvailable() const
{
Q_D(const QNetworkReplyHttpImpl);
// if we load from cache device
if (d->cacheLoadDevice) {
return QNetworkReply::bytesAvailable() + d->cacheLoadDevice->bytesAvailable();
}
// zerocopy buffer
if (d->downloadZerocopyBuffer) {
return QNetworkReply::bytesAvailable() + d->downloadBufferCurrentSize - d->downloadBufferReadPosition;
}
if (d->decompressHelper.isValid()) {
if (d->decompressHelper.isCountingBytes())
return QNetworkReply::bytesAvailable() + d->decompressHelper.uncompressedSize();
if (d->decompressHelper.hasData())
return QNetworkReply::bytesAvailable() + 1;
}
// normal buffer
return QNetworkReply::bytesAvailable();
}
bool QNetworkReplyHttpImpl::isSequential () const
{
// FIXME In the cache of a cached load or the zero-copy buffer we could actually be non-sequential.
// FIXME however this requires us to implement stuff like seek() too.
return true;
}
qint64 QNetworkReplyHttpImpl::size() const
{
// FIXME At some point, this could return a proper value, e.g. if we're non-sequential.
return QNetworkReply::size();
}
qint64 QNetworkReplyHttpImpl::readData(char* data, qint64 maxlen)
{
Q_D(QNetworkReplyHttpImpl);
// cacheload device
if (d->cacheLoadDevice) {
// FIXME bytesdownloaded, position etc?
qint64 ret = d->cacheLoadDevice->read(data, maxlen);
return ret;
}
// zerocopy buffer
if (d->downloadZerocopyBuffer) {
// FIXME bytesdownloaded, position etc?
qint64 howMuch = qMin(maxlen, (d->downloadBufferCurrentSize - d->downloadBufferReadPosition));
memcpy(data, d->downloadZerocopyBuffer + d->downloadBufferReadPosition, howMuch);
d->downloadBufferReadPosition += howMuch;
return howMuch;
}
if (d->decompressHelper.isValid() && (d->decompressHelper.hasData() || !isFinished())) {
if (maxlen == 0 || !d->decompressHelper.hasData())
return 0;
const qint64 bytesRead = d->decompressHelper.read(data, maxlen);
if (!d->decompressHelper.isValid()) {
d->error(QNetworkReplyImpl::NetworkError::UnknownContentError,
QCoreApplication::translate("QHttp", "Decompression failed: %1")
.arg(d->decompressHelper.errorString()));
d->decompressHelper.clear();
return -1;
}
if (d->cacheSaveDevice) {
// Need to write to the cache now that we have the data
d->cacheSaveDevice->write(data, bytesRead);
// ... and if we've read everything then the cache can be closed.
if (isFinished() && !d->decompressHelper.hasData())
d->completeCacheSave();
}
// In case of buffer size restriction we need to emit that it has been emptied
qint64 wasBuffered = d->bytesBuffered;
d->bytesBuffered = 0;
if (readBufferSize())
emit readBufferFreed(wasBuffered);
return bytesRead;
}
// normal buffer
if (d->state == d->Finished || d->state == d->Aborted)
return -1;
qint64 wasBuffered = d->bytesBuffered;
d->bytesBuffered = 0;
if (readBufferSize())
emit readBufferFreed(wasBuffered);
return 0;
}
void QNetworkReplyHttpImpl::setReadBufferSize(qint64 size)
{
QNetworkReply::setReadBufferSize(size);
emit readBufferSizeChanged(size);
return;
}
bool QNetworkReplyHttpImpl::canReadLine () const
{
Q_D(const QNetworkReplyHttpImpl);
if (QNetworkReply::canReadLine())
return true;
if (d->cacheLoadDevice)
return d->cacheLoadDevice->canReadLine();
if (d->downloadZerocopyBuffer)
return memchr(d->downloadZerocopyBuffer + d->downloadBufferReadPosition, '\n', d->downloadBufferCurrentSize - d->downloadBufferReadPosition);
return false;
}
#ifndef QT_NO_SSL
void QNetworkReplyHttpImpl::ignoreSslErrors()
{
Q_D(QNetworkReplyHttpImpl);
Q_ASSERT(d->managerPrivate);
if (d->managerPrivate->stsEnabled && d->managerPrivate->stsCache.isKnownHost(url())) {
// We cannot ignore any Security Transport-related errors for this host.
return;
}
d->pendingIgnoreAllSslErrors = true;
}
void QNetworkReplyHttpImpl::ignoreSslErrorsImplementation(const QList<QSslError> &errors)
{
Q_D(QNetworkReplyHttpImpl);
Q_ASSERT(d->managerPrivate);
if (d->managerPrivate->stsEnabled && d->managerPrivate->stsCache.isKnownHost(url())) {
// We cannot ignore any Security Transport-related errors for this host.
return;
}
// the pending list is set if QNetworkReply::ignoreSslErrors(const QList<QSslError> &errors)
// is called before QNetworkAccessManager::get() (or post(), etc.)
d->pendingIgnoreSslErrorsList = errors;
}
void QNetworkReplyHttpImpl::setSslConfigurationImplementation(const QSslConfiguration &newconfig)
{
// Setting a SSL configuration on a reply is not supported. The user needs to set
// her/his QSslConfiguration on the QNetworkRequest.
Q_UNUSED(newconfig);
}
void QNetworkReplyHttpImpl::sslConfigurationImplementation(QSslConfiguration &configuration) const
{
Q_D(const QNetworkReplyHttpImpl);
if (d->sslConfiguration)
configuration = *d->sslConfiguration;
else
configuration = request().sslConfiguration();
}
#endif
QNetworkReplyHttpImplPrivate::QNetworkReplyHttpImplPrivate()
: QNetworkReplyPrivate()
, manager(nullptr)
, managerPrivate(nullptr)
, synchronous(false)
, state(Idle)
, statusCode(0)
, uploadByteDevicePosition(false)
, uploadDeviceChoking(false)
, outgoingData(nullptr)
, bytesUploaded(-1)
, cacheLoadDevice(nullptr)
, loadingFromCache(false)
, cacheSaveDevice(nullptr)
, cacheEnabled(false)
, resumeOffset(0)
, bytesDownloaded(0)
, bytesBuffered(0)
, transferTimeout(nullptr)
, downloadBufferReadPosition(0)
, downloadBufferCurrentSize(0)
, downloadZerocopyBuffer(nullptr)
, pendingDownloadDataEmissions(std::make_shared<QAtomicInt>())
, pendingDownloadProgressEmissions(std::make_shared<QAtomicInt>())
#ifndef QT_NO_SSL
, pendingIgnoreAllSslErrors(false)
#endif
{
}
QNetworkReplyHttpImplPrivate::~QNetworkReplyHttpImplPrivate()
{
if (cacheSaveDevice)
managerPrivate->networkCache->remove(url);
}
/*
For a given httpRequest
1) If AlwaysNetwork, return
2) If we have a cache entry for this url populate headers so the server can return 304
3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true
*/
bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest)
{
QNetworkRequest::CacheLoadControl CacheLoadControlAttribute =
(QNetworkRequest::CacheLoadControl)request.attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt();
auto requestHeaders = request.headers();
if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) {
// If the request does not already specify preferred cache-control
// force reload from the network and tell any caching proxy servers to reload too
if (!requestHeaders.contains(QHttpHeaders::WellKnownHeader::CacheControl)) {
const auto noCache = "no-cache"_ba;
httpRequest.setHeaderField(cacheControlName(), noCache);
httpRequest.setHeaderField("Pragma"_ba, noCache);
}
return false;
}
// The disk cache API does not currently support partial content retrieval.
// That is why we don't use the disk cache for any such requests.
if (requestHeaders.contains(QHttpHeaders::WellKnownHeader::Range))
return false;
QAbstractNetworkCache *nc = managerPrivate->networkCache;
if (!nc)
return false; // no local cache
QNetworkCacheMetaData metaData = nc->metaData(httpRequest.url());
if (!metaData.isValid())
return false; // not in cache
if (!metaData.saveToDisk())
return false;
QHttpHeaders cacheHeaders = metaData.headers();
const auto sizeOpt = QNetworkHeadersPrivate::toInt(
cacheHeaders.value(QHttpHeaders::WellKnownHeader::ContentLength));
if (sizeOpt) {
std::unique_ptr<QIODevice> data(nc->data(httpRequest.url()));
if (!data || data->size() < sizeOpt.value())
return false; // The data is smaller than the content-length specified
}
auto value = cacheHeaders.value(QHttpHeaders::WellKnownHeader::ETag);
if (!value.empty())
httpRequest.setHeaderField("If-None-Match"_ba, value.toByteArray());
QDateTime lastModified = metaData.lastModified();
if (lastModified.isValid())
httpRequest.setHeaderField("If-Modified-Since"_ba, QNetworkHeadersPrivate::toHttpDate(lastModified));
value = cacheHeaders.value(QHttpHeaders::WellKnownHeader::CacheControl);
if (!value.empty()) {
QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(value);
if (cacheControl.contains("no-cache"_ba))
return false;
}
QDateTime currentDateTime = QDateTime::currentDateTimeUtc();
QDateTime expirationDate = metaData.expirationDate();
bool response_is_fresh;
if (!expirationDate.isValid()) {
/*
* age_value
* is the value of Age: header received by the cache with
* this response.
* date_value
* is the value of the origin server's Date: header
* request_time
* is the (local) time when the cache made the request
* that resulted in this cached response
* response_time
* is the (local) time when the cache received the
* response
* now
* is the current (local) time
*/
const auto ageOpt = QNetworkHeadersPrivate::toInt(
cacheHeaders.value(QHttpHeaders::WellKnownHeader::Age));
const qint64 age_value = ageOpt.value_or(0);
QDateTime dateHeader;
qint64 date_value = 0;
value = cacheHeaders.value(QHttpHeaders::WellKnownHeader::Date);
if (!value.empty()) {
dateHeader = QNetworkHeadersPrivate::fromHttpDate(value);
date_value = dateHeader.toSecsSinceEpoch();
}
qint64 now = currentDateTime.toSecsSinceEpoch();
qint64 request_time = now;
qint64 response_time = now;
// Algorithm from RFC 2616 section 13.2.3
qint64 apparent_age = qMax<qint64>(0, response_time - date_value);
qint64 corrected_received_age = qMax(apparent_age, age_value);
qint64 response_delay = response_time - request_time;
qint64 corrected_initial_age = corrected_received_age + response_delay;
qint64 resident_time = now - response_time;
qint64 current_age = corrected_initial_age + resident_time;
qint64 freshness_lifetime = 0;
// RFC 2616 13.2.4 Expiration Calculations
if (lastModified.isValid() && dateHeader.isValid()) {
qint64 diff = lastModified.secsTo(dateHeader);
freshness_lifetime = diff / 10;
const auto warningHeader = "Warning"_ba;
if (httpRequest.headerField(warningHeader).isEmpty()) {
QDateTime dt = currentDateTime.addSecs(current_age);
if (currentDateTime.daysTo(dt) > 1)
httpRequest.setHeaderField(warningHeader, "113"_ba);
}
}
// the cache-saving code below sets the freshness_lifetime with (dateHeader - last_modified) / 10
// if "last-modified" is present, or to Expires otherwise
response_is_fresh = (freshness_lifetime > current_age);
} else {
// expiration date was calculated earlier (e.g. when storing object to the cache)
response_is_fresh = currentDateTime.secsTo(expirationDate) >= 0;
}
if (!response_is_fresh)
return false;
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "response_is_fresh" << CacheLoadControlAttribute;
#endif
return sendCacheContents(metaData);
}
QHttpNetworkRequest::Priority QNetworkReplyHttpImplPrivate::convert(QNetworkRequest::Priority prio)
{
switch (prio) {
case QNetworkRequest::LowPriority:
return QHttpNetworkRequest::LowPriority;
case QNetworkRequest::HighPriority:
return QHttpNetworkRequest::HighPriority;
case QNetworkRequest::NormalPriority:
return QHttpNetworkRequest::NormalPriority;
}
Q_UNREACHABLE_RETURN(QHttpNetworkRequest::NormalPriority);
}
void QNetworkReplyHttpImplPrivate::maybeDropUploadDevice(const QNetworkRequest &newHttpRequest)
{
// Check for 0-length upload device. Following RFC9110, we are discouraged
// from sending "content-length: 0" for methods where a content-length would
// not normally be expected. E.g. get, connect, head, delete
auto contentLength0Allowed = [&]{
switch (operation) {
case QNetworkAccessManager::CustomOperation: {
const QByteArray customVerb = newHttpRequest.attribute(QNetworkRequest::CustomVerbAttribute)
.toByteArray();
if (customVerb.compare("connect", Qt::CaseInsensitive) != 0)
return true; // Trust user => content-length 0 is presumably okay!
// else:
[[fallthrough]];
}
case QNetworkAccessManager::HeadOperation:
case QNetworkAccessManager::GetOperation:
case QNetworkAccessManager::DeleteOperation:
// no content-length 0
return false;
case QNetworkAccessManager::PutOperation:
case QNetworkAccessManager::PostOperation:
case QNetworkAccessManager::UnknownOperation:
// yes content-length 0
return true;
}
Q_UNREACHABLE_RETURN(false);
};
const auto hasEmptyOutgoingPayload = [&]() {
if (!outgoingData)
return false;
if (outgoingDataBuffer)
return outgoingDataBuffer->isEmpty();
return outgoingData->size() == 0;
};
if (Q_UNLIKELY(hasEmptyOutgoingPayload()) && !contentLength0Allowed()) {
outgoingData = nullptr;
outgoingDataBuffer.reset();
}
}
void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpRequest)
{
Q_Q(QNetworkReplyHttpImpl);
QThread *thread = nullptr;
if (synchronous) {
// A synchronous HTTP request uses its own thread
thread = new QThread();
thread->setObjectName(QStringLiteral("Qt HTTP synchronous thread"));
QObject::connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater()));
thread->start();
} else {
// We use the manager-global thread.
// At some point we could switch to having multiple threads if it makes sense.
thread = managerPrivate->createThread();
}
QUrl url = newHttpRequest.url();
httpRequest.setUrl(url);
httpRequest.setRedirectCount(newHttpRequest.maximumRedirectsAllowed());
QString scheme = url.scheme();
bool ssl = (scheme == "https"_L1 || scheme == "preconnect-https"_L1);
q->setAttribute(QNetworkRequest::ConnectionEncryptedAttribute, ssl);
httpRequest.setSsl(ssl);
bool preConnect = (scheme == "preconnect-http"_L1 || scheme == "preconnect-https"_L1);
httpRequest.setPreConnect(preConnect);
#ifndef QT_NO_NETWORKPROXY
QNetworkProxy transparentProxy, cacheProxy;
// FIXME the proxy stuff should be done in the HTTP thread
const auto proxies = managerPrivate->queryProxy(QNetworkProxyQuery(newHttpRequest.url()));
for (const QNetworkProxy &p : proxies) {
// use the first proxy that works
// for non-encrypted connections, any transparent or HTTP proxy
// for encrypted, only transparent proxies
if (!ssl
&& (p.capabilities() & QNetworkProxy::CachingCapability)
&& (p.type() == QNetworkProxy::HttpProxy ||
p.type() == QNetworkProxy::HttpCachingProxy)) {
cacheProxy = p;
transparentProxy = QNetworkProxy::NoProxy;
break;
}
if (p.isTransparentProxy()) {
transparentProxy = p;
cacheProxy = QNetworkProxy::NoProxy;
break;
}
}
// check if at least one of the proxies
if (transparentProxy.type() == QNetworkProxy::DefaultProxy &&
cacheProxy.type() == QNetworkProxy::DefaultProxy) {
// unsuitable proxies
QMetaObject::invokeMethod(q, "_q_error", synchronous ? Qt::DirectConnection : Qt::QueuedConnection,
Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ProxyNotFoundError),
Q_ARG(QString, QNetworkReplyHttpImpl::tr("No suitable proxy found")));
QMetaObject::invokeMethod(q, "_q_finished", synchronous ? Qt::DirectConnection : Qt::QueuedConnection);
return;
}
#endif
auto redirectPolicy = QNetworkRequest::NoLessSafeRedirectPolicy;
const QVariant value = newHttpRequest.attribute(QNetworkRequest::RedirectPolicyAttribute);
if (value.isValid())
redirectPolicy = qvariant_cast<QNetworkRequest::RedirectPolicy>(value);
httpRequest.setRedirectPolicy(redirectPolicy);
// If, for some reason, it turns out we won't use the upload device we drop
// it in the following call:
maybeDropUploadDevice(newHttpRequest);
httpRequest.setPriority(convert(newHttpRequest.priority()));
loadingFromCache = false;
switch (operation) {
case QNetworkAccessManager::GetOperation:
httpRequest.setOperation(QHttpNetworkRequest::Get);
// If the request has a body, createUploadByteDevice() and don't use caching
if (outgoingData) {
invalidateCache();
createUploadByteDevice();
} else if (loadFromCacheIfAllowed(httpRequest)) {
return; // no need to send the request! :)
}
break;
case QNetworkAccessManager::HeadOperation:
httpRequest.setOperation(QHttpNetworkRequest::Head);
if (loadFromCacheIfAllowed(httpRequest))
return; // no need to send the request! :)
break;
case QNetworkAccessManager::PostOperation:
invalidateCache();
httpRequest.setOperation(QHttpNetworkRequest::Post);
createUploadByteDevice();
break;
case QNetworkAccessManager::PutOperation:
invalidateCache();
httpRequest.setOperation(QHttpNetworkRequest::Put);
createUploadByteDevice();
break;
case QNetworkAccessManager::DeleteOperation:
invalidateCache();
httpRequest.setOperation(QHttpNetworkRequest::Delete);
createUploadByteDevice();
break;
case QNetworkAccessManager::CustomOperation:
invalidateCache(); // for safety reasons, we don't know what the operation does
httpRequest.setOperation(QHttpNetworkRequest::Custom);
createUploadByteDevice();
httpRequest.setCustomVerb(newHttpRequest.attribute(
QNetworkRequest::CustomVerbAttribute).toByteArray());
break;
default:
break; // can't happen
}
QHttpHeaders newRequestHeaders = newHttpRequest.headers();
if (resumeOffset != 0) {
if (newRequestHeaders.contains(QHttpHeaders::WellKnownHeader::Range)) {
// Need to adjust resume offset for user specified range
// We've already verified that requestRange starts with "bytes=", see canResume.
const auto rangeHeader = newRequestHeaders.value(QHttpHeaders::WellKnownHeader::Range);
const auto requestRange = QByteArrayView(rangeHeader).mid(bytesEqualPrefix().size());
newRequestHeaders.removeAll(QHttpHeaders::WellKnownHeader::Range);
int index = requestRange.indexOf('-');
quint64 requestStartOffset = requestRange.left(index).toULongLong();
quint64 requestEndOffset = requestRange.mid(index + 1).toULongLong();
// In case an end offset is not given it is skipped from the request range
QByteArray newRange = bytesEqualPrefix() + QByteArray::number(resumeOffset + requestStartOffset) +
'-' + (requestEndOffset ? QByteArray::number(requestEndOffset) : QByteArray());
httpRequest.setHeaderField(rangeName(), newRange);
} else {
httpRequest.setHeaderField(rangeName(), bytesEqualPrefix() + QByteArray::number(resumeOffset) + '-');
}
}
for (int i = 0; i < newRequestHeaders.size(); i++) {
const auto name = newRequestHeaders.nameAt(i);
const auto value = newRequestHeaders.valueAt(i);
httpRequest.setHeaderField(QByteArray(name.data(), name.size()), value.toByteArray());
}
if (newHttpRequest.attribute(QNetworkRequest::HttpPipeliningAllowedAttribute).toBool())
httpRequest.setPipeliningAllowed(true);
if (auto allowed = request.attribute(QNetworkRequest::Http2AllowedAttribute);
allowed.isValid() && allowed.canConvert<bool>()) {
httpRequest.setHTTP2Allowed(allowed.value<bool>());
}
auto h2cAttribute = request.attribute(QNetworkRequest::Http2CleartextAllowedAttribute);
// ### Qt7: Stop checking the environment variable
if (h2cAttribute.toBool()
|| (!h2cAttribute.isValid() && qEnvironmentVariableIsSet("QT_NETWORK_H2C_ALLOWED"))) {
httpRequest.setH2cAllowed(true);
}
if (request.attribute(QNetworkRequest::Http2DirectAttribute).toBool()) {
// Intentionally mutually exclusive - cannot be both direct and 'allowed'
httpRequest.setHTTP2Direct(true);
httpRequest.setHTTP2Allowed(false);
}
if (static_cast<QNetworkRequest::LoadControl>
(newHttpRequest.attribute(QNetworkRequest::AuthenticationReuseAttribute,
QNetworkRequest::Automatic).toInt()) == QNetworkRequest::Manual)
httpRequest.setWithCredentials(false);
if (request.attribute(QNetworkRequest::EmitAllUploadProgressSignalsAttribute).toBool())
emitAllUploadProgressSignals = true;
httpRequest.setPeerVerifyName(newHttpRequest.peerVerifyName());
if (scheme.startsWith(("unix"_L1))) {
if (QVariant path = newHttpRequest.attribute(QNetworkRequest::FullLocalServerNameAttribute);
path.isValid() && path.canConvert<QString>()) {
httpRequest.setFullLocalServerName(path.toString());
}
}
// Create the HTTP thread delegate
QHttpThreadDelegate *delegate = new QHttpThreadDelegate;
// Propagate Http/2 settings:
delegate->http2Parameters = request.http2Configuration();
delegate->http1Parameters = request.http1Configuration();
delegate->tcpKeepAliveParameters.idleTimeBeforeProbes = request.tcpKeepAliveIdleTimeBeforeProbes();
delegate->tcpKeepAliveParameters.intervalBetweenProbes = request.tcpKeepAliveIntervalBetweenProbes();
delegate->tcpKeepAliveParameters.probeCount = request.tcpKeepAliveProbeCount();
if (request.attribute(QNetworkRequest::ConnectionCacheExpiryTimeoutSecondsAttribute).isValid())
delegate->connectionCacheExpiryTimeoutSeconds = request.attribute(QNetworkRequest::ConnectionCacheExpiryTimeoutSecondsAttribute).toInt();
// For the synchronous HTTP, this is the normal way the delegate gets deleted
// For the asynchronous HTTP this is a safety measure, the delegate deletes itself when HTTP is finished
QMetaObject::Connection threadFinishedConnection =
QObject::connect(thread, SIGNAL(finished()), delegate, SLOT(deleteLater()));
// QTBUG-88063: When 'delegate' is deleted the connection will be added to 'thread''s orphaned
// connections list. This orphaned list will be cleaned up next time 'thread' emits a signal,
// unfortunately that's the finished signal. It leads to a soft-leak so we do this to disconnect
// it on deletion so that it cleans up the orphan immediately.
QObject::connect(delegate, &QObject::destroyed,
delegate, [threadFinishedConnection] () mutable {
if (bool(threadFinishedConnection))
QObject::disconnect(threadFinishedConnection);
});
// Set the properties it needs
delegate->httpRequest = httpRequest;
#ifndef QT_NO_NETWORKPROXY
delegate->cacheProxy = cacheProxy;
delegate->transparentProxy = transparentProxy;
#endif
delegate->ssl = ssl;
#ifndef QT_NO_SSL
if (ssl)
delegate->incomingSslConfiguration.reset(new QSslConfiguration(newHttpRequest.sslConfiguration()));
#endif
// Do we use synchronous HTTP?
delegate->synchronous = synchronous;
// The authentication manager is used to avoid the BlockingQueuedConnection communication
// from HTTP thread to user thread in some cases.
delegate->authenticationManager = managerPrivate->authenticationManager;
if (!synchronous) {
// Tell our zerocopy policy to the delegate
QVariant downloadBufferMaximumSizeAttribute = newHttpRequest.attribute(QNetworkRequest::MaximumDownloadBufferSizeAttribute);
if (downloadBufferMaximumSizeAttribute.isValid()) {
delegate->downloadBufferMaximumSize = downloadBufferMaximumSizeAttribute.toLongLong();
} else {
// If there is no MaximumDownloadBufferSizeAttribute set (which is for the majority
// of QNetworkRequest) then we can assume we'll do it anyway for small HTTP replies.
// This helps with performance and memory fragmentation.
delegate->downloadBufferMaximumSize = 128*1024;
}
// These atomic integers are used for signal compression
delegate->pendingDownloadData = pendingDownloadDataEmissions;
delegate->pendingDownloadProgress = pendingDownloadProgressEmissions;
// Connect the signals of the delegate to us
QObject::connect(delegate, SIGNAL(downloadData(QByteArray)),
q, SLOT(replyDownloadData(QByteArray)),
Qt::QueuedConnection);
QObject::connect(delegate, SIGNAL(downloadFinished()),
q, SLOT(replyFinished()),
Qt::QueuedConnection);
QObject::connect(delegate, &QHttpThreadDelegate::socketStartedConnecting,
q, &QNetworkReply::socketStartedConnecting, Qt::QueuedConnection);
QObject::connect(delegate, &QHttpThreadDelegate::requestSent,
q, &QNetworkReply::requestSent, Qt::QueuedConnection);
connect(delegate, &QHttpThreadDelegate::downloadMetaData, this,
&QNetworkReplyHttpImplPrivate::replyDownloadMetaData, Qt::QueuedConnection);
QObject::connect(delegate, SIGNAL(downloadProgress(qint64,qint64)),
q, SLOT(replyDownloadProgressSlot(qint64,qint64)),
Qt::QueuedConnection);
QObject::connect(delegate, SIGNAL(error(QNetworkReply::NetworkError,QString)),
q, SLOT(httpError(QNetworkReply::NetworkError,QString)),
Qt::QueuedConnection);
QObject::connect(delegate, SIGNAL(redirected(QUrl,int,int)),
q, SLOT(onRedirected(QUrl,int,int)),
Qt::QueuedConnection);
#ifndef QT_NO_SSL
QObject::connect(delegate, SIGNAL(sslConfigurationChanged(QSslConfiguration)),
q, SLOT(replySslConfigurationChanged(QSslConfiguration)),
Qt::QueuedConnection);
#endif
// Those need to report back, therefore BlockingQueuedConnection
QObject::connect(delegate, SIGNAL(authenticationRequired(QHttpNetworkRequest,QAuthenticator*)),
q, SLOT(httpAuthenticationRequired(QHttpNetworkRequest,QAuthenticator*)),
Qt::BlockingQueuedConnection);
#ifndef QT_NO_NETWORKPROXY
QObject::connect(delegate, SIGNAL(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)),
q, SLOT(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)),
Qt::BlockingQueuedConnection);
#endif
#ifndef QT_NO_SSL
QObject::connect(delegate, SIGNAL(encrypted()), q, SLOT(replyEncrypted()),
Qt::BlockingQueuedConnection);
QObject::connect(delegate, SIGNAL(sslErrors(QList<QSslError>,bool*,QList<QSslError>*)),
q, SLOT(replySslErrors(QList<QSslError>,bool*,QList<QSslError>*)),
Qt::BlockingQueuedConnection);
QObject::connect(delegate, SIGNAL(preSharedKeyAuthenticationRequired(QSslPreSharedKeyAuthenticator*)),
q, SLOT(replyPreSharedKeyAuthenticationRequiredSlot(QSslPreSharedKeyAuthenticator*)),
Qt::BlockingQueuedConnection);
#endif
// This signal we will use to start the request.
QObject::connect(q, SIGNAL(startHttpRequest()), delegate, SLOT(startRequest()));
QObject::connect(q, SIGNAL(abortHttpRequest()), delegate, SLOT(abortRequest()));
// To throttle the connection.
QObject::connect(q, SIGNAL(readBufferSizeChanged(qint64)), delegate, SLOT(readBufferSizeChanged(qint64)));
QObject::connect(q, SIGNAL(readBufferFreed(qint64)), delegate, SLOT(readBufferFreed(qint64)));
if (uploadByteDevice) {
QNonContiguousByteDeviceThreadForwardImpl *forwardUploadDevice =
new QNonContiguousByteDeviceThreadForwardImpl(uploadByteDevice->atEnd(), uploadByteDevice->size());
forwardUploadDevice->setParent(delegate); // needed to make sure it is moved on moveToThread()
delegate->httpRequest.setUploadByteDevice(forwardUploadDevice);
// If the device in the user thread claims it has more data, keep the flow to HTTP thread going
QObject::connect(uploadByteDevice.get(), SIGNAL(readyRead()),
q, SLOT(uploadByteDeviceReadyReadSlot()),
Qt::QueuedConnection);
// From user thread to http thread:
QObject::connect(q, SIGNAL(haveUploadData(qint64,QByteArray,bool,qint64)),
forwardUploadDevice, SLOT(haveDataSlot(qint64,QByteArray,bool,qint64)), Qt::QueuedConnection);
QObject::connect(uploadByteDevice.get(), SIGNAL(readyRead()),
forwardUploadDevice, SIGNAL(readyRead()),
Qt::QueuedConnection);
// From http thread to user thread:
QObject::connect(forwardUploadDevice, SIGNAL(wantData(qint64)),
q, SLOT(wantUploadDataSlot(qint64)));
QObject::connect(forwardUploadDevice,SIGNAL(processedData(qint64,qint64)),
q, SLOT(sentUploadDataSlot(qint64,qint64)));
QObject::connect(forwardUploadDevice, SIGNAL(resetData(bool*)),
q, SLOT(resetUploadDataSlot(bool*)),
Qt::BlockingQueuedConnection); // this is the only one with BlockingQueued!
}
} else if (synchronous) {
QObject::connect(q, SIGNAL(startHttpRequestSynchronously()), delegate, SLOT(startRequestSynchronously()), Qt::BlockingQueuedConnection);
if (uploadByteDevice) {
// For the synchronous HTTP use case the use thread (this one here) is blocked
// so we cannot use the asynchronous upload architecture.
// We therefore won't use the QNonContiguousByteDeviceThreadForwardImpl but directly
// use the uploadByteDevice provided to us by the QNetworkReplyImpl.
// The code that is in start() makes sure it is safe to use from a thread
// since it only wraps a QRingBuffer
delegate->httpRequest.setUploadByteDevice(uploadByteDevice.get());
}
}
// Move the delegate to the http thread
delegate->moveToThread(thread);
// This call automatically moves the uploadDevice too for the asynchronous case.
// Prepare timers for progress notifications
downloadProgressSignalChoke.start();
uploadProgressSignalChoke.invalidate();
// Send an signal to the delegate so it starts working in the other thread
if (synchronous) {
emit q->startHttpRequestSynchronously(); // This one is BlockingQueuedConnection, so it will return when all work is done
replyDownloadMetaData
(delegate->incomingHeaders,
delegate->incomingStatusCode,
delegate->incomingReasonPhrase,
delegate->isPipeliningUsed,
QSharedPointer<char>(),
delegate->incomingContentLength,
delegate->removedContentLength,
delegate->isHttp2Used,
delegate->isCompressed);
replyDownloadData(delegate->synchronousDownloadData);
if (delegate->incomingErrorCode != QNetworkReply::NoError)
httpError(delegate->incomingErrorCode, delegate->incomingErrorDetail);
thread->quit();
thread->wait(QDeadlineTimer(5000));
if (thread->isFinished())
delete thread;
else
QObject::connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater()));
finished();
} else {
emit q->startHttpRequest(); // Signal to the HTTP thread and go back to user.
}
}
void QNetworkReplyHttpImplPrivate::invalidateCache()
{
QAbstractNetworkCache *nc = managerPrivate->networkCache;
if (nc)
nc->remove(httpRequest.url());
}
void QNetworkReplyHttpImplPrivate::initCacheSaveDevice()
{
Q_Q(QNetworkReplyHttpImpl);
// The disk cache does not support partial content, so don't even try to
// save any such content into the cache.
if (q->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() == 206) {
cacheEnabled = false;
return;
}
// save the meta data
QNetworkCacheMetaData metaData;
metaData.setUrl(url);
metaData = fetchCacheMetaData(metaData);
// save the redirect request also in the cache
QVariant redirectionTarget = q->attribute(QNetworkRequest::RedirectionTargetAttribute);
if (redirectionTarget.isValid()) {
QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes();
attributes.insert(QNetworkRequest::RedirectionTargetAttribute, redirectionTarget);
metaData.setAttributes(attributes);
}
cacheSaveDevice = managerPrivate->networkCache->prepare(metaData);
if (cacheSaveDevice)
q->connect(cacheSaveDevice, SIGNAL(aboutToClose()), SLOT(_q_cacheSaveDeviceAboutToClose()));
if (!cacheSaveDevice || !cacheSaveDevice->isOpen()) {
if (Q_UNLIKELY(cacheSaveDevice && !cacheSaveDevice->isOpen()))
qCritical("QNetworkReplyImpl: network cache returned a device that is not open -- "
"class %s probably needs to be fixed",
managerPrivate->networkCache->metaObject()->className());
managerPrivate->networkCache->remove(url);
cacheSaveDevice = nullptr;
cacheEnabled = false;
}
}
void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d)
{
Q_Q(QNetworkReplyHttpImpl);
// If we're closed just ignore this data
if (!q->isOpen())
return;
// cache this, we need it later and it's invalidated when dealing with compressed data
auto dataSize = d.size();
if (cacheEnabled && isCachingAllowed() && !cacheSaveDevice)
initCacheSaveDevice();
if (decompressHelper.isValid()) {
qint64 uncompressedBefore = -1;
if (decompressHelper.isCountingBytes())
uncompressedBefore = decompressHelper.uncompressedSize();
decompressHelper.feed(std::move(d));
if (!decompressHelper.isValid()) {
error(QNetworkReplyImpl::NetworkError::UnknownContentError,
QCoreApplication::translate("QHttp", "Decompression failed: %1")
.arg(decompressHelper.errorString()));
decompressHelper.clear();
return;
}
if (!isHttpRedirectResponse()) {
if (decompressHelper.isCountingBytes())
bytesDownloaded += (decompressHelper.uncompressedSize() - uncompressedBefore);
setupTransferTimeout();
}
if (synchronous) {
d = QByteArray();
const qsizetype increments = 16 * 1024;
qint64 bytesRead = 0;
while (decompressHelper.hasData()) {
quint64 nextSize = quint64(d.size()) + quint64(increments);
if (nextSize > quint64(std::numeric_limits<QByteArray::size_type>::max())) {
error(QNetworkReplyImpl::NetworkError::UnknownContentError,
QCoreApplication::translate("QHttp",
"Data downloaded is too large to store"));
decompressHelper.clear();
return;
}
d.resize(nextSize);
bytesRead += decompressHelper.read(d.data() + bytesRead, increments);
if (!decompressHelper.isValid()) {
error(QNetworkReplyImpl::NetworkError::UnknownContentError,
QCoreApplication::translate("QHttp", "Decompression failed: %1")
.arg(decompressHelper.errorString()));
decompressHelper.clear();
return;
}
}
d.resize(bytesRead);
// we're synchronous so we're not calling this function again; reset the decompressHelper
decompressHelper.clear();
}
}
// This is going to look a little strange. When downloading data while a
// HTTP redirect is happening (and enabled), we write the redirect
// response to the cache. However, we do not append it to our internal
// buffer as that will contain the response data only for the final
// response
// Note: For compressed data this is done in readData()
if (cacheSaveDevice && !decompressHelper.isValid()) {
cacheSaveDevice->write(d);
}
// if decompressHelper is valid then we have compressed data, and this is handled above
if (!decompressHelper.isValid() && !isHttpRedirectResponse()) {
buffer.append(d);
bytesDownloaded += dataSize;
setupTransferTimeout();
}
bytesBuffered += dataSize;
int pendingSignals = pendingDownloadDataEmissions->fetchAndSubAcquire(1) - 1;
if (pendingSignals > 0) {
// Some more signal emissions to this slot are pending.
// Instead of writing the downstream data, we wait
// and do it in the next call we get
// (signal comppression)
return;
}
if (isHttpRedirectResponse())
return;
// This can occur when downloading compressed data as some of the data may be the content
// encoding's header. Don't emit anything for this.
if (lastReadyReadEmittedSize == bytesDownloaded) {
if (readBufferMaxSize)
emit q->readBufferFreed(dataSize);
return;
}
lastReadyReadEmittedSize = bytesDownloaded;
const auto totalSizeOpt = QNetworkHeadersPrivate::toInt(
headers().value(QHttpHeaders::WellKnownHeader::ContentLength));
emit q->readyRead();
// emit readyRead before downloadProgress in case this will cause events to be
// processed and we get into a recursive call (as in QProgressDialog).
if (downloadProgressSignalChoke.isValid() &&
downloadProgressSignalChoke.elapsed() >= progressSignalInterval
&& (!decompressHelper.isValid() || decompressHelper.isCountingBytes())) {
downloadProgressSignalChoke.start();
emit q->downloadProgress(bytesDownloaded, totalSizeOpt.value_or(-1));
}
}
void QNetworkReplyHttpImplPrivate::replyFinished()
{
// We are already loading from cache, we still however
// got this signal because it was posted already
if (loadingFromCache)
return;
finished();
}
QNetworkAccessManager::Operation QNetworkReplyHttpImplPrivate::getRedirectOperation(QNetworkAccessManager::Operation currentOp, int httpStatus)
{
// HTTP status code can be used to decide if we can redirect with a GET
// more details
// We MUST keep using the verb that was used originally when being redirected with 307 or 308.
if (httpStatus == 307 || httpStatus == 308)
return currentOp;
switch (currentOp) {
case QNetworkAccessManager::HeadOperation:
return QNetworkAccessManager::HeadOperation;
default:
break;
}
// Use GET for everything else.
return QNetworkAccessManager::GetOperation;
}
bool QNetworkReplyHttpImplPrivate::isHttpRedirectResponse() const
{
return httpRequest.isFollowRedirects() && QHttpNetworkReply::isHttpRedirect(statusCode);
}
QNetworkRequest QNetworkReplyHttpImplPrivate::createRedirectRequest(const QNetworkRequest &originalRequest,
const QUrl &url,
int maxRedirectsRemaining)
{
QNetworkRequest newRequest(originalRequest);
newRequest.setUrl(url);
newRequest.setMaximumRedirectsAllowed(maxRedirectsRemaining);
return newRequest;
}
void QNetworkReplyHttpImplPrivate::onRedirected(const QUrl &redirectUrl, int httpStatus, int maxRedirectsRemaining)
{
Q_Q(QNetworkReplyHttpImpl);
Q_ASSERT(manager);
Q_ASSERT(managerPrivate);
if (isFinished)
return;
const QString schemeBefore(url.scheme());
if (httpRequest.isFollowRedirects()) // update the reply's url as it could've changed
url = redirectUrl;
const bool wasLocalSocket = schemeBefore.startsWith("unix"_L1);
if (!wasLocalSocket && managerPrivate->stsEnabled && managerPrivate->stsCache.isKnownHost(url)) {
// RFC6797, 8.3:
// The UA MUST replace the URI scheme with "https" [RFC2818],
// and if the URI contains an explicit port component of "80",
// then the UA MUST convert the port component to be "443", or
// if the URI contains an explicit port component that is not
// equal to "80", the port component value MUST be preserved;
// otherwise, if the URI does not contain an explicit port
// component, the UA MUST NOT add one.
url.setScheme("https"_L1);
if (url.port() == 80)
url.setPort(443);
}
// Just to be on the safe side for local sockets, any changes to the scheme
// are considered less safe
const bool changingLocalScheme = wasLocalSocket && url.scheme() != schemeBefore;
const bool isLessSafe = changingLocalScheme
|| (schemeBefore == "https"_L1 && url.scheme() == "http"_L1);
if (httpRequest.redirectPolicy() == QNetworkRequest::NoLessSafeRedirectPolicy && isLessSafe) {
error(QNetworkReply::InsecureRedirectError,
QCoreApplication::translate("QHttp", "Insecure redirect"));
return;
}
// If the original operation was a GET with a body and the status code is
// 308 then keep the message body. The body must still be present though:
// a previous redirect (e.g. 301/302/303) has already dropped it, and once
// dropped it cannot be restored, so the headers must be dropped too to keep
// them in sync (QTBUG-133976).
const bool getOperationKeepsBody = (operation == QNetworkAccessManager::GetOperation)
&& httpStatus == 308
&& (outgoingData || outgoingDataBuffer);
redirectRequest = createRedirectRequest(originalRequest, url, maxRedirectsRemaining);
operation = getRedirectOperation(operation, httpStatus);
// Clear stale headers, the relevant ones get set again later
httpRequest.clearHeaders();
auto newHeaders = redirectRequest.headers();
if ((operation == QNetworkAccessManager::GetOperation
|| operation == QNetworkAccessManager::HeadOperation) && !getOperationKeepsBody) {
// possibly changed from not-GET/HEAD to GET/HEAD, make sure to get rid of upload device
uploadByteDevice.reset();
uploadByteDevicePosition = 0;
if (outgoingData) {
QObject::disconnect(outgoingData, SIGNAL(readyRead()), q,
SLOT(_q_bufferOutgoingData()));
QObject::disconnect(outgoingData, SIGNAL(readChannelFinished()), q,
SLOT(_q_bufferOutgoingDataFinished()));
}
outgoingData = nullptr;
outgoingDataBuffer.reset();
// We need to explicitly unset these headers so they're not reapplied to the httpRequest
newHeaders.removeAll(QHttpHeaders::WellKnownHeader::ContentLength);
newHeaders.removeAll(QHttpHeaders::WellKnownHeader::ContentType);
}
if (const QNetworkCookieJar *const cookieJar = manager->cookieJar()) {
auto cookies = cookieJar->cookiesForUrl(url);
if (!cookies.empty()) {
auto cookieHeader = QNetworkHeadersPrivate::fromCookieList(cookies);
newHeaders.replaceOrAppend(QHttpHeaders::WellKnownHeader::Cookie, cookieHeader);
}
}
redirectRequest.setHeaders(std::move(newHeaders));
if (httpRequest.redirectPolicy() != QNetworkRequest::UserVerifiedRedirectPolicy)
followRedirect();
emit q->redirected(url);
}
void QNetworkReplyHttpImplPrivate::followRedirect()
{
Q_Q(QNetworkReplyHttpImpl);
Q_ASSERT(managerPrivate);
decompressHelper.clear();
clearHeaders();
if (managerPrivate->thread)
managerPrivate->thread->disconnect();
QMetaObject::invokeMethod(
q,
[this] {
postRequest(redirectRequest);
setupTransferTimeout();
},
Qt::QueuedConnection);
}
static constexpr QLatin1StringView locationHeader() noexcept { return "location"_L1; }
void QNetworkReplyHttpImplPrivate::checkForRedirect(const int statusCode)
{
Q_Q(QNetworkReplyHttpImpl);
switch (statusCode) {
case 301: // Moved Permanently
case 302: // Found
case 303: // See Other
case 307: // Temporary Redirect
case 308: // Permanent Redirect
// What do we do about the caching of the HTML note?
// The response to a 303 MUST NOT be cached, while the response to
// all of the others is cacheable if the headers indicate it to be
QByteArrayView header = q->headers().value(locationHeader());
QUrl url = QUrl(QString::fromUtf8(header));
if (!url.isValid())
url = QUrl(QLatin1StringView(header));
q->setAttribute(QNetworkRequest::RedirectionTargetAttribute, url);
}
}
void QNetworkReplyHttpImplPrivate::replyDownloadMetaData(const QHttpHeaders &hm,
int sc, const QString &rp, bool pu,
QSharedPointer<char> db,
qint64 contentLength,
qint64 removedContentLength,
bool h2Used, bool isCompressed)
{
Q_Q(QNetworkReplyHttpImpl);
Q_UNUSED(contentLength);
statusCode = sc;
reasonPhrase = rp;
#ifndef QT_NO_SSL
// We parse this header only if we're using secure transport:
//
// RFC6797, 8.1
// If an HTTP response is received over insecure transport, the UA MUST
// ignore any present STS header field(s).
if (url.scheme() == "https"_L1 && managerPrivate->stsEnabled)
managerPrivate->stsCache.updateFromHeaders(hm, url);
#endif
// Download buffer
if (!db.isNull()) {
downloadBufferPointer = db;
downloadZerocopyBuffer = downloadBufferPointer.data();
downloadBufferCurrentSize = 0;
q->setAttribute(QNetworkRequest::DownloadBufferAttribute, QVariant::fromValue<QSharedPointer<char> > (downloadBufferPointer));
}
q->setAttribute(QNetworkRequest::HttpPipeliningWasUsedAttribute, pu);
q->setAttribute(QNetworkRequest::Http2WasUsedAttribute, h2Used);
// A user having manually defined which encodings they accept is, for
// somwehat unknown (presumed legacy compatibility) reasons treated as
// disabling our decompression:
const bool autoDecompress = !request.headers().contains(QHttpHeaders::WellKnownHeader::AcceptEncoding);
const bool shouldDecompress = isCompressed && autoDecompress;
// reconstruct the HTTP header
auto h = q->headers();
for (qsizetype i = 0; i < hm.size(); ++i) {
const auto key = hm.nameAt(i);
const auto originValue = hm.valueAt(i);
// Reset any previous "location" header set in the reply. In case of
// redirects, we don't want to 'append' multiple location header values,
// rather we keep only the latest one
if (key.compare(locationHeader(), Qt::CaseInsensitive) == 0)
h.removeAll(key);
if (shouldDecompress && !decompressHelper.isValid()
&& key == QHttpHeaders::wellKnownHeaderName(QHttpHeaders::WellKnownHeader::ContentEncoding)) {
if (!synchronous) // with synchronous all the data is expected to be handled at once
decompressHelper.setCountingBytesEnabled(true);
if (!decompressHelper.setEncoding(originValue)) {
error(QNetworkReplyImpl::NetworkError::UnknownContentError,
QCoreApplication::translate("QHttp", "Failed to initialize decompression: %1")
.arg(decompressHelper.errorString()));
return;
}
decompressHelper.setDecompressedSafetyCheckThreshold(
request.decompressedSafetyCheckThreshold());
}
h.append(key, originValue);
}
q->setHeaders(std::move(h));
q->setAttribute(QNetworkRequest::HttpStatusCodeAttribute, statusCode);
q->setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, reasonPhrase);
if (removedContentLength != -1)
q->setAttribute(QNetworkRequest::OriginalContentLengthAttribute, removedContentLength);
// is it a redirection?
if (!isHttpRedirectResponse())
checkForRedirect(statusCode);
if (statusCode >= 500 && statusCode < 600) {
QAbstractNetworkCache *nc = managerPrivate->networkCache;
if (nc) {
QNetworkCacheMetaData metaData = nc->metaData(httpRequest.url());
auto value = metaData.headers().value(QHttpHeaders::WellKnownHeader::CacheControl);
bool mustReValidate = false;
if (!value.empty()) {
QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(value);
if (cacheControl.contains("must-revalidate"_ba))
mustReValidate = true;
}
if (!mustReValidate && sendCacheContents(metaData))
return;
}
}
if (statusCode == 304) {
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "Received a 304 from" << request.url();
#endif
QAbstractNetworkCache *nc = managerPrivate->networkCache;
if (nc) {
QNetworkCacheMetaData oldMetaData = nc->metaData(httpRequest.url());
QNetworkCacheMetaData metaData = fetchCacheMetaData(oldMetaData);
if (oldMetaData != metaData)
nc->updateMetaData(metaData);
if (sendCacheContents(metaData))
return;
}
}
if (statusCode != 304 && statusCode != 303) {
if (!isCachingEnabled())
setCachingEnabled(true);
}
_q_metaDataChanged();
}
void QNetworkReplyHttpImplPrivate::replyDownloadProgressSlot(qint64 bytesReceived, qint64 bytesTotal)
{
Q_Q(QNetworkReplyHttpImpl);
// If we're closed just ignore this data
if (!q->isOpen())
return;
// we can be sure here that there is a download buffer
int pendingSignals = (int)pendingDownloadProgressEmissions->fetchAndAddAcquire(-1) - 1;
if (pendingSignals > 0) {
// Let's ignore this signal and look at the next one coming in
// (signal comppression)
return;
}
if (!q->isOpen())
return;
if (cacheEnabled && isCachingAllowed() && bytesReceived == bytesTotal) {
// Write everything in one go if we use a download buffer. might be more performant.
initCacheSaveDevice();
// need to check again if cache enabled and device exists
if (cacheSaveDevice && cacheEnabled)
cacheSaveDevice->write(downloadZerocopyBuffer, bytesTotal);
// FIXME where is it closed?
}
if (isHttpRedirectResponse())
return;
bytesDownloaded = bytesReceived;
setupTransferTimeout();
downloadBufferCurrentSize = bytesReceived;
// Only emit readyRead when actual data is there
// emit readyRead before downloadProgress in case this will cause events to be
// processed and we get into a recursive call (as in QProgressDialog).
if (bytesDownloaded > 0)
emit q->readyRead();
if (downloadProgressSignalChoke.isValid() &&
downloadProgressSignalChoke.elapsed() >= progressSignalInterval) {
downloadProgressSignalChoke.start();
emit q->downloadProgress(bytesDownloaded, bytesTotal);
}
}
void QNetworkReplyHttpImplPrivate::httpAuthenticationRequired(const QHttpNetworkRequest &request,
QAuthenticator *auth)
{
managerPrivate->authenticationRequired(auth, q_func(), synchronous, url, &urlForLastAuthentication, request.withCredentials());
}
#ifndef QT_NO_NETWORKPROXY
void QNetworkReplyHttpImplPrivate::proxyAuthenticationRequired(const QNetworkProxy &proxy,
QAuthenticator *authenticator)
{
managerPrivate->proxyAuthenticationRequired(request.url(), proxy, synchronous, authenticator, &lastProxyAuthentication);
}
#endif
void QNetworkReplyHttpImplPrivate::httpError(QNetworkReply::NetworkError errorCode,
const QString &errorString)
{
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "http error!" << errorCode << errorString;
#endif
// FIXME?
error(errorCode, errorString);
}
#ifndef QT_NO_SSL
void QNetworkReplyHttpImplPrivate::replyEncrypted()
{
Q_Q(QNetworkReplyHttpImpl);
emit q->encrypted();
}
void QNetworkReplyHttpImplPrivate::replySslErrors(
const QList<QSslError> &list, bool *ignoreAll, QList<QSslError> *toBeIgnored)
{
Q_Q(QNetworkReplyHttpImpl);
emit q->sslErrors(list);
// Check if the callback set any ignore and return this here to http thread
if (pendingIgnoreAllSslErrors)
*ignoreAll = true;
if (!pendingIgnoreSslErrorsList.isEmpty())
*toBeIgnored = pendingIgnoreSslErrorsList;
}
void QNetworkReplyHttpImplPrivate::replySslConfigurationChanged(const QSslConfiguration &newSslConfiguration)
{
// Receiving the used SSL configuration from the HTTP thread
if (sslConfiguration)
*sslConfiguration = newSslConfiguration;
else
sslConfiguration.reset(new QSslConfiguration(newSslConfiguration));
}
void QNetworkReplyHttpImplPrivate::replyPreSharedKeyAuthenticationRequiredSlot(QSslPreSharedKeyAuthenticator *authenticator)
{
Q_Q(QNetworkReplyHttpImpl);
emit q->preSharedKeyAuthenticationRequired(authenticator);
}
#endif
// Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread
void QNetworkReplyHttpImplPrivate::resetUploadDataSlot(bool *r)
{
*r = uploadByteDevice->reset();
if (*r) {
// reset our own position which is used for the inter-thread communication
uploadByteDevicePosition = 0;
}
}
// Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread
void QNetworkReplyHttpImplPrivate::sentUploadDataSlot(qint64 pos, qint64 amount)
{
if (!uploadByteDevice) // uploadByteDevice is no longer available
return;
if (uploadByteDevicePosition + amount != pos) {
// Sanity check, should not happen.
error(QNetworkReply::UnknownNetworkError, QString());
return;
}
uploadByteDevice->advanceReadPointer(amount);
uploadByteDevicePosition += amount;
}
// Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread
void QNetworkReplyHttpImplPrivate::wantUploadDataSlot(qint64 maxSize)
{
Q_Q(QNetworkReplyHttpImpl);
if (!uploadByteDevice) // uploadByteDevice is no longer available
return;
// call readPointer
qint64 currentUploadDataLength = 0;
char *data = const_cast<char*>(uploadByteDevice->readPointer(maxSize, currentUploadDataLength));
if (currentUploadDataLength == 0) {
uploadDeviceChoking = true;
// No bytes from upload byte device. There will be bytes later, it will emit readyRead()
// and our uploadByteDeviceReadyReadSlot() is called.
return;
} else {
uploadDeviceChoking = false;
}
// Let's make a copy of this data
QByteArray dataArray(data, currentUploadDataLength);
// Communicate back to HTTP thread
emit q->haveUploadData(uploadByteDevicePosition, dataArray, uploadByteDevice->atEnd(), uploadByteDevice->size());
}
void QNetworkReplyHttpImplPrivate::uploadByteDeviceReadyReadSlot()
{
// Start the flow between this thread and the HTTP thread again by triggering a upload.
// However only do this when we were choking before, else the state in
// QNonContiguousByteDeviceThreadForwardImpl gets messed up.
if (uploadDeviceChoking) {
uploadDeviceChoking = false;
wantUploadDataSlot(1024);
}
}
/*
*/
bool QNetworkReplyHttpImplPrivate::sendCacheContents(const QNetworkCacheMetaData &metaData)
{
Q_Q(QNetworkReplyHttpImpl);
setCachingEnabled(false);
if (!metaData.isValid())
return false;
QAbstractNetworkCache *nc = managerPrivate->networkCache;
Q_ASSERT(nc);
QIODevice *contents = nc->data(url);
if (!contents) {
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "Cannot send cache, the contents are 0" << url;
#endif
return false;
}
contents->setParent(q);
QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes();
int status = attributes.value(QNetworkRequest::HttpStatusCodeAttribute).toInt();
if (status < 100)
status = 200; // fake it
statusCode = status;
q->setAttribute(QNetworkRequest::HttpStatusCodeAttribute, status);
q->setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute));
q->setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true);
QHttpHeaders cachedHeaders = metaData.headers();
QHttpHeaders h = headers();
QUrl redirectUrl;
for (qsizetype i = 0; i < cachedHeaders.size(); ++i) {
const auto name = cachedHeaders.nameAt(i);
const auto value = cachedHeaders.valueAt(i);
if (httpRequest.isFollowRedirects()
&& !name.compare(locationHeader(), Qt::CaseInsensitive)) {
redirectUrl = QUrl::fromEncoded(value);
}
h.replaceOrAppend(name, value);
}
setHeaders(std::move(h));
if (!isHttpRedirectResponse())
checkForRedirect(status);
cacheLoadDevice = contents;
q->connect(cacheLoadDevice, SIGNAL(readyRead()), SLOT(_q_cacheLoadReadyRead()));
q->connect(cacheLoadDevice, SIGNAL(readChannelFinished()), SLOT(_q_cacheLoadReadyRead()));
// This needs to be emitted in the event loop because it can be reached at
// the direct code path of qnam.get(...) before the user has a chance
// to connect any signals.
QMetaObject::invokeMethod(q, "_q_metaDataChanged", Qt::QueuedConnection);
QMetaObject::invokeMethod(q, "_q_cacheLoadReadyRead", Qt::QueuedConnection);
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
qDebug() << "Successfully sent cache:" << url << contents->size() << "bytes";
#endif
// Do redirect processing
if (httpRequest.isFollowRedirects() && QHttpNetworkReply::isHttpRedirect(status)) {
QMetaObject::invokeMethod(q, "onRedirected", Qt::QueuedConnection,
Q_ARG(QUrl, redirectUrl),
Q_ARG(int, status),
Q_ARG(int, httpRequest.redirectCount() - 1));
}
// Set the following flag so we can ignore some signals from HTTP thread
// that would still come
loadingFromCache = true;
return true;
}
static auto caseInsensitiveCompare(QByteArrayView value)
{
return [value](QByteArrayView element)
{
return value.compare(element, Qt::CaseInsensitive) == 0;
};
}
static bool isHopByHop(QByteArrayView header)
{
constexpr QByteArrayView headers[] = { "connection",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"te",
"trailers",
"transfer-encoding",
"upgrade"};
return std::any_of(std::begin(headers), std::end(headers), caseInsensitiveCompare(header));
}
QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNetworkCacheMetaData &oldMetaData) const
{
Q_Q(const QNetworkReplyHttpImpl);
QNetworkCacheMetaData metaData = oldMetaData;
QHttpHeaders cacheHeaders = metaData.headers();
const auto newHeaders = q->headers();
for (qsizetype i = 0; i < newHeaders.size(); ++i) {
const auto name = newHeaders.nameAt(i);
const auto value = newHeaders.valueAt(i);
if (isHopByHop(name))
continue;
if (name.compare("set-cookie", Qt::CaseInsensitive) == 0)
continue;
// for 4.6.0, we were planning to not store the date header in the
// cached resource; through that we planned to reduce the number
// of writes to disk when using a QNetworkDiskCache (i.e. don't
// write to disk when only the date changes).
// However, without the date we cannot calculate the age of the page
// anymore.
//if (header == "date")
//continue;
// Don't store Warning 1xx headers
if (name.compare("warning", Qt::CaseInsensitive) == 0) {
if (value.size() == 3
&& value[0] == '1'
&& isAsciiDigit(value[1])
&& isAsciiDigit(value[2]))
continue;
}
if (cacheHeaders.contains(name)) {
// Match the behavior of Firefox and assume Cache-Control: "no-transform"
constexpr QByteArrayView headers[]=
{"content-encoding", "content-range", "content-type"};
if (std::any_of(std::begin(headers), std::end(headers), caseInsensitiveCompare(name)))
continue;
}
// IIS has been known to send "Content-Length: 0" on 304 responses, so
// ignore this too
if (statusCode == 304 && name.compare("content-length", Qt::CaseInsensitive) == 0)
continue;
#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
QByteArrayView n = newHeaders.value(name);
QByteArrayView o = cacheHeaders.value(name);
if (n != o && name.compare("date", Qt::CaseInsensitive) != 0) {
qDebug() << "replacing" << name;
qDebug() << "new" << n;
qDebug() << "old" << o;
}
#endif
cacheHeaders.replaceOrAppend(name, value);
}
metaData.setHeaders(cacheHeaders);
bool checkExpired = true;
QHash<QByteArray, QByteArray> cacheControl;
auto value = cacheHeaders.value(QHttpHeaders::WellKnownHeader::CacheControl);
if (!value.empty()) {
cacheControl = parseHttpOptionHeader(value);
QByteArray maxAge = cacheControl.value("max-age"_ba);
if (!maxAge.isEmpty()) {
checkExpired = false;
QDateTime dt = QDateTime::currentDateTimeUtc();
dt = dt.addSecs(maxAge.toInt());
metaData.setExpirationDate(dt);
}
}
if (checkExpired) {
if (const auto value = cacheHeaders.value(
QHttpHeaders::WellKnownHeader::Expires); !value.isEmpty()) {
QDateTime expiredDateTime = QNetworkHeadersPrivate::fromHttpDate(value);
metaData.setExpirationDate(expiredDateTime);
}
}
if (const auto value = cacheHeaders.value(
QHttpHeaders::WellKnownHeader::LastModified); !value.isEmpty()) {
metaData.setLastModified(QNetworkHeadersPrivate::fromHttpDate(value));
}
bool canDiskCache;
// only cache GET replies by default, all other replies (POST, PUT, DELETE)
// are not cacheable by default (according to RFC 2616 section 9)
if (httpRequest.operation() == QHttpNetworkRequest::Get) {
canDiskCache = true;
// HTTP/1.1. Check the Cache-Control header
if (cacheControl.contains("no-store"_ba))
canDiskCache = false;
// responses to POST might be cacheable
} else if (httpRequest.operation() == QHttpNetworkRequest::Post) {
canDiskCache = false;
// some pages contain "expires:" and "cache-control: no-cache" field,
// so we only might cache POST requests if we get "cache-control: max-age ..."
if (cacheControl.contains("max-age"_ba))
canDiskCache = true;
// responses to PUT and DELETE are not cacheable
} else {
canDiskCache = false;
}
metaData.setSaveToDisk(canDiskCache);
QNetworkCacheMetaData::AttributesMap attributes;
if (statusCode != 304) {
// update the status code
attributes.insert(QNetworkRequest::HttpStatusCodeAttribute, statusCode);
attributes.insert(QNetworkRequest::HttpReasonPhraseAttribute, reasonPhrase);
} else {
// this is a redirection, keep the attributes intact
attributes = oldMetaData.attributes();
}
metaData.setAttributes(attributes);
return metaData;
}
bool QNetworkReplyHttpImplPrivate::canResume() const
{
Q_Q(const QNetworkReplyHttpImpl);
// Only GET operation supports resuming.
if (operation != QNetworkAccessManager::GetOperation)
return false;
const auto h = q->headers();
// Can only resume if server/resource supports Range header.
const auto acceptRanges = h.value(QHttpHeaders::WellKnownHeader::AcceptRanges);
if (acceptRanges.empty() || acceptRanges == "none")
return false;
// We only support resuming for byte ranges.
const auto range = h.value(QHttpHeaders::WellKnownHeader::Range);
if (!range.empty()) {
if (!range.startsWith(bytesEqualPrefix()))
return false;
}
// If we're using a download buffer then we don't support resuming/migration
// right now. Too much trouble.
if (downloadZerocopyBuffer)
return false;
return true;
}
void QNetworkReplyHttpImplPrivate::setResumeOffset(quint64 offset)
{
resumeOffset = offset;
}
void QNetworkReplyHttpImplPrivate::_q_startOperation()
{
// Ensure this function is only being called once, and not at all if we were
// cancelled
if (state >= Working)
return;
state = Working;
postRequest(request);
setupTransferTimeout();
if (synchronous) {
state = Finished;
q_func()->setFinished(true);
}
}
void QNetworkReplyHttpImplPrivate::_q_cacheLoadReadyRead()
{
Q_Q(QNetworkReplyHttpImpl);
if (state != Working)
return;
if (!cacheLoadDevice || !q->isOpen() || !cacheLoadDevice->bytesAvailable())
return;
// FIXME Optimize to use zerocopy download buffer if it is a QBuffer.
// Needs to be done where sendCacheContents() (?) of HTTP is emitting
// metaDataChanged ?
const auto totalSizeOpt = QNetworkHeadersPrivate::toInt(
headers().value(QHttpHeaders::WellKnownHeader::ContentLength));
// emit readyRead before downloadProgress in case this will cause events to be
// processed and we get into a recursive call (as in QProgressDialog).
if (!(isHttpRedirectResponse())) {
// This readyRead() goes to the user. The user then may or may not read() anything.
emit q->readyRead();
if (downloadProgressSignalChoke.isValid() &&
downloadProgressSignalChoke.elapsed() >= progressSignalInterval) {
downloadProgressSignalChoke.start();
emit q->downloadProgress(bytesDownloaded, totalSizeOpt.value_or(-1));
}
}
// A signal we've emitted might be handled by a slot that aborts,
// so we need to check for that and bail out if it's happened:
if (!q->isOpen())
return;
// If there are still bytes available in the cacheLoadDevice then the user did not read
// in response to the readyRead() signal. This means we have to load from the cacheLoadDevice
// and buffer that stuff. This is needed to be able to properly emit finished() later.
while (cacheLoadDevice->bytesAvailable() && !isHttpRedirectResponse())
buffer.append(cacheLoadDevice->readAll());
if (cacheLoadDevice->isSequential()) {
// check if end and we can read the EOF -1
char c;
qint64 actualCount = cacheLoadDevice->read(&c, 1);
if (actualCount < 0) {
cacheLoadDevice->deleteLater();
cacheLoadDevice = nullptr;
QMetaObject::invokeMethod(q, "_q_finished", Qt::QueuedConnection);
} else if (actualCount == 1) {
// This is most probably not happening since most QIODevice returned something proper for bytesAvailable()
// and had already been "emptied".
cacheLoadDevice->ungetChar(c);
}
} else if ((!cacheLoadDevice->isSequential() && cacheLoadDevice->atEnd())) {
// This codepath is in case the cache device is a QBuffer, e.g. from QNetworkDiskCache.
cacheLoadDevice->deleteLater();
cacheLoadDevice = nullptr;
QMetaObject::invokeMethod(q, "_q_finished", Qt::QueuedConnection);
}
}
void QNetworkReplyHttpImplPrivate::_q_bufferOutgoingDataFinished()
{
Q_Q(QNetworkReplyHttpImpl);
// make sure this is only called once, ever.
//_q_bufferOutgoingData may call it or the readChannelFinished emission
if (state != Buffering)
return;
// disconnect signals
QObject::disconnect(outgoingData, SIGNAL(readyRead()), q, SLOT(_q_bufferOutgoingData()));
QObject::disconnect(outgoingData, SIGNAL(readChannelFinished()), q, SLOT(_q_bufferOutgoingDataFinished()));
// finally, start the request
QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection);
}
void QNetworkReplyHttpImplPrivate::_q_cacheSaveDeviceAboutToClose()
{
// do not keep a dangling pointer to the device around (device
// is closing because e.g. QAbstractNetworkCache::remove() was called).
cacheSaveDevice = nullptr;
}
void QNetworkReplyHttpImplPrivate::_q_bufferOutgoingData()
{
Q_Q(QNetworkReplyHttpImpl);
if (!outgoingDataBuffer) {
// first call, create our buffer
outgoingDataBuffer = std::make_shared<QRingBuffer>();
QObject::connect(outgoingData, SIGNAL(readyRead()), q, SLOT(_q_bufferOutgoingData()));
QObject::connect(outgoingData, SIGNAL(readChannelFinished()), q, SLOT(_q_bufferOutgoingDataFinished()));
}
qint64 bytesBuffered = 0;
qint64 bytesToBuffer = 0;
// read data into our buffer
forever {
bytesToBuffer = outgoingData->bytesAvailable();
// unknown? just try 2 kB, this also ensures we always try to read the EOF
if (bytesToBuffer <= 0)
bytesToBuffer = 2*1024;
char *dst = outgoingDataBuffer->reserve(bytesToBuffer);
bytesBuffered = outgoingData->read(dst, bytesToBuffer);
if (bytesBuffered == -1) {
// EOF has been reached.
outgoingDataBuffer->chop(bytesToBuffer);
_q_bufferOutgoingDataFinished();
break;
} else if (bytesBuffered == 0) {
// nothing read right now, just wait until we get called again
outgoingDataBuffer->chop(bytesToBuffer);
break;
} else {
// don't break, try to read() again
outgoingDataBuffer->chop(bytesToBuffer - bytesBuffered);
}
}
}
void QNetworkReplyHttpImplPrivate::_q_transferTimedOut()
{
Q_Q(QNetworkReplyHttpImpl);
q->abortImpl(QNetworkReply::TimeoutError);
}
void QNetworkReplyHttpImplPrivate::setupTransferTimeout()
{
Q_Q(QNetworkReplyHttpImpl);
if (!transferTimeout) {
transferTimeout = new QTimer(q);
QObject::connect(transferTimeout, SIGNAL(timeout()),
q, SLOT(_q_transferTimedOut()),
Qt::QueuedConnection);
}
transferTimeout->stop();
if (request.transferTimeoutAsDuration() > 0ms) {
transferTimeout->setSingleShot(true);
transferTimeout->setInterval(request.transferTimeoutAsDuration());
QMetaObject::invokeMethod(transferTimeout, "start",
Qt::QueuedConnection);
}
}
// need to have this function since the reply is a private member variable
// and the special backends need to access this.
void QNetworkReplyHttpImplPrivate::emitReplyUploadProgress(qint64 bytesSent, qint64 bytesTotal)
{
Q_Q(QNetworkReplyHttpImpl);
if (isFinished)
return;
setupTransferTimeout();
if (!emitAllUploadProgressSignals) {
//choke signal emissions, except the first and last signals which are unconditional
if (uploadProgressSignalChoke.isValid()) {
if (bytesSent != bytesTotal && uploadProgressSignalChoke.elapsed() < progressSignalInterval) {
return;
}
}
uploadProgressSignalChoke.start();
}
emit q->uploadProgress(bytesSent, bytesTotal);
}
QNonContiguousByteDevice* QNetworkReplyHttpImplPrivate::createUploadByteDevice()
{
Q_Q(QNetworkReplyHttpImpl);
if (outgoingDataBuffer)
uploadByteDevice = QNonContiguousByteDeviceFactory::createShared(outgoingDataBuffer);
else if (outgoingData) {
uploadByteDevice = QNonContiguousByteDeviceFactory::createShared(outgoingData);
} else {
return nullptr;
}
// We want signal emissions only for normal asynchronous uploads
if (!synchronous)
QObject::connect(uploadByteDevice.get(), SIGNAL(readProgress(qint64,qint64)),
q, SLOT(emitReplyUploadProgress(qint64,qint64)));
return uploadByteDevice.get();
}
void QNetworkReplyHttpImplPrivate::_q_finished()
{
// This gets called queued, just forward to real call then
finished();
}
void QNetworkReplyHttpImplPrivate::finished()
{
Q_Q(QNetworkReplyHttpImpl);
if (transferTimeout)
transferTimeout->stop();
if (state == Finished || state == Aborted)
return;
const auto totalSizeOpt = QNetworkHeadersPrivate::toInt(
headers().value(QHttpHeaders::WellKnownHeader::ContentLength));
const qint64 totalSize = totalSizeOpt.value_or(-1);
// if we don't know the total size of or we received everything save the cache.
// If the data is compressed then this is done in readData()
if ((totalSize == -1 || bytesDownloaded == totalSize)
&& !decompressHelper.isValid()) {
completeCacheSave();
}
// We check for errorCode too as in case of SSL handshake failure, we still
// get the HTTP redirect status code (301, 303 etc)
if (isHttpRedirectResponse() && errorCode == QNetworkReply::NoError)
return;
state = Finished;
q->setFinished(true);
if (totalSize == -1) {
emit q->downloadProgress(bytesDownloaded, bytesDownloaded);
} else {
emit q->downloadProgress(bytesDownloaded, totalSize);
}
if (bytesUploaded == -1 && (outgoingData || outgoingDataBuffer))
emit q->uploadProgress(0, 0);
emit q->readChannelFinished();
emit q->finished();
}
void QNetworkReplyHttpImplPrivate::_q_error(QNetworkReplyImpl::NetworkError code, const QString &errorMessage)
{
this->error(code, errorMessage);
}
void QNetworkReplyHttpImplPrivate::error(QNetworkReplyImpl::NetworkError code, const QString &errorMessage)
{
Q_Q(QNetworkReplyHttpImpl);
// Can't set and emit multiple errors.
if (errorCode != QNetworkReply::NoError) {
// But somewhat unavoidable if we have cancelled the request:
if (errorCode != QNetworkReply::OperationCanceledError
&& errorCode != QNetworkReply::TimeoutError) {
qWarning("QNetworkReplyImplPrivate::error: Internal problem, this method must only be called once.");
}
return;
}
errorCode = code;
q->setErrorString(errorMessage);
// note: might not be a good idea, since users could decide to delete us
// which would delete the backend too...
// maybe we should protect the backend
emit q->errorOccurred(code);
}
void QNetworkReplyHttpImplPrivate::_q_metaDataChanged()
{
// FIXME merge this with replyDownloadMetaData(); ?
Q_Q(QNetworkReplyHttpImpl);
// 1. do we have cookies?
// 2. are we allowed to set them?
Q_ASSERT(manager);
const auto cookiesOpt = QNetworkHeadersPrivate::toSetCookieList(
headers().values(QHttpHeaders::WellKnownHeader::SetCookie));
const auto cookies = cookiesOpt.value_or(QList<QNetworkCookie>());
if (!cookies.empty()
&& request.attribute(QNetworkRequest::CookieSaveControlAttribute,
QNetworkRequest::Automatic).toInt() == QNetworkRequest::Automatic) {
QNetworkCookieJar *jar = manager->cookieJar();
if (jar) {
jar->setCookiesFromUrl(cookies, url);
}
}
emit q->metaDataChanged();
}
void QNetworkReplyHttpImplPrivate::createCache()
{
// check if we can save and if we're allowed to
if (!managerPrivate->networkCache
|| !request.attribute(QNetworkRequest::CacheSaveControlAttribute, true).toBool())
return;
cacheEnabled = true;
}
bool QNetworkReplyHttpImplPrivate::isCachingEnabled() const
{
return (cacheEnabled && managerPrivate->networkCache != nullptr);
}
void QNetworkReplyHttpImplPrivate::setCachingEnabled(bool enable)
{
if (!enable && !cacheEnabled)
return; // nothing to do
if (enable && cacheEnabled)
return; // nothing to do either!
if (enable) {
if (Q_UNLIKELY(bytesDownloaded)) {
qDebug() << "setCachingEnabled: " << bytesDownloaded << " bytesDownloaded";
// refuse to enable in this case
qCritical("QNetworkReplyImpl: backend error: caching was enabled after some bytes had been written");
return;
}
createCache();
} else {
// someone told us to turn on, then back off?
// ok... but you should make up your mind
qDebug("QNetworkReplyImpl: setCachingEnabled(true) called after setCachingEnabled(false)");
managerPrivate->networkCache->remove(url);
cacheSaveDevice = nullptr;
cacheEnabled = false;
}
}
bool QNetworkReplyHttpImplPrivate::isCachingAllowed() const
{
return operation == QNetworkAccessManager::GetOperation || operation == QNetworkAccessManager::HeadOperation;
}
void QNetworkReplyHttpImplPrivate::completeCacheSave()
{
if (cacheEnabled && errorCode != QNetworkReplyImpl::NoError) {
managerPrivate->networkCache->remove(url);
} else if (cacheEnabled && cacheSaveDevice) {
managerPrivate->networkCache->insert(cacheSaveDevice);
}
cacheSaveDevice = nullptr;
cacheEnabled = false;
}
QT_END_NAMESPACE
#include "moc_qnetworkreplyhttpimpl_p.cpp"