Downloading http page with several panes

jcga

Hi,
I wrote a simple code based on the Download Manager example (https://code.qt.io/cgit/qt/qtbase.git/tree/examples/network/downloadmanager?h=6.3) to download html pages. But when I download a page with several panes, only one of the panes is downloaded, the other are missing. If, instead, I save the page on the disk (using "save as" command of the chrome browser), all panes are present.

Here is the code (this is code I'm currently working on, so it is not very clean):

************************ file downloadmanger.h ************************
#ifndef DOWNLOADMANAGER_H
#define DOWNLOADMANAGER_H
#include <QtCore>
#include <QtNetwork>

class DownloadManager: public QObject
{
    Q_OBJECT
    QNetworkAccessManager manager;
    QVector<QNetworkReply *> currentDownloads;

public:
    DownloadManager();
    QNetworkReply* doDownload(const QUrl &url);
    static QString saveFileName(const QUrl &url);
    bool saveToDisk(const QString &filename, QIODevice *data);
    static bool isHttpRedirect(QNetworkReply *reply);

public slots:
    void downloadFinished(QNetworkReply *reply);
};

#endif // DOWNLOADMANAGER_H

******************** file downloadmanager.cpp *********************

#include "downloadmanager.h"
#include "../../../libraries/commons/debugtools.h"
#include <QUrl>

DownloadManager::DownloadManager()
{
    connect(&manager, &QNetworkAccessManager::finished,
            this, &DownloadManager::downloadFinished);
}

QNetworkReply* DownloadManager::doDownload(const QUrl &url)
{
    QNetworkRequest request(url);
    QNetworkReply *reply = manager.get(request);

    return reply;
}

QString DownloadManager::saveFileName(const QUrl &url)
{
    QString path = url.path();
    QString basename = QFileInfo(path).fileName();

    if (basename.isEmpty())
        basename = "download";

    if (QFile::exists(basename)) {
        // already exists, don't overwrite
        int i = 0;
        basename += '.';
        while (QFile::exists(basename + QString::number(i)))
            ++i;

        basename += QString::number(i);
    }

    return basename;
}

bool DownloadManager::saveToDisk(const QString &filename, QIODevice *data)
{
    QFile file(filename);
    VERIFY( file.open(QIODevice::WriteOnly | QIODevice::Text) );
    file.write(data->readAll());
    file.close();



    return true;
}

bool DownloadManager::isHttpRedirect(QNetworkReply *reply)
{
    int statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
    return statusCode == 301 || statusCode == 302 || statusCode == 303
           || statusCode == 305 || statusCode == 307 || statusCode == 308;
}

void DownloadManager::downloadFinished(QNetworkReply *reply)
{
    QUrl url = reply->url();
    if (reply->error())
    {
        fprintf(stderr, "Download of %s failed: %s\n",
                url.toEncoded().constData(),
                qPrintable(reply->errorString()));
    } else
    {
        if (isHttpRedirect(reply)) {
            fputs("Request was redirected.\n", stderr);
        } else
        {
            QString filename = saveFileName(url);
            if (saveToDisk(filename, reply)) {
                printf("Download of %s succeeded (saved to %s)\n",
                       url.toEncoded().constData(), qPrintable(filename));
            }
        }
    }

    currentDownloads.removeAll(reply);
    reply->deleteLater();

    if (currentDownloads.isEmpty()) {
        // all downloads finished
        QCoreApplication::instance()->quit();
    }
}

******************** file main.cpp *******************************

#include <QtCore>
#include <QtNetwork>

#include "downloadmanager.h"

#include <cstdio>

QT_BEGIN_NAMESPACE
class QSslError;
QT_END_NAMESPACE

using namespace std;


int main(int argc, char **argv)
{
    QCoreApplication app(argc, argv);

    DownloadManager manager;
    QNetworkReply* r=manager.doDownload(QUrl("www.amazon.fr/s?k=hp+notebook"));

    app.exec();
}

Here is a website where this phenomenon occurs:
https://www.amazon.fr/s?k=hp+notebook
Download manager downloads only the left pane, but not the offers on the right pane.

All suggestions are welcome.

Best,
Jean-Claude