kioslave: reimplement HTTP(S) slave via curl

Signed-off-by: Ivailo Monev <xakepa10@gmail.com>

.github/workflows/codeql-analysis.yml

@@ -42,7 +42,7 @@ jobs:
         sudo apt-get install -qq wget
         sudo wget https://raw.githubusercontent.com/fluxer/katana-ubuntu/master/katana.list -O /etc/apt/sources.list.d/katana.list
         sudo apt-get update -qq
-        sudo apt-get install -qq cmake katie-dev libenchant-dev libmagick++-dev libmpv-dev xorg-dev mesa-common-dev libavahi-common-dev libwebp-dev libudev-dev liblzma-dev libexiv2-dev libbz2-dev libattr1-dev libacl1-dev libcdio-dev strigi libdbusmenu-katie media-player-info shared-mime-info media-player-info xdg-utils
+        sudo apt-get install -qq cmake katie-dev libenchant-dev libmagick++-dev libmpv-dev xorg-dev mesa-common-dev libavahi-common-dev libwebp-dev libudev-dev liblzma-dev libexiv2-dev libbz2-dev libattr1-dev libacl1-dev libcdio-dev libcurl4-openssl-dev strigi libdbusmenu-katie media-player-info shared-mime-info media-player-info xdg-utils
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

CMakeLists.txt

@@ -69,6 +69,14 @@ set_package_properties(ENCHANT PROPERTIES
     PURPOSE "Spell checking support via Enchant"
 )
 
+find_package(Curl)
+set_package_properties(Curl PROPERTIES
+    DESCRIPTION "Command line tool and library for transferring data with URLs"
+    URL "https://curl.se/"
+    TYPE REQUIRED
+    PURPOSE "KIO HTTP(S) slave"
+)
+
 # optional features
 if(X11_FOUND)
     macro_bool_to_01(X11_XTest_FOUND HAVE_XTEST)
@@ -88,14 +96,6 @@ if(X11_FOUND)
     endif(NOT HAVE_XSYNC AND NOT HAVE_XSCREENSAVER)
 endif(X11_FOUND)
 
-macro_optional_find_package(OpenSSL)
-set_package_properties(OpenSSL PROPERTIES
-    DESCRIPTION "Support for secure network communications (SSL and TLS)"
-    URL "http://openssl.org"
-    TYPE RECOMMENDED
-    PURPOSE "For the bulk of secure communications, including secure web browsing via HTTPS"
-)
-
 macro_optional_find_package(Libintl)
 set_package_properties(Libintl PROPERTIES
     DESCRIPTION "Support for multiple languages"

appveyor.yml

@@ -21,9 +21,9 @@ build_script:
     sudo apt-get install -qq cmake katie-dev libenchant-dev \
         libmagick++-dev libmpv-dev xorg-dev mesa-common-dev \
         libavahi-common-dev libwebp-dev libudev-dev liblzma-dev \
-        libexiv2-dev libbz2-dev libacl1-dev libcdio-dev strigi \
+        libexiv2-dev libbz2-dev libacl1-dev libcdio-dev \
-        libdbusmenu-katie media-player-info shared-mime-info \
+        libcurl4-openssl-dev strigi libdbusmenu-katie media-player-info \
-        media-player-info xdg-utils ccache
+        shared-mime-info media-player-info xdg-utils ccache
 
     export PATH="/usr/lib/ccache/:$PATH"
 

cmake/modules/CMakeLists.txt

@@ -14,6 +14,7 @@ set(cmakeFilesDontInstall
     FindACL.cmake
     FindLibCDIO.cmake
     FindDevinfo.cmake
+    FindCurl.cmake
 )
 
 # Explicitly list all files which will be installed.

cmake/modules/FindCurl.cmake

@@ -0,0 +1,42 @@
+# Try to find curl library, once done this will define:
+#
+#  CURL_FOUND - system has curl
+#  CURL_INCLUDE_DIR - the curl include directory
+#  CURL_LIBRARIES - the libraries needed to use curl
+#  CURL_DEFINITIONS - compiler switches required for using curl
+#
+# Copyright (c) 2022 Ivailo Monev <xakepa10@gmail.com>
+#
+# Redistribution and use is allowed according to the terms of the BSD license.
+# For details see the accompanying COPYING-CMAKE-SCRIPTS file.
+
+if(NOT WIN32)
+    include(FindPkgConfig)
+    pkg_check_modules(PC_CURL QUIET curl)
+
+    set(CURL_INCLUDE_DIR ${PC_CURL_INCLUDE_DIRS})
+    set(CURL_LIBRARIES ${PC_CURL_LIBRARIES})
+endif()
+
+set(CURL_VERSION ${PC_CURL_VERSION})
+set(CURL_DEFINITIONS ${PC_CURL_CFLAGS_OTHER})
+
+if(NOT CURL_INCLUDE_DIR OR NOT CURL_LIBRARIES)
+    find_path(CURL_INCLUDE_DIR
+        NAMES curl/curl.h
+        HINTS $ENV{CURLDIR}/include
+    )
+
+    find_library(CURL_LIBRARIES
+        NAMES curl
+        HINTS $ENV{CURLDIR}/lib
+    )
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Curl
+    VERSION_VAR CURL_VERSION
+    REQUIRED_VARS CURL_LIBRARIES CURL_INCLUDE_DIR
+)
+
+mark_as_advanced(CURL_INCLUDE_DIR CURL_LIBRARIES)

kioslave/http/CMakeLists.txt

@@ -1,12 +1,19 @@
 project(kioslave-http)
 
-include_directories(${KDE4_KIO_INCLUDES})
+include_directories(
+    ${KDE4_KIO_INCLUDES}
+    ${CURL_INCLUDES}
+)
 
 ########### next target ###############
 
 kde4_add_plugin(kio_http http.cpp)
 
-target_link_libraries(kio_http ${KDE4_KDECORE_LIBS} ${KDE4_KIO_LIBS})
+target_link_libraries(kio_http
+    ${KDE4_KDECORE_LIBS}
+    ${KDE4_KIO_LIBS}
+    ${CURL_LIBRARIES}
+)
 
 install(TARGETS kio_http DESTINATION ${KDE4_PLUGIN_INSTALL_DIR})
 

kioslave/http/http.cpp

@@ -22,9 +22,6 @@
 
 #include <QCoreApplication>
 #include <QThread>
-#include <QNetworkAccessManager>
-#include <QNetworkDiskCache>
-#include <QNetworkReply>
 
 #include <sys/types.h>
 #include <unistd.h>
@@ -38,6 +35,33 @@ static inline QByteArray HTTPMIMEType(const QByteArray &contenttype)
     return splitcontenttype.at(0);
 }
 
+size_t curlWriteCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
+{
+    HttpProtocol* httpprotocol = static_cast<HttpProtocol*>(userdata);
+    if (!httpprotocol) {
+        return 0;
+    }
+    // emit MIME before data
+    if (httpprotocol->firstchunk) {
+        httpprotocol->slotMIME();
+        httpprotocol->firstchunk = false;
+    }
+    // size is always 1
+    Q_UNUSED(size);
+    httpprotocol->slotData(ptr, nmemb);
+    return nmemb;
+}
+
+int curlProgressCallback(void *userdata, double dltotal, double dlnow, double ultotal, double ulnow)
+{
+    HttpProtocol* httpprotocol = static_cast<HttpProtocol*>(userdata);
+    if (!httpprotocol) {
+        return CURLE_BAD_FUNCTION_ARGUMENT;
+    }
+    httpprotocol->slotProgress(dlnow, dltotal);
+    return CURLE_OK;
+}
+
 extern "C" int Q_DECL_EXPORT kdemain(int argc, char **argv)
 {
     QCoreApplication app(argc, argv);
@@ -59,85 +83,96 @@ extern "C" int Q_DECL_EXPORT kdemain(int argc, char **argv)
 }
 
 HttpProtocol::HttpProtocol(const QByteArray &pool, const QByteArray &app)
-    : SlaveBase("http", pool, app)
+    : SlaveBase("http", pool, app), firstchunk(true), m_curl(nullptr)
 {
+    m_curl = curl_easy_init();
+    if (!m_curl) {
+        kWarning(7103) << "Could not create context";
+        return;
+    }
+
+    curl_easy_setopt(m_curl, CURLOPT_FOLLOWLOCATION, 1L);
+    curl_easy_setopt(m_curl, CURLOPT_MAXREDIRS, 10L);
+    curl_easy_setopt(m_curl, CURLOPT_TIMEOUT, 30L);
+    curl_easy_setopt(m_curl, CURLOPT_WRITEDATA, this);
+    curl_easy_setopt(m_curl, CURLOPT_WRITEFUNCTION, curlWriteCallback);
+    curl_easy_setopt(m_curl, CURLOPT_NOPROGRESS, 0L); // otherwise the progress callback is not called
+    curl_easy_setopt(m_curl, CURLOPT_PROGRESSDATA, this);
+    curl_easy_setopt(m_curl, CURLOPT_PROGRESSFUNCTION, curlProgressCallback);
 }
 
 HttpProtocol::~HttpProtocol()
 {
+    if (m_curl) {
+        curl_easy_cleanup(m_curl);
+    }
 }
 
 void HttpProtocol::get(const KUrl &url)
 {
     kDebug(7103) << "URL" << url.prettyUrl();
 
-    QNetworkAccessManager netmanager(this);
+    if (Q_UNLIKELY(!m_curl)) {
-    QNetworkDiskCache netcache(this);
+        error(KIO::ERR_OUT_OF_MEMORY, QString::fromLatin1("Null context"));
-    QNetworkRequest netrequest(url);
+        return;
+    }
+
+    firstchunk = true;
+    const QByteArray urlbytes = url.prettyUrl().toLocal8Bit();
+    curl_easy_setopt(m_curl, CURLOPT_URL, urlbytes.constData());
 
     kDebug(7103) << "Metadata" << allMetaData();
+    struct curl_slist *curllist = NULL;
     // metadata from scheduler
     if (hasMetaData("Languages")) {
-        netrequest.setRawHeader("Accept-Language", metaData("Languages").toAscii());
+        curllist = curl_slist_append(curllist, QByteArray("Accept-Language: ") + metaData("Languages").toAscii());
     }
     if (hasMetaData("Charsets")) {
-        netrequest.setRawHeader("Accept-Charset", metaData("Charsets").toAscii());
+        curllist = curl_slist_append(curllist, QByteArray("Accept-Charset: ") + metaData("Charsets").toAscii());
-    }
-    if (hasMetaData("CacheDir")) {
-        netcache.setCacheDirectory(metaData("CacheDir"));
     }
     if (hasMetaData("UserAgent")) {
-        netrequest.setRawHeader("User-Agent", metaData("UserAgent").toAscii());
+        const QByteArray useragentbytes = metaData("UserAgent").toAscii();
+        curl_easy_setopt(m_curl, CURLOPT_USERAGENT, useragentbytes.constData());
     }
-    netmanager.setCache(&netcache);
+    CURLcode curlresult = curl_easy_setopt(m_curl, CURLOPT_HTTPHEADER, curllist);
-    // KIO metadata
+    if (curlresult != CURLE_OK) {
-    if (hasMetaData("cache")) {
+        curl_slist_free_all(curllist);
-        const QByteArray kiocache = metaData("cache").toAscii();
+        kWarning(7103) << "Error" << curl_easy_strerror(curlresult);
-        if (qstricmp(kiocache.constData(), "cache") == 0) {
+        error(KIO::ERR_COULD_NOT_CONNECT, curl_easy_strerror(curlresult));
-            netrequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QVariant(QNetworkRequest::PreferCache));
-        } else if (qstricmp(kiocache.constData(), "cacheonly") == 0) {
-            netrequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QVariant(QNetworkRequest::AlwaysCache));
-        } else if (qstricmp(kiocache.constData(), "verify") == 0 || qstricmp(kiocache.constData(), "refresh") == 0) {
-            netrequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QVariant(QNetworkRequest::PreferNetwork));
-        } else {
-            // reload or unknown
-            netrequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QVariant(QNetworkRequest::AlwaysNetwork));
-        }
-    }
-
-    QNetworkReply* netreply = netmanager.get(netrequest);
-    connect(netreply, SIGNAL(downloadProgress(qint64,qint64)), this, SLOT(slotProgress(qint64,qint64)));
-    while (!netreply->isFinished()) {
-        QCoreApplication::processEvents();
-        QThread::msleep(400);
-    }
-
-    if (netreply->error() != QNetworkReply::NoError) {
-        kWarning(7103) << "Error" << netreply->url() << netreply->error();
-        error(KIO::ERR_COULD_NOT_CONNECT, url.prettyUrl());
         return;
     }
 
-    const QVariant netredirect = netreply->attribute(QNetworkRequest::RedirectionTargetAttribute);
+    curlresult = curl_easy_perform(m_curl);
-    if (netredirect.isValid()) {
+    if (curlresult != CURLE_OK) {
-        const QUrl netredirecturl = netreply->url().resolved(netredirect.toUrl());
+        curl_slist_free_all(curllist);
-        kDebug(7103) << "Redirecting to" << netredirecturl;
+        kWarning(7103) << "Error" << curl_easy_strerror(curlresult);
-        redirection(netredirecturl);
+        error(KIO::ERR_COULD_NOT_CONNECT, curl_easy_strerror(curlresult));
-        finished();
         return;
     }
 
-    kDebug(7103) << "Headers" << netreply->rawHeaderPairs();
+    curl_slist_free_all(curllist);
-
-    const QByteArray httpmimetype = HTTPMIMEType(netreply->rawHeader("content-type"));
-    kDebug(7103) << "MIME type" << httpmimetype;
-    emit mimeType(httpmimetype);
-
-    data(netreply->readAll());
-
     finished();
 }
 
+void HttpProtocol::slotMIME()
+{
+    char *curlcontenttype = nullptr;
+    CURLcode curlresult = curl_easy_getinfo(m_curl, CURLINFO_CONTENT_TYPE, &curlcontenttype);
+    if (curlresult == CURLE_OK) {
+        const QByteArray httpmimetype = HTTPMIMEType(QByteArray(curlcontenttype));
+        kDebug(7103) << "MIME type" << httpmimetype;
+        emit mimeType(httpmimetype);
+    } else {
+        kWarning(7103) << "Could not get info" << curl_easy_strerror(curlresult);
+    }
+
+}
+
+void HttpProtocol::slotData(const char* curldata, const size_t curldatasize)
+{
+    data(QByteArray::fromRawData(curldata, curldatasize));
+}
+
 void HttpProtocol::slotProgress(qint64 received, qint64 total)
 {
     kDebug(7103) << "Received" << received << "from" << total;

kioslave/http/http.h

@@ -24,6 +24,8 @@
 #include <kurl.h>
 #include <kio/slavebase.h>
 
+#include <curl/curl.h>
+
 class HttpProtocol : public QObject, public KIO::SlaveBase
 {
     Q_OBJECT
@@ -34,8 +36,13 @@ public:
 
     virtual void get(const KUrl &url);
 
-private Q_SLOTS:
+    void slotMIME();
+    void slotData(const char* curldata, const size_t curldatasize);
     void slotProgress(qint64 received, qint64 total);
+
+    bool firstchunk;
+private:
+    CURL* m_curl;
 };
 
 #endif // KDELIBS_HTTP_H