diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 67b855a5..4615516f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -42,7 +42,7 @@ jobs: sudo apt-get install -qq wget sudo wget https://raw.githubusercontent.com/fluxer/katana-ubuntu/master/katana.list -O /etc/apt/sources.list.d/katana.list sudo apt-get update -qq - sudo apt-get install -qq cmake katie-dev libenchant-dev libmagick++-dev libmpv-dev xorg-dev mesa-common-dev libavahi-common-dev libwebp-dev libudev-dev liblzma-dev libexiv2-dev libbz2-dev libattr1-dev libacl1-dev libcdio-dev strigi libdbusmenu-katie media-player-info shared-mime-info media-player-info xdg-utils + sudo apt-get install -qq cmake katie-dev libenchant-dev libmagick++-dev libmpv-dev xorg-dev mesa-common-dev libavahi-common-dev libwebp-dev libudev-dev liblzma-dev libexiv2-dev libbz2-dev libattr1-dev libacl1-dev libcdio-dev libcurl4-openssl-dev strigi libdbusmenu-katie media-player-info shared-mime-info media-player-info xdg-utils # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/CMakeLists.txt b/CMakeLists.txt index fa2bb54b..6bb1188a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,6 +69,14 @@ set_package_properties(ENCHANT PROPERTIES PURPOSE "Spell checking support via Enchant" ) +find_package(Curl) +set_package_properties(Curl PROPERTIES + DESCRIPTION "Command line tool and library for transferring data with URLs" + URL "https://curl.se/" + TYPE REQUIRED + PURPOSE "KIO HTTP(S) slave" +) + # optional features if(X11_FOUND) macro_bool_to_01(X11_XTest_FOUND HAVE_XTEST) @@ -88,14 +96,6 @@ if(X11_FOUND) endif(NOT HAVE_XSYNC AND NOT HAVE_XSCREENSAVER) endif(X11_FOUND) -macro_optional_find_package(OpenSSL) -set_package_properties(OpenSSL PROPERTIES - DESCRIPTION "Support for secure network communications (SSL and TLS)" - URL "http://openssl.org" - TYPE RECOMMENDED - PURPOSE "For the bulk of secure communications, including secure web browsing via HTTPS" -) - macro_optional_find_package(Libintl) set_package_properties(Libintl PROPERTIES DESCRIPTION "Support for multiple languages" diff --git a/appveyor.yml b/appveyor.yml index 12e1f177..23423688 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -21,9 +21,9 @@ build_script: sudo apt-get install -qq cmake katie-dev libenchant-dev \ libmagick++-dev libmpv-dev xorg-dev mesa-common-dev \ libavahi-common-dev libwebp-dev libudev-dev liblzma-dev \ - libexiv2-dev libbz2-dev libacl1-dev libcdio-dev strigi \ - libdbusmenu-katie media-player-info shared-mime-info \ - media-player-info xdg-utils ccache + libexiv2-dev libbz2-dev libacl1-dev libcdio-dev \ + libcurl4-openssl-dev strigi libdbusmenu-katie media-player-info \ + shared-mime-info media-player-info xdg-utils ccache export PATH="/usr/lib/ccache/:$PATH" diff --git a/cmake/modules/CMakeLists.txt b/cmake/modules/CMakeLists.txt index e9bb2894..049ac84f 100644 --- a/cmake/modules/CMakeLists.txt +++ b/cmake/modules/CMakeLists.txt @@ -14,6 +14,7 @@ set(cmakeFilesDontInstall FindACL.cmake FindLibCDIO.cmake FindDevinfo.cmake + FindCurl.cmake ) # Explicitly list all files which will be installed. diff --git a/cmake/modules/FindCurl.cmake b/cmake/modules/FindCurl.cmake new file mode 100644 index 00000000..94ce61d1 --- /dev/null +++ b/cmake/modules/FindCurl.cmake @@ -0,0 +1,42 @@ +# Try to find curl library, once done this will define: +# +# CURL_FOUND - system has curl +# CURL_INCLUDE_DIR - the curl include directory +# CURL_LIBRARIES - the libraries needed to use curl +# CURL_DEFINITIONS - compiler switches required for using curl +# +# Copyright (c) 2022 Ivailo Monev +# +# Redistribution and use is allowed according to the terms of the BSD license. +# For details see the accompanying COPYING-CMAKE-SCRIPTS file. + +if(NOT WIN32) + include(FindPkgConfig) + pkg_check_modules(PC_CURL QUIET curl) + + set(CURL_INCLUDE_DIR ${PC_CURL_INCLUDE_DIRS}) + set(CURL_LIBRARIES ${PC_CURL_LIBRARIES}) +endif() + +set(CURL_VERSION ${PC_CURL_VERSION}) +set(CURL_DEFINITIONS ${PC_CURL_CFLAGS_OTHER}) + +if(NOT CURL_INCLUDE_DIR OR NOT CURL_LIBRARIES) + find_path(CURL_INCLUDE_DIR + NAMES curl/curl.h + HINTS $ENV{CURLDIR}/include + ) + + find_library(CURL_LIBRARIES + NAMES curl + HINTS $ENV{CURLDIR}/lib + ) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Curl + VERSION_VAR CURL_VERSION + REQUIRED_VARS CURL_LIBRARIES CURL_INCLUDE_DIR +) + +mark_as_advanced(CURL_INCLUDE_DIR CURL_LIBRARIES) diff --git a/kioslave/http/CMakeLists.txt b/kioslave/http/CMakeLists.txt index 47573e85..48d93a01 100644 --- a/kioslave/http/CMakeLists.txt +++ b/kioslave/http/CMakeLists.txt @@ -1,12 +1,19 @@ project(kioslave-http) -include_directories(${KDE4_KIO_INCLUDES}) +include_directories( + ${KDE4_KIO_INCLUDES} + ${CURL_INCLUDES} +) ########### next target ############### kde4_add_plugin(kio_http http.cpp) -target_link_libraries(kio_http ${KDE4_KDECORE_LIBS} ${KDE4_KIO_LIBS}) +target_link_libraries(kio_http + ${KDE4_KDECORE_LIBS} + ${KDE4_KIO_LIBS} + ${CURL_LIBRARIES} +) install(TARGETS kio_http DESTINATION ${KDE4_PLUGIN_INSTALL_DIR}) diff --git a/kioslave/http/http.cpp b/kioslave/http/http.cpp index e1e1a10e..1c454c67 100644 --- a/kioslave/http/http.cpp +++ b/kioslave/http/http.cpp @@ -22,9 +22,6 @@ #include #include -#include -#include -#include #include #include @@ -38,6 +35,33 @@ static inline QByteArray HTTPMIMEType(const QByteArray &contenttype) return splitcontenttype.at(0); } +size_t curlWriteCallback(char *ptr, size_t size, size_t nmemb, void *userdata) +{ + HttpProtocol* httpprotocol = static_cast(userdata); + if (!httpprotocol) { + return 0; + } + // emit MIME before data + if (httpprotocol->firstchunk) { + httpprotocol->slotMIME(); + httpprotocol->firstchunk = false; + } + // size is always 1 + Q_UNUSED(size); + httpprotocol->slotData(ptr, nmemb); + return nmemb; +} + +int curlProgressCallback(void *userdata, double dltotal, double dlnow, double ultotal, double ulnow) +{ + HttpProtocol* httpprotocol = static_cast(userdata); + if (!httpprotocol) { + return CURLE_BAD_FUNCTION_ARGUMENT; + } + httpprotocol->slotProgress(dlnow, dltotal); + return CURLE_OK; +} + extern "C" int Q_DECL_EXPORT kdemain(int argc, char **argv) { QCoreApplication app(argc, argv); @@ -59,85 +83,96 @@ extern "C" int Q_DECL_EXPORT kdemain(int argc, char **argv) } HttpProtocol::HttpProtocol(const QByteArray &pool, const QByteArray &app) - : SlaveBase("http", pool, app) + : SlaveBase("http", pool, app), firstchunk(true), m_curl(nullptr) { + m_curl = curl_easy_init(); + if (!m_curl) { + kWarning(7103) << "Could not create context"; + return; + } + + curl_easy_setopt(m_curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(m_curl, CURLOPT_MAXREDIRS, 10L); + curl_easy_setopt(m_curl, CURLOPT_TIMEOUT, 30L); + curl_easy_setopt(m_curl, CURLOPT_WRITEDATA, this); + curl_easy_setopt(m_curl, CURLOPT_WRITEFUNCTION, curlWriteCallback); + curl_easy_setopt(m_curl, CURLOPT_NOPROGRESS, 0L); // otherwise the progress callback is not called + curl_easy_setopt(m_curl, CURLOPT_PROGRESSDATA, this); + curl_easy_setopt(m_curl, CURLOPT_PROGRESSFUNCTION, curlProgressCallback); } HttpProtocol::~HttpProtocol() { + if (m_curl) { + curl_easy_cleanup(m_curl); + } } void HttpProtocol::get(const KUrl &url) { kDebug(7103) << "URL" << url.prettyUrl(); - QNetworkAccessManager netmanager(this); - QNetworkDiskCache netcache(this); - QNetworkRequest netrequest(url); + if (Q_UNLIKELY(!m_curl)) { + error(KIO::ERR_OUT_OF_MEMORY, QString::fromLatin1("Null context")); + return; + } + + firstchunk = true; + const QByteArray urlbytes = url.prettyUrl().toLocal8Bit(); + curl_easy_setopt(m_curl, CURLOPT_URL, urlbytes.constData()); kDebug(7103) << "Metadata" << allMetaData(); + struct curl_slist *curllist = NULL; // metadata from scheduler if (hasMetaData("Languages")) { - netrequest.setRawHeader("Accept-Language", metaData("Languages").toAscii()); + curllist = curl_slist_append(curllist, QByteArray("Accept-Language: ") + metaData("Languages").toAscii()); } if (hasMetaData("Charsets")) { - netrequest.setRawHeader("Accept-Charset", metaData("Charsets").toAscii()); - } - if (hasMetaData("CacheDir")) { - netcache.setCacheDirectory(metaData("CacheDir")); + curllist = curl_slist_append(curllist, QByteArray("Accept-Charset: ") + metaData("Charsets").toAscii()); } if (hasMetaData("UserAgent")) { - netrequest.setRawHeader("User-Agent", metaData("UserAgent").toAscii()); + const QByteArray useragentbytes = metaData("UserAgent").toAscii(); + curl_easy_setopt(m_curl, CURLOPT_USERAGENT, useragentbytes.constData()); } - netmanager.setCache(&netcache); - // KIO metadata - if (hasMetaData("cache")) { - const QByteArray kiocache = metaData("cache").toAscii(); - if (qstricmp(kiocache.constData(), "cache") == 0) { - netrequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QVariant(QNetworkRequest::PreferCache)); - } else if (qstricmp(kiocache.constData(), "cacheonly") == 0) { - netrequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QVariant(QNetworkRequest::AlwaysCache)); - } else if (qstricmp(kiocache.constData(), "verify") == 0 || qstricmp(kiocache.constData(), "refresh") == 0) { - netrequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QVariant(QNetworkRequest::PreferNetwork)); - } else { - // reload or unknown - netrequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QVariant(QNetworkRequest::AlwaysNetwork)); - } - } - - QNetworkReply* netreply = netmanager.get(netrequest); - connect(netreply, SIGNAL(downloadProgress(qint64,qint64)), this, SLOT(slotProgress(qint64,qint64))); - while (!netreply->isFinished()) { - QCoreApplication::processEvents(); - QThread::msleep(400); - } - - if (netreply->error() != QNetworkReply::NoError) { - kWarning(7103) << "Error" << netreply->url() << netreply->error(); - error(KIO::ERR_COULD_NOT_CONNECT, url.prettyUrl()); + CURLcode curlresult = curl_easy_setopt(m_curl, CURLOPT_HTTPHEADER, curllist); + if (curlresult != CURLE_OK) { + curl_slist_free_all(curllist); + kWarning(7103) << "Error" << curl_easy_strerror(curlresult); + error(KIO::ERR_COULD_NOT_CONNECT, curl_easy_strerror(curlresult)); return; } - const QVariant netredirect = netreply->attribute(QNetworkRequest::RedirectionTargetAttribute); - if (netredirect.isValid()) { - const QUrl netredirecturl = netreply->url().resolved(netredirect.toUrl()); - kDebug(7103) << "Redirecting to" << netredirecturl; - redirection(netredirecturl); - finished(); + curlresult = curl_easy_perform(m_curl); + if (curlresult != CURLE_OK) { + curl_slist_free_all(curllist); + kWarning(7103) << "Error" << curl_easy_strerror(curlresult); + error(KIO::ERR_COULD_NOT_CONNECT, curl_easy_strerror(curlresult)); return; } - kDebug(7103) << "Headers" << netreply->rawHeaderPairs(); - - const QByteArray httpmimetype = HTTPMIMEType(netreply->rawHeader("content-type")); - kDebug(7103) << "MIME type" << httpmimetype; - emit mimeType(httpmimetype); - - data(netreply->readAll()); - + curl_slist_free_all(curllist); finished(); } +void HttpProtocol::slotMIME() +{ + char *curlcontenttype = nullptr; + CURLcode curlresult = curl_easy_getinfo(m_curl, CURLINFO_CONTENT_TYPE, &curlcontenttype); + if (curlresult == CURLE_OK) { + const QByteArray httpmimetype = HTTPMIMEType(QByteArray(curlcontenttype)); + kDebug(7103) << "MIME type" << httpmimetype; + emit mimeType(httpmimetype); + } else { + kWarning(7103) << "Could not get info" << curl_easy_strerror(curlresult); + } + +} + +void HttpProtocol::slotData(const char* curldata, const size_t curldatasize) +{ + data(QByteArray::fromRawData(curldata, curldatasize)); +} + void HttpProtocol::slotProgress(qint64 received, qint64 total) { kDebug(7103) << "Received" << received << "from" << total; diff --git a/kioslave/http/http.h b/kioslave/http/http.h index 56e02846..11f38ded 100644 --- a/kioslave/http/http.h +++ b/kioslave/http/http.h @@ -24,6 +24,8 @@ #include #include +#include + class HttpProtocol : public QObject, public KIO::SlaveBase { Q_OBJECT @@ -34,8 +36,13 @@ public: virtual void get(const KUrl &url); -private Q_SLOTS: + void slotMIME(); + void slotData(const char* curldata, const size_t curldatasize); void slotProgress(qint64 received, qint64 total); + + bool firstchunk; +private: + CURL* m_curl; }; #endif // KDELIBS_HTTP_H