Katana_mirror/kdelibs
1
0
Fork 0
mirror of https://bitbucket.org/smil3y/kdelibs.git synced 2025-02-23 10:22:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

kioslave: reimplement HTTP(S) slave via curl

Browse source 
Signed-off-by: Ivailo Monev <xakepa10@gmail.com>
This commit is contained in:
Ivailo Monev 2022-02-19 16:40:06 +02:00
parent 68f3abe724
commit 69af879278
8 changed files with 160 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/codeql-analysis.yml vendored
View file

@ -42,7 +42,7 @@ jobs:
sudo apt-get install -qq wget
sudo wget https://raw.githubusercontent.com/fluxer/katana-ubuntu/master/katana.list -O /etc/apt/sources.list.d/katana.list
sudo apt-get update -qq
sudo apt-get install -qq cmake katie-dev libenchant-dev libmagick++-dev libmpv-dev xorg-dev mesa-common-dev libavahi-common-dev libwebp-dev libudev-dev liblzma-dev libexiv2-dev libbz2-dev libattr1-dev libacl1-dev libcdio-dev strigi libdbusmenu-katie media-player-info shared-mime-info media-player-info xdg-utils
sudo apt-get install -qq cmake katie-dev libenchant-dev libmagick++-dev libmpv-dev xorg-dev mesa-common-dev libavahi-common-dev libwebp-dev libudev-dev liblzma-dev libexiv2-dev libbz2-dev libattr1-dev libacl1-dev libcdio-dev libcurl4-openssl-dev strigi libdbusmenu-katie media-player-info shared-mime-info media-player-info xdg-utils
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL

16
CMakeLists.txt
View file

@ -69,6 +69,14 @@ set_package_properties(ENCHANT PROPERTIES
PURPOSE "Spell checking support via Enchant"
)
find_package(Curl)
set_package_properties(Curl PROPERTIES
DESCRIPTION "Command line tool and library for transferring data with URLs"
URL "https://curl.se/"
TYPE REQUIRED
PURPOSE "KIO HTTP(S) slave"
)
# optional features
if(X11_FOUND)
macro_bool_to_01(X11_XTest_FOUND HAVE_XTEST)
@ -88,14 +96,6 @@ if(X11_FOUND)
endif(NOT HAVE_XSYNC AND NOT HAVE_XSCREENSAVER)
endif(X11_FOUND)
macro_optional_find_package(OpenSSL)
set_package_properties(OpenSSL PROPERTIES
DESCRIPTION "Support for secure network communications (SSL and TLS)"
URL "http://openssl.org"
TYPE RECOMMENDED
PURPOSE "For the bulk of secure communications, including secure web browsing via HTTPS"
)
macro_optional_find_package(Libintl)
set_package_properties(Libintl PROPERTIES
DESCRIPTION "Support for multiple languages"

6
appveyor.yml
View file

@ -21,9 +21,9 @@ build_script:
sudo apt-get install -qq cmake katie-dev libenchant-dev \
libmagick++-dev libmpv-dev xorg-dev mesa-common-dev \
libavahi-common-dev libwebp-dev libudev-dev liblzma-dev \
libexiv2-dev libbz2-dev libacl1-dev libcdio-dev strigi \
libdbusmenu-katie media-player-info shared-mime-info \
media-player-info xdg-utils ccache
libexiv2-dev libbz2-dev libacl1-dev libcdio-dev \
libcurl4-openssl-dev strigi libdbusmenu-katie media-player-info \
shared-mime-info media-player-info xdg-utils ccache
export PATH="/usr/lib/ccache/:$PATH"

1
cmake/modules/CMakeLists.txt
View file

@ -14,6 +14,7 @@ set(cmakeFilesDontInstall
FindACL.cmake
FindLibCDIO.cmake
FindDevinfo.cmake
FindCurl.cmake
)
# Explicitly list all files which will be installed.

42
cmake/modules/FindCurl.cmake Normal file
View file

@ -0,0 +1,42 @@
# Try to find curl library, once done this will define:
#
# CURL_FOUND - system has curl
# CURL_INCLUDE_DIR - the curl include directory
# CURL_LIBRARIES - the libraries needed to use curl
# CURL_DEFINITIONS - compiler switches required for using curl
#
# Copyright (c) 2022 Ivailo Monev <xakepa10@gmail.com>
#
# Redistribution and use is allowed according to the terms of the BSD license.
# For details see the accompanying COPYING-CMAKE-SCRIPTS file.
if(NOT WIN32)
include(FindPkgConfig)
pkg_check_modules(PC_CURL QUIET curl)
set(CURL_INCLUDE_DIR ${PC_CURL_INCLUDE_DIRS})
set(CURL_LIBRARIES ${PC_CURL_LIBRARIES})
endif()
set(CURL_VERSION ${PC_CURL_VERSION})
set(CURL_DEFINITIONS ${PC_CURL_CFLAGS_OTHER})
if(NOT CURL_INCLUDE_DIR OR NOT CURL_LIBRARIES)
find_path(CURL_INCLUDE_DIR
NAMES curl/curl.h
HINTS $ENV{CURLDIR}/include
)
find_library(CURL_LIBRARIES
NAMES curl
HINTS $ENV{CURLDIR}/lib
)
endif()
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Curl
VERSION_VAR CURL_VERSION
REQUIRED_VARS CURL_LIBRARIES CURL_INCLUDE_DIR
)
mark_as_advanced(CURL_INCLUDE_DIR CURL_LIBRARIES)

11
kioslave/http/CMakeLists.txt
View file

@ -1,12 +1,19 @@
project(kioslave-http)
include_directories(${KDE4_KIO_INCLUDES})
include_directories(
${KDE4_KIO_INCLUDES}
${CURL_INCLUDES}
)
########### next target ###############
kde4_add_plugin(kio_http http.cpp)
target_link_libraries(kio_http ${KDE4_KDECORE_LIBS} ${KDE4_KIO_LIBS})
target_link_libraries(kio_http
${KDE4_KDECORE_LIBS}
${KDE4_KIO_LIBS}
${CURL_LIBRARIES}
)
install(TARGETS kio_http DESTINATION ${KDE4_PLUGIN_INSTALL_DIR})

141
kioslave/http/http.cpp
View file

@ -22,9 +22,6 @@
#include <QCoreApplication>
#include <QThread>
#include <QNetworkAccessManager>
#include <QNetworkDiskCache>
#include <QNetworkReply>
#include <sys/types.h>
#include <unistd.h>
@ -38,6 +35,33 @@ static inline QByteArray HTTPMIMEType(const QByteArray &contenttype)
return splitcontenttype.at(0);
}
size_t curlWriteCallback(char *ptr, size_t size, size_t nmemb, void *userdata)
{
HttpProtocol* httpprotocol = static_cast<HttpProtocol*>(userdata);
if (!httpprotocol) {
return 0;
}
// emit MIME before data
if (httpprotocol->firstchunk) {
httpprotocol->slotMIME();
httpprotocol->firstchunk = false;
}
// size is always 1
Q_UNUSED(size);
httpprotocol->slotData(ptr, nmemb);
return nmemb;
}
int curlProgressCallback(void *userdata, double dltotal, double dlnow, double ultotal, double ulnow)
{
HttpProtocol* httpprotocol = static_cast<HttpProtocol*>(userdata);
if (!httpprotocol) {
return CURLE_BAD_FUNCTION_ARGUMENT;
}
httpprotocol->slotProgress(dlnow, dltotal);
return CURLE_OK;
}
extern "C" int Q_DECL_EXPORT kdemain(int argc, char **argv)
{
QCoreApplication app(argc, argv);
@ -59,85 +83,96 @@ extern "C" int Q_DECL_EXPORT kdemain(int argc, char **argv)
}
HttpProtocol::HttpProtocol(const QByteArray &pool, const QByteArray &app)
: SlaveBase("http", pool, app)
: SlaveBase("http", pool, app), firstchunk(true), m_curl(nullptr)
{
m_curl = curl_easy_init();
if (!m_curl) {
kWarning(7103) << "Could not create context";
return;
}
curl_easy_setopt(m_curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(m_curl, CURLOPT_MAXREDIRS, 10L);
curl_easy_setopt(m_curl, CURLOPT_TIMEOUT, 30L);
curl_easy_setopt(m_curl, CURLOPT_WRITEDATA, this);
curl_easy_setopt(m_curl, CURLOPT_WRITEFUNCTION, curlWriteCallback);
curl_easy_setopt(m_curl, CURLOPT_NOPROGRESS, 0L); // otherwise the progress callback is not called
curl_easy_setopt(m_curl, CURLOPT_PROGRESSDATA, this);
curl_easy_setopt(m_curl, CURLOPT_PROGRESSFUNCTION, curlProgressCallback);
}
HttpProtocol::~HttpProtocol()
{
if (m_curl) {
curl_easy_cleanup(m_curl);
}
}
void HttpProtocol::get(const KUrl &url)
{
kDebug(7103) << "URL" << url.prettyUrl();
QNetworkAccessManager netmanager(this);
QNetworkDiskCache netcache(this);
QNetworkRequest netrequest(url);
if (Q_UNLIKELY(!m_curl)) {
error(KIO::ERR_OUT_OF_MEMORY, QString::fromLatin1("Null context"));
return;
}
firstchunk = true;
const QByteArray urlbytes = url.prettyUrl().toLocal8Bit();
curl_easy_setopt(m_curl, CURLOPT_URL, urlbytes.constData());
kDebug(7103) << "Metadata" << allMetaData();
struct curl_slist *curllist = NULL;
// metadata from scheduler
if (hasMetaData("Languages")) {
netrequest.setRawHeader("Accept-Language", metaData("Languages").toAscii());
curllist = curl_slist_append(curllist, QByteArray("Accept-Language: ") + metaData("Languages").toAscii());
}
if (hasMetaData("Charsets")) {
netrequest.setRawHeader("Accept-Charset", metaData("Charsets").toAscii());
}
if (hasMetaData("CacheDir")) {
netcache.setCacheDirectory(metaData("CacheDir"));
curllist = curl_slist_append(curllist, QByteArray("Accept-Charset: ") + metaData("Charsets").toAscii());
}
if (hasMetaData("UserAgent")) {
netrequest.setRawHeader("User-Agent", metaData("UserAgent").toAscii());
const QByteArray useragentbytes = metaData("UserAgent").toAscii();
curl_easy_setopt(m_curl, CURLOPT_USERAGENT, useragentbytes.constData());
}
netmanager.setCache(&netcache);
// KIO metadata
if (hasMetaData("cache")) {
const QByteArray kiocache = metaData("cache").toAscii();
if (qstricmp(kiocache.constData(), "cache") == 0) {
netrequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QVariant(QNetworkRequest::PreferCache));
} else if (qstricmp(kiocache.constData(), "cacheonly") == 0) {
netrequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QVariant(QNetworkRequest::AlwaysCache));
} else if (qstricmp(kiocache.constData(), "verify") == 0 || qstricmp(kiocache.constData(), "refresh") == 0) {
netrequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QVariant(QNetworkRequest::PreferNetwork));
} else {
// reload or unknown
netrequest.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QVariant(QNetworkRequest::AlwaysNetwork));
}
}
QNetworkReply* netreply = netmanager.get(netrequest);
connect(netreply, SIGNAL(downloadProgress(qint64,qint64)), this, SLOT(slotProgress(qint64,qint64)));
while (!netreply->isFinished()) {
QCoreApplication::processEvents();
QThread::msleep(400);
}
if (netreply->error() != QNetworkReply::NoError) {
kWarning(7103) << "Error" << netreply->url() << netreply->error();
error(KIO::ERR_COULD_NOT_CONNECT, url.prettyUrl());
CURLcode curlresult = curl_easy_setopt(m_curl, CURLOPT_HTTPHEADER, curllist);
if (curlresult != CURLE_OK) {
curl_slist_free_all(curllist);
kWarning(7103) << "Error" << curl_easy_strerror(curlresult);
error(KIO::ERR_COULD_NOT_CONNECT, curl_easy_strerror(curlresult));
return;
}
const QVariant netredirect = netreply->attribute(QNetworkRequest::RedirectionTargetAttribute);
if (netredirect.isValid()) {
const QUrl netredirecturl = netreply->url().resolved(netredirect.toUrl());
kDebug(7103) << "Redirecting to" << netredirecturl;
redirection(netredirecturl);
finished();
curlresult = curl_easy_perform(m_curl);
if (curlresult != CURLE_OK) {
curl_slist_free_all(curllist);
kWarning(7103) << "Error" << curl_easy_strerror(curlresult);
error(KIO::ERR_COULD_NOT_CONNECT, curl_easy_strerror(curlresult));
return;
}
kDebug(7103) << "Headers" << netreply->rawHeaderPairs();
const QByteArray httpmimetype = HTTPMIMEType(netreply->rawHeader("content-type"));
kDebug(7103) << "MIME type" << httpmimetype;
emit mimeType(httpmimetype);
data(netreply->readAll());
curl_slist_free_all(curllist);
finished();
}
void HttpProtocol::slotMIME()
{
char *curlcontenttype = nullptr;
CURLcode curlresult = curl_easy_getinfo(m_curl, CURLINFO_CONTENT_TYPE, &curlcontenttype);
if (curlresult == CURLE_OK) {
const QByteArray httpmimetype = HTTPMIMEType(QByteArray(curlcontenttype));
kDebug(7103) << "MIME type" << httpmimetype;
emit mimeType(httpmimetype);
} else {
kWarning(7103) << "Could not get info" << curl_easy_strerror(curlresult);
}
}
void HttpProtocol::slotData(const char* curldata, const size_t curldatasize)
{
data(QByteArray::fromRawData(curldata, curldatasize));
}
void HttpProtocol::slotProgress(qint64 received, qint64 total)
{
kDebug(7103) << "Received" << received << "from" << total;

9
kioslave/http/http.h
View file

@ -24,6 +24,8 @@
#include <kurl.h>
#include <kio/slavebase.h>
#include <curl/curl.h>
class HttpProtocol : public QObject, public KIO::SlaveBase
{
Q_OBJECT
@ -34,8 +36,13 @@ public:
virtual void get(const KUrl &url);
private Q_SLOTS:
void slotMIME();
void slotData(const char* curldata, const size_t curldatasize);
void slotProgress(qint64 received, qint64 total);
bool firstchunk;
private:
CURL* m_curl;
};
#endif // KDELIBS_HTTP_H