/* This file is part of the KDE project Copyright (C) 2010 Klarälvdalens Datakonsult AB, a KDAB Group company, info@kdab.net Author: Kevin Krammer, krake@kdab.com Copyright (C) 2011 Kevin Krammer, kevin.krammer@gmx.at This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "imapcachecollectionmigrator.h" #include "kmigratorbase.h" #include "libmaildir/maildir.h" #include "mixedmaildirstore.h" #include "subscriptionjob_p.h" #include "filestore/itemfetchjob.h" #include "filestore/itemdeletejob.h" #include "createandsettagsjob.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace Akonadi; using KPIM::Maildir; typedef QHash UidHash; class ImapCacheCollectionMigrator::Private { ImapCacheCollectionMigrator *const q; public: Private( ImapCacheCollectionMigrator *parent ) : q( parent ), mImportNewMessages( false ), mImportCachedMessages( false ), mRemoveDeletedMessages( false ), mDeleteImportedMessages( false ), mItemProgress( -1 ) { } bool isUnsubscribedImapFolder( const Collection &collection, QString &idPath ) const; public: Collection mCurrentCollection; Item::List mItems; UidHash mUidHash; QStringList mDeletedUids; bool mImportNewMessages; bool mImportCachedMessages; bool mRemoveDeletedMessages; bool mDeleteImportedMessages; KConfigGroup mCurrentFolderGroup; QHash mTagListHash; int mItemProgress; QSet mUnsubscribedImapFolders; Collection::List mUnsubscribedCollections; public: // slots void fetchItemsResult( KJob *job ); void processNextItem(); void processNextDeletedUid(); void fetchItemResult( KJob *job ); void itemCreateResult( KJob *job ); void itemDeletePhase1Result( KJob *job ); void itemDeletePhase2Result( KJob *job ); void cacheItemDeleteResult( KJob *job ); void unsubscribeCollections(); void unsubscribeCollectionsResult( KJob *job ); }; bool ImapCacheCollectionMigrator::Private::isUnsubscribedImapFolder( const Collection &collection, QString &idPath ) const { if ( collection.parentCollection() == Collection::root() ) { idPath = QString(); return false; } bool parentResult = isUnsubscribedImapFolder( collection.parentCollection(), idPath ); idPath = idPath + collection.remoteId(); return parentResult || mUnsubscribedImapFolders.contains( idPath ); } void ImapCacheCollectionMigrator::Private::fetchItemsResult( KJob *job ) { if ( job->error() != 0 ) { kWarning() << "Store Fetch for item list in collection" << mCurrentCollection.remoteId() << "returned error: code=" << job->error() << "text=" << job->errorString(); if ( mRemoveDeletedMessages ) { processNextDeletedUid(); } else { emit q->status( QString() ); mCurrentCollection = Collection(); q->collectionProcessed(); } return; } FileStore::ItemFetchJob *fetchJob = qobject_cast( job ); Q_ASSERT( fetchJob != 0 ); const QVariant uidHashVar = fetchJob->property( "remoteIdToIndexUid" ); if ( !uidHashVar.isValid() ) { kDebug( KDE_DEFAULT_DEBUG_AREA ) << "No UIDs from index for collection" << mCurrentCollection.name(); if ( mRemoveDeletedMessages ) { processNextDeletedUid(); } else { emit q->status( QString() ); mCurrentCollection = Collection(); q->collectionProcessed(); } return; } mUidHash.clear(); const QHash uidHash = uidHashVar.value< QHash >(); QHash::const_iterator it = uidHash.constBegin(); QHash::const_iterator endIt = uidHash.constEnd(); for ( ; it != endIt; ++it ) { const QString uid = it.value().isValid() ? it.value().value() : QString(); mUidHash.insert( it.key(), uid ); } mItems = fetchJob->items(); kDebug( KDE_DEFAULT_DEBUG_AREA ) << mItems.count() << "items for target collection" << mCurrentCollection.remoteId(); const QVariant tagListHashVar = fetchJob->property( "remoteIdToTagList" ); if ( !tagListHashVar.isValid() ) { kDebug( KDE_DEFAULT_DEBUG_AREA ) << "No tags from index for collection" << mCurrentCollection.name(); } else { mTagListHash = tagListHashVar.value< QHash >(); kDebug( KDE_DEFAULT_DEBUG_AREA ) << mTagListHash.count() << "items have tags"; } // filter out items we don't process later on const int oldCount = mItems.count(); Item::List::iterator itemIt = mItems.begin(); while ( itemIt != mItems.end() ) { const QString storeRemoteId = ( *itemIt ).remoteId(); if ( mImportCachedMessages || ( mImportNewMessages && !mUidHash.contains( storeRemoteId ) ) || mTagListHash.contains( storeRemoteId ) ) { ++itemIt; } else { itemIt = mItems.erase( itemIt ); } } if ( oldCount != mItems.count() ) { kDebug( KDE_DEFAULT_DEBUG_AREA ) << "After filtering:" << mItems.count() << "items remaining"; } mItemProgress = -1; emit q->progress( 0, mItems.count(), 0 ); processNextItem(); } void ImapCacheCollectionMigrator::Private::processNextItem() { // kDebug( KDE_DEFAULT_DEBUG_AREA ) << "mCurrentCollection=" << mCurrentCollection.name() // << mItems.count() << "items to go"; emit q->progress( ++mItemProgress ); emit q->status( i18ncp( "@info:status folder name and number of messages to import before finished", "%1: one message left to import", "%1: %2 messages left to import", mCurrentCollection.name(), mItems.count() ) ); if ( mItems.isEmpty() ) { if ( mDeletedUids.isEmpty() ) { emit q->status( QString() ); mCurrentCollection = Collection(); q->collectionProcessed(); } else if ( mRemoveDeletedMessages ) { processNextDeletedUid(); } return; } Item item = mItems.front(); mItems.pop_front(); // don't import items that are marked deleted. These come from normal/online IMAP caches // which didn't get their mbox cache files compacted Akonadi::MessageStatus status; status.setStatusFromFlags( item.flags() ); if ( status.isDeleted() ) { //kDebug() << "Cache item" << item.remoteId() << "is marked as Deleted. Skip it"; QMetaObject::invokeMethod( q, "processNextItem", Qt::QueuedConnection ); return; } const QString storeRemoteId = item.remoteId(); if ( mImportCachedMessages || ( mImportNewMessages && !mUidHash.contains( storeRemoteId ) ) ) { FileStore::ItemFetchJob *job = q->store()->fetchItem( item ); job->fetchScope().fetchFullPayload( true ); connect( job, SIGNAL(result(KJob*)), q, SLOT(fetchItemResult(KJob*)) ); } else if ( mTagListHash.contains( storeRemoteId ) ) { ItemCreateJob *createJob = 0; if ( !mUidHash.contains( storeRemoteId ) ) { item.setRemoteId( QString() ); createJob = new ItemCreateJob( item, mCurrentCollection ); } else { const QString uid = mUidHash[ storeRemoteId ]; item.setRemoteId( uid ); createJob = new ItemCreateJob( item, mCurrentCollection, q->hiddenSession() ); } createJob->setProperty( "storeRemoteId", storeRemoteId ); createJob->setProperty( "storeParentCollection", QVariant::fromValue( item.parentCollection() ) ); connect( createJob, SIGNAL(result(KJob*)), q, SLOT(itemCreateResult(KJob*)) ); } else { QMetaObject::invokeMethod( q, "processNextItem", Qt::QueuedConnection ); } } void ImapCacheCollectionMigrator::Private::processNextDeletedUid() { // kDebug( KDE_DEFAULT_DEBUG_AREA ) << "mCurrentCollection=" << mCurrentCollection.name() // << mDeletedUids.count() << "items to go"; if ( mDeletedUids.isEmpty() ) { if ( mCurrentFolderGroup.isValid() ) { const QString key = QLatin1String( "UIDSDeletedSinceLastSync" ); if ( !mCurrentFolderGroup.readEntry( key, QStringList() ).isEmpty() ) { mCurrentFolderGroup.deleteEntry( QLatin1String( "UIDSDeletedSinceLastSync" ) ); } } mCurrentFolderGroup = KConfigGroup(); mCurrentCollection = Collection(); emit q->status( QString() ); q->collectionProcessed(); return; } const QString uid = mDeletedUids.front(); mDeletedUids.pop_front(); // we need to first create an item using the hidden session so that Akonadi knows the item // and then delete it using the normal session so that the IMAP resource gets the delete Item item; item.setMimeType( KMime::Message::mimeType() ); item.setRemoteId( uid ); ItemCreateJob *createJob = new ItemCreateJob( item, mCurrentCollection, q->hiddenSession() ); connect( createJob, SIGNAL(result(KJob*)), q, SLOT(itemDeletePhase1Result(KJob*)) ); } void ImapCacheCollectionMigrator::Private::fetchItemResult( KJob *job ) { FileStore::ItemFetchJob *fetchJob = qobject_cast( job ); Q_ASSERT( fetchJob != 0 ); Item item = fetchJob->item(); if ( job->error() != 0 ) { kWarning() << "Store Fetch for single item" << item.remoteId() << "returned error: code=" << job->error() << "text=" << job->errorString(); processNextItem(); return; } const Item::List items = fetchJob->items(); if ( items.isEmpty() ) { kWarning() << "Store Fetch for single item" << item.remoteId() << "returned empty list"; } else { const Item cacheItem = items[ 0 ]; if ( !cacheItem.hasPayload() ) { kWarning() << "Store Fetch for single item" << item.remoteId() << "returned item without payload"; } else { item.setPayload( cacheItem.payload() ); } } ItemCreateJob *createJob = 0; const QString storeRemoteId = item.remoteId(); const QString uid = mUidHash[ item.remoteId() ]; if ( uid.isEmpty() && mImportNewMessages ) { item.setRemoteId( QString() ); createJob = new ItemCreateJob( item, mCurrentCollection ); // kDebug( KDE_DEFAULT_DEBUG_AREA ) << "unsynchronized cacheItem: remoteId=" << item.remoteId() // << "mimeType=" << item.mimeType() // << "flags=" << item.flags(); } else if ( mImportCachedMessages ) { item.setRemoteId( uid ); createJob = new ItemCreateJob( item, mCurrentCollection, q->hiddenSession() ); // kDebug( KDE_DEFAULT_DEBUG_AREA ) << "synchronized cacheItem: remoteId=" << item.remoteId() // << "mimeType=" << item.mimeType() // << "flags=" << item.flags(); } if ( createJob != 0 ) { createJob->setProperty( "storeRemoteId", storeRemoteId ); createJob->setProperty( "storeParentCollection", QVariant::fromValue( item.parentCollection() ) ); connect( createJob, SIGNAL(result(KJob*)), q, SLOT(itemCreateResult(KJob*)) ); } else { kDebug( KDE_DEFAULT_DEBUG_AREA ) << "Skipping cacheItem: remoteId=" << item.remoteId() << "mimeType=" << item.mimeType() << "flags=" << item.flags(); processNextItem(); } } void ImapCacheCollectionMigrator::Private::itemCreateResult( KJob *job ) { ItemCreateJob *createJob = qobject_cast( job ); Q_ASSERT( createJob != 0 ); const Item item = createJob->item(); const QString storeRemoteId = job->property( "storeRemoteId" ).value(); if ( job->error() != 0 ) { kWarning() << "Akonadi Create for single item" << item.remoteId() << "returned error: code=" << job->error() << "text=" << job->errorString(); processNextItem(); } else if ( !storeRemoteId.isEmpty() ) { const QStringList tagList = mTagListHash[ storeRemoteId ].value(); if ( !tagList.isEmpty() ) { kDebug( KDE_DEFAULT_DEBUG_AREA ) << "Tagging item" << item.url() << "with" << tagList; Akonadi::Tag::List tags; Q_FOREACH( const QString &tag, tagList ) { if ( tag.isEmpty() ) { kWarning() << "TagList for item" << item.url() << "contains an empty tag"; } else { tags << Akonadi::Tag(tag); } } new CreateAndSetTagsJob(item, tags); } const Collection storeCollection = job->property( "storeParentCollection" ).value(); if ( mDeleteImportedMessages && !storeCollection.remoteId().isEmpty() ) { Item cacheItem = item; cacheItem.setRemoteId( storeRemoteId ); cacheItem.setParentCollection( storeCollection ); FileStore::ItemDeleteJob *deleteJob = q->store()->deleteItem( cacheItem ); connect( deleteJob, SIGNAL(result(KJob*)), q, SLOT(cacheItemDeleteResult(KJob*)) ); } else { processNextItem(); } } else { processNextItem(); } } void ImapCacheCollectionMigrator::Private::itemDeletePhase1Result( KJob *job ) { ItemCreateJob *createJob = qobject_cast( job ); Q_ASSERT( createJob != 0 ); const Item item = createJob->item(); if ( job->error() != 0 ) { kWarning() << "Akonadi Create for single item" << item.remoteId() << "returned error: code=" << job->error() << "text=" << job->errorString(); processNextDeletedUid(); } else { ItemDeleteJob *deleteJob = new ItemDeleteJob( item ); connect( deleteJob, SIGNAL(result(KJob*)), q, SLOT(itemDeletePhase2Result(KJob*)) ); } } void ImapCacheCollectionMigrator::Private::itemDeletePhase2Result( KJob *job ) { ItemDeleteJob *deleteJob = qobject_cast( job ); Q_ASSERT( deleteJob != 0 ); const Item::List items = deleteJob->deletedItems(); const Item item = items.isEmpty() ? Item() : items[ 0 ]; if ( job->error() != 0 ) { kWarning() << "Akonadi Delete for single item" << item.remoteId() << "returned error: code=" << job->error() << "text=" << job->errorString(); } processNextDeletedUid(); } void ImapCacheCollectionMigrator::Private::cacheItemDeleteResult( KJob *job ) { FileStore::ItemDeleteJob *deleteJob = qobject_cast( job ); Q_ASSERT( deleteJob != 0 ); const Item item = deleteJob->item(); if ( job->error() != 0 ) { kWarning() << "Store Delete for single item" << item.remoteId() << "returned error: code=" << job->error() << "text=" << job->errorString(); } processNextItem(); } void ImapCacheCollectionMigrator::Private::unsubscribeCollections() { if ( !mUnsubscribedCollections.isEmpty() ) { kDebug( KDE_DEFAULT_DEBUG_AREA ) << "Locally Unsubscribe" << mUnsubscribedCollections.count() << "collections"; SubscriptionJob *job = new SubscriptionJob( q ); job->unsubscribe( mUnsubscribedCollections ); QObject::connect( job, SIGNAL(result(KJob*)), q, SLOT(unsubscribeCollectionsResult(KJob*)) ); } } void ImapCacheCollectionMigrator::Private::unsubscribeCollectionsResult( KJob *job ) { if ( job->error() != 0 ) { kError() << "Unsubscribing of " << mUnsubscribedCollections.count() << "collections failed:" << job->error(); } else { kDebug( KDE_DEFAULT_DEBUG_AREA ) << "Unsubscribing of " << mUnsubscribedCollections.count() << "collections succeeded"; } } ImapCacheCollectionMigrator::ImapCacheCollectionMigrator( const AgentInstance &resource, const QString &resourceName, MixedMaildirStore *store, QObject *parent ) : AbstractCollectionMigrator( resource, resourceName, store, parent ), d( new Private( this ) ) { connect( this, SIGNAL(migrationFinished(Akonadi::AgentInstance,QString)), SLOT(unsubscribeCollections()) ); } ImapCacheCollectionMigrator::~ImapCacheCollectionMigrator() { delete d; } void ImapCacheCollectionMigrator::setMigrationOptions( const MigrationOptions &options ) { MigrationOptions actualOptions = options; if ( store() == 0 ) { emit message( KMigratorBase::Skip, i18nc( "@info:status", "No cache for account %1 available", resourceName() ) ); kWarning() << "No store for folder" << topLevelFolder() << "so only config migration (instead of" << options << ") for" << resource().identifier() << resourceName(); actualOptions = ConfigOnly; } d->mImportNewMessages = actualOptions.testFlag( ImportNewMessages ); d->mImportCachedMessages = actualOptions.testFlag( ImportCachedMessages ); d->mRemoveDeletedMessages = actualOptions.testFlag( RemoveDeletedMessages ); d->mDeleteImportedMessages = actualOptions.testFlag( DeleteImportedMessages ); } ImapCacheCollectionMigrator::MigrationOptions ImapCacheCollectionMigrator::migrationOptions() const { MigrationOptions options; if ( d->mImportNewMessages ) { options |= ImportNewMessages; } if ( d->mImportCachedMessages ) { options |= ImportCachedMessages; } if ( d->mRemoveDeletedMessages ) { options |= RemoveDeletedMessages; } if ( d->mDeleteImportedMessages ) { options |= DeleteImportedMessages; } return options; } void ImapCacheCollectionMigrator::setUnsubscribedImapFolders( const QStringList &imapFolders ) { d->mUnsubscribedImapFolders.clear(); Q_FOREACH ( const QString &imapFolder, imapFolders ) { if ( imapFolder.endsWith( QLatin1Char( '/' ) ) ) { d->mUnsubscribedImapFolders << imapFolder.left( imapFolder.size() - 1 ); } else { d->mUnsubscribedImapFolders << imapFolder; } } kDebug( KDE_DEFAULT_DEBUG_AREA ) << "unsubscribed imap folders:" << d->mUnsubscribedImapFolders; } void ImapCacheCollectionMigrator::migrateCollection( const Collection &collection, const QString &folderId ) { QString imapIdPath; if ( d->isUnsubscribedImapFolder( collection, imapIdPath ) ) { kDebug( KDE_DEFAULT_DEBUG_AREA ) << "Collection id=" << collection.id() << ", remoteId=" << collection.remoteId() << ", imapIdPath=" << imapIdPath << "is locally unsubscribed"; // could check if this very collection is one of the unsubscribed using imapIdPath. // however, KMail treats subfolders of unsubscribed folders as unsubscribed as well // so unsubscribe the too. otherwise their contents get downloaded on first interval check d->mUnsubscribedCollections << collection; emit status( QString() ); collectionProcessed(); return; } if ( migrationOptions() == ConfigOnly ) { emit status( QString() ); collectionProcessed(); return; } // check that we don't get entered while we are still processing Q_ASSERT( !d->mCurrentCollection.isValid() ); Q_ASSERT( store() != 0 ); if ( collection.parentCollection() == Collection::root() ) { emit status( QString() ); collectionProcessed(); return; } kDebug( KDE_DEFAULT_DEBUG_AREA ) << "Akonadi collection remoteId=" << collection.remoteId() << ", parent=" << collection.parentCollection().remoteId(); Collection cache = currentStoreCollection(); kDebug( KDE_DEFAULT_DEBUG_AREA ) << "Cache collection remoteId=" << cache.remoteId() << ", parent=" << cache.parentCollection().remoteId(); kDebug( KDE_DEFAULT_DEBUG_AREA ) << "folderId=" << folderId << "imapIdPath=" << imapIdPath; d->mDeletedUids.clear(); if ( d->mRemoveDeletedMessages ) { Q_ASSERT( kmailConfig() ); const QString groupName = QLatin1String( "Folder-" ) + folderId; if ( kmailConfig()->hasGroup( groupName ) ) { d->mCurrentFolderGroup = KConfigGroup( kmailConfig(), groupName ); d->mDeletedUids = d->mCurrentFolderGroup.readEntry( QLatin1String( "UIDSDeletedSinceLastSync" ), QStringList() ); } kDebug( KDE_DEFAULT_DEBUG_AREA ) << "DeleteUids=" << d->mDeletedUids; } d->mCurrentCollection = collection; d->mTagListHash.clear(); emit message( KMigratorBase::Info, i18nc( "@info:status", "Starting cache migration for folder %1 of account %2", collection.name(), resourceName() ) ); emit status( collection.name() ); if ( d->mImportNewMessages || d->mImportCachedMessages ) { FileStore::ItemFetchJob *job = store()->fetchItems( cache ); connect( job, SIGNAL(result(KJob*)), SLOT(fetchItemsResult(KJob*)) ); emit status( i18nc( "@info:status foldername", "%1: listing messages...", collection.name() ) ); } else if ( d->mRemoveDeletedMessages ) { emit status( collection.name() ); d->processNextDeletedUid(); } else { emit status( QString() ); collectionProcessed(); } } void ImapCacheCollectionMigrator::migrationProgress( int processedCollections, int seenCollections ) { // if we potentially migrate items, use item progress instead // use base implementation if only collections are processed if ( migrationOptions() == ConfigOnly ) { AbstractCollectionMigrator::migrationProgress( processedCollections, seenCollections ); } } QString ImapCacheCollectionMigrator::mapRemoteIdFromStore( const QString &storeRemotedId ) const { const QString accountId = topLevelFolder(); const KConfigGroup accountGroup = kmailConfig()->group( QString::fromLatin1( "Account %1" ).arg( accountId ) ); const QString folderId = currentStoreFolderId(); const KConfigGroup folderGroup = kmailConfig()->group( QString::fromLatin1( "Folder-%1" ).arg( folderId ) ); QString imapPath = folderGroup.readEntry( "ImapPath", QString() ); imapPath.remove( 0, 1 ); imapPath.chop( 1 ); QChar separator = QLatin1Char('/'); int namespaceLength = -1; for ( int i=0; i<=2; i++ ) { QStringList namespaces = accountGroup.readEntry( QString::number( i ), QStringList() ); namespaces.replaceInStrings( QRegExp( QLatin1String("\"") ), QLatin1String("") ); foreach ( const QString &ns, namespaces ) { QString imapNs = ns; imapNs.chop( 1 ); if ( ( ns.size()>namespaceLength ) && imapPath.startsWith( imapNs ) ) { const QString potentialSeparator = accountGroup.readEntry( QString::fromLatin1( "Namespace:%1" ).arg( ns ), QString() ); if ( !potentialSeparator.isEmpty() ) { namespaceLength = ns.size(); separator = potentialSeparator.at( 0 ); } } } } return separator + storeRemotedId; } #include "moc_imapcachecollectionmigrator.cpp" // kate: space-indent on; indent-width 2; replace-tabs on;