qBittorrent
rss_feed.cpp
Go to the documentation of this file.
1 /*
2  * Bittorrent Client using Qt and libtorrent.
3  * Copyright (C) 2015, 2017 Vladimir Golovnev <glassez@yandex.ru>
4  * Copyright (C) 2010 Christophe Dumez <chris@qbittorrent.org>
5  * Copyright (C) 2010 Arnaud Demaiziere <arnaud@qbittorrent.org>
6  *
7  * This program is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU General Public License
9  * as published by the Free Software Foundation; either version 2
10  * of the License, or (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20  *
21  * In addition, as a special exception, the copyright holders give permission to
22  * link this program with the OpenSSL project's "OpenSSL" library (or with
23  * modified versions of it that use the same license as the "OpenSSL" library),
24  * and distribute the linked executables. You must obey the GNU General Public
25  * License in all respects for all of the code used other than "OpenSSL". If you
26  * modify file(s), you may extend this exception to your version of the file(s),
27  * but you are not obligated to do so. If you do not wish to do so, delete this
28  * exception statement from your version.
29  */
30 
31 #include "rss_feed.h"
32 
33 #include <algorithm>
34 #include <utility>
35 #include <vector>
36 
37 #include <QDir>
38 #include <QJsonArray>
39 #include <QJsonDocument>
40 #include <QJsonObject>
41 #include <QJsonValue>
42 #include <QUrl>
43 
44 #include "base/asyncfilestorage.h"
45 #include "base/global.h"
46 #include "base/logger.h"
48 #include "base/profile.h"
49 #include "base/utils/fs.h"
50 #include "rss_article.h"
51 #include "rss_parser.h"
52 #include "rss_session.h"
53 
54 const QString KEY_UID(QStringLiteral("uid"));
55 const QString KEY_URL(QStringLiteral("url"));
56 const QString KEY_TITLE(QStringLiteral("title"));
57 const QString KEY_LASTBUILDDATE(QStringLiteral("lastBuildDate"));
58 const QString KEY_ISLOADING(QStringLiteral("isLoading"));
59 const QString KEY_HASERROR(QStringLiteral("hasError"));
60 const QString KEY_ARTICLES(QStringLiteral("articles"));
61 
62 using namespace RSS;
63 
64 Feed::Feed(const QUuid &uid, const QString &url, const QString &path, Session *session)
65  : Item(path)
66  , m_session(session)
67  , m_uid(uid)
68  , m_url(url)
69 {
70  const auto uidHex = QString::fromLatin1(m_uid.toRfc4122().toHex());
71  m_dataFileName = uidHex + QLatin1String(".json");
72 
73  // Move to new file naming scheme (since v4.1.2)
74  const QString legacyFilename
75  {Utils::Fs::toValidFileSystemName(m_url, false, QLatin1String("_"))
76  + QLatin1String(".json")};
77  const QDir storageDir {m_session->dataFileStorage()->storageDir()};
78  if (!QFile::exists(storageDir.absoluteFilePath(m_dataFileName)))
79  QFile::rename(storageDir.absoluteFilePath(legacyFilename), storageDir.absoluteFilePath(m_dataFileName));
80 
81  m_iconPath = Utils::Fs::toUniformPath(storageDir.absoluteFilePath(uidHex + QLatin1String(".ico")));
82 
84  m_parser->moveToThread(m_session->workingThread());
85  connect(this, &Feed::destroyed, m_parser, &Private::Parser::deleteLater);
87 
89 
91  downloadIcon();
92  else
94 
96 
97  load();
98 }
99 
101 {
102  emit aboutToBeDestroyed(this);
103 }
104 
105 QList<Article *> Feed::articles() const
106 {
107  return m_articlesByDate;
108 }
109 
111 {
112  const int oldUnreadCount = m_unreadCount;
113  for (Article *article : asConst(m_articles))
114  {
115  if (!article->isRead())
116  {
117  article->disconnect(this);
118  article->markAsRead();
119  --m_unreadCount;
120  emit articleRead(article);
121  }
122  }
123 
124  if (m_unreadCount != oldUnreadCount)
125  {
126  m_dirty = true;
127  store();
128  emit unreadCountChanged(this);
129  }
130 }
131 
133 {
134  if (m_downloadHandler)
136 
137  // NOTE: Should we allow manually refreshing for disabled session?
138 
141 
142  if (!QFile::exists(m_iconPath))
143  downloadIcon();
144 
145  m_isLoading = true;
146  emit stateChanged(this);
147 }
148 
149 QUuid Feed::uid() const
150 {
151  return m_uid;
152 }
153 
154 QString Feed::url() const
155 {
156  return m_url;
157 }
158 
159 QString Feed::title() const
160 {
161  return m_title;
162 }
163 
164 bool Feed::isLoading() const
165 {
166  return m_isLoading;
167 }
168 
169 QString Feed::lastBuildDate() const
170 {
171  return m_lastBuildDate;
172 }
173 
174 int Feed::unreadCount() const
175 {
176  return m_unreadCount;
177 }
178 
179 Article *Feed::articleByGUID(const QString &guid) const
180 {
181  return m_articles.value(guid);
182 }
183 
185 {
186  while (m_articlesByDate.size() > n)
188  // We don't need store articles here
189 }
190 
192 {
193  if (result.status == Net::DownloadStatus::Success)
194  {
195  emit iconLoaded(this);
196  }
197 }
198 
199 bool Feed::hasError() const
200 {
201  return m_hasError;
202 }
203 
205 {
206  m_downloadHandler = nullptr; // will be deleted by DownloadManager later
207 
208  if (result.status == Net::DownloadStatus::Success)
209  {
210  LogMsg(tr("RSS feed at '%1' is successfully downloaded. Starting to parse it.")
211  .arg(result.url));
212  // Parse the download RSS
213  m_parser->parse(result.data);
214  }
215  else
216  {
217  m_isLoading = false;
218  m_hasError = true;
219 
220  LogMsg(tr("Failed to download RSS feed at '%1'. Reason: %2")
221  .arg(result.url, result.errorString), Log::WARNING);
222 
223  emit stateChanged(this);
224  }
225 }
226 
228 {
229  m_hasError = !result.error.isEmpty();
230 
231  if (!result.title.isEmpty() && (title() != result.title))
232  {
233  m_title = result.title;
234  m_dirty = true;
235  emit titleChanged(this);
236  }
237 
238  if (!result.lastBuildDate.isEmpty())
239  {
241  m_dirty = true;
242  }
243 
244  // For some reason, the RSS feed may contain malformed XML data and it may not be
245  // successfully parsed by the XML parser. We are still trying to load as many articles
246  // as possible until we encounter corrupted data. So we can have some articles here
247  // even in case of parsing error.
248  const int newArticlesCount = updateArticles(result.articles);
249  store();
250 
251  if (m_hasError)
252  {
253  LogMsg(tr("Failed to parse RSS feed at '%1'. Reason: %2").arg(m_url, result.error)
254  , Log::WARNING);
255  }
256  LogMsg(tr("RSS feed at '%1' updated. Added %2 new articles.")
257  .arg(url(), QString::number(newArticlesCount)));
258 
259  m_isLoading = false;
260  emit stateChanged(this);
261 }
262 
264 {
265  QFile file(m_session->dataFileStorage()->storageDir().absoluteFilePath(m_dataFileName));
266 
267  if (!file.exists())
268  {
270  m_dirty = true;
271  store(); // convert to new format
272  }
273  else if (file.open(QFile::ReadOnly))
274  {
275  loadArticles(file.readAll());
276  file.close();
277  }
278  else
279  {
280  LogMsg(tr("Couldn't read RSS Session data from %1. Error: %2")
281  .arg(m_dataFileName, file.errorString())
282  , Log::WARNING);
283  }
284 }
285 
286 void Feed::loadArticles(const QByteArray &data)
287 {
288  QJsonParseError jsonError;
289  const QJsonDocument jsonDoc = QJsonDocument::fromJson(data, &jsonError);
290  if (jsonError.error != QJsonParseError::NoError)
291  {
292  LogMsg(tr("Couldn't parse RSS Session data. Error: %1").arg(jsonError.errorString())
293  , Log::WARNING);
294  return;
295  }
296 
297  if (!jsonDoc.isArray())
298  {
299  LogMsg(tr("Couldn't load RSS Session data. Invalid data format."), Log::WARNING);
300  return;
301  }
302 
303  const QJsonArray jsonArr = jsonDoc.array();
304  int i = -1;
305  for (const QJsonValue &jsonVal : jsonArr)
306  {
307  ++i;
308  if (!jsonVal.isObject())
309  {
310  LogMsg(tr("Couldn't load RSS article '%1#%2'. Invalid data format.").arg(m_url).arg(i)
311  , Log::WARNING);
312  continue;
313  }
314 
315  try
316  {
317  auto article = new Article(this, jsonVal.toObject());
318  if (!addArticle(article))
319  delete article;
320  }
321  catch (const RuntimeError &) {}
322  }
323 }
324 
326 {
327  const SettingsPtr qBTRSSFeeds = Profile::instance()->applicationSettings(QStringLiteral("qBittorrent-rss-feeds"));
328  const QVariantHash allOldItems = qBTRSSFeeds->value("old_items").toHash();
329 
330  for (const QVariant &var : asConst(allOldItems.value(m_url).toList()))
331  {
332  auto hash = var.toHash();
333  // update legacy keys
334  hash[Article::KeyLink] = hash.take(QLatin1String("news_link"));
335  hash[Article::KeyTorrentURL] = hash.take(QLatin1String("torrent_url"));
336  hash[Article::KeyIsRead] = hash.take(QLatin1String("read"));
337  try
338  {
339  auto article = new Article(this, hash);
340  if (!addArticle(article))
341  delete article;
342  }
343  catch (const RuntimeError &) {}
344  }
345 }
346 
348 {
349  if (!m_dirty) return;
350 
351  m_dirty = false;
352  m_savingTimer.stop();
353 
354  QJsonArray jsonArr;
355  for (Article *article :asConst(m_articles))
356  jsonArr << article->toJsonObject();
357 
358  m_session->dataFileStorage()->store(m_dataFileName, QJsonDocument(jsonArr).toJson());
359 }
360 
362 {
363  if (!m_savingTimer.isActive())
364  m_savingTimer.start(5 * 1000, this);
365 }
366 
367 bool Feed::addArticle(Article *article)
368 {
369  Q_ASSERT(article);
370  Q_ASSERT(!m_articles.contains(article->guid()));
371 
372  // Insertion sort
373  const int maxArticles = m_session->maxArticlesPerFeed();
374  const auto lowerBound = std::lower_bound(m_articlesByDate.begin(), m_articlesByDate.end()
375  , article->date(), Article::articleDateRecentThan);
376  if ((lowerBound - m_articlesByDate.begin()) >= maxArticles)
377  return false; // we reach max articles
378 
379  m_articles[article->guid()] = article;
380  m_articlesByDate.insert(lowerBound, article);
381  if (!article->isRead())
382  {
384  connect(article, &Article::read, this, &Feed::handleArticleRead);
385  }
386 
387  m_dirty = true;
388  emit newArticle(article);
389 
390  if (m_articlesByDate.size() > maxArticles)
392 
393  return true;
394 }
395 
397 {
398  auto oldestArticle = m_articlesByDate.last();
399  emit articleAboutToBeRemoved(oldestArticle);
400 
401  m_articles.remove(oldestArticle->guid());
402  m_articlesByDate.removeLast();
403  const bool isRead = oldestArticle->isRead();
404  delete oldestArticle;
405 
406  if (!isRead)
408 }
409 
411 {
412  ++m_unreadCount;
413  emit unreadCountChanged(this);
414 }
415 
417 {
418  Q_ASSERT(m_unreadCount > 0);
419 
420  --m_unreadCount;
421  emit unreadCountChanged(this);
422 }
423 
425 {
426  // Download the RSS Feed icon
427  // XXX: This works for most sites but it is not perfect
428  const QUrl url(m_url);
429  const auto iconUrl = QString::fromLatin1("%1://%2/favicon.ico").arg(url.scheme(), url.host());
431  Net::DownloadRequest(iconUrl).saveToFile(true).destFileName(m_iconPath)
433 }
434 
435 int Feed::updateArticles(const QList<QVariantHash> &loadedArticles)
436 {
437  if (loadedArticles.empty())
438  return 0;
439 
440  QDateTime dummyPubDate {QDateTime::currentDateTime()};
441  QVector<QVariantHash> newArticles;
442  newArticles.reserve(loadedArticles.size());
443  for (QVariantHash article : loadedArticles)
444  {
445  // If article has no publication date we use feed update time as a fallback.
446  // To prevent processing of "out-of-limit" articles we must not assign dates
447  // that are earlier than the dates of existing articles.
448  const Article *existingArticle = articleByGUID(article[Article::KeyId].toString());
449  if (existingArticle)
450  {
451  dummyPubDate = existingArticle->date().addMSecs(-1);
452  continue;
453  }
454 
455  QVariant &articleDate = article[Article::KeyDate];
456  if (!articleDate.toDateTime().isValid())
457  articleDate = dummyPubDate;
458 
459  newArticles.append(article);
460  }
461 
462  if (newArticles.empty())
463  return 0;
464 
465  using ArticleSortAdaptor = std::pair<QDateTime, const QVariantHash *>;
466  std::vector<ArticleSortAdaptor> sortData;
467  const QList<Article *> existingArticles = articles();
468  sortData.reserve(existingArticles.size() + newArticles.size());
469  std::transform(existingArticles.begin(), existingArticles.end(), std::back_inserter(sortData)
470  , [](const Article *article)
471  {
472  return std::make_pair(article->date(), nullptr);
473  });
474  std::transform(newArticles.begin(), newArticles.end(), std::back_inserter(sortData)
475  , [](const QVariantHash &article)
476  {
477  return std::make_pair(article[Article::KeyDate].toDateTime(), &article);
478  });
479 
480  // Sort article list in reverse chronological order
481  std::sort(sortData.begin(), sortData.end()
482  , [](const ArticleSortAdaptor &a1, const ArticleSortAdaptor &a2)
483  {
484  return (a1.first > a2.first);
485  });
486 
487  if (sortData.size() > static_cast<uint>(m_session->maxArticlesPerFeed()))
488  sortData.resize(m_session->maxArticlesPerFeed());
489 
490  int newArticlesCount = 0;
491  std::for_each(sortData.crbegin(), sortData.crend(), [this, &newArticlesCount](const ArticleSortAdaptor &a)
492  {
493  if (a.second)
494  {
495  addArticle(new Article {this, *a.second});
496  ++newArticlesCount;
497  }
498  });
499 
500  return newArticlesCount;
501 }
502 
503 QString Feed::iconPath() const
504 {
505  return m_iconPath;
506 }
507 
508 QJsonValue Feed::toJsonValue(const bool withData) const
509 {
510  QJsonObject jsonObj;
511  jsonObj.insert(KEY_UID, uid().toString());
512  jsonObj.insert(KEY_URL, url());
513 
514  if (withData)
515  {
516  jsonObj.insert(KEY_TITLE, title());
517  jsonObj.insert(KEY_LASTBUILDDATE, lastBuildDate());
518  jsonObj.insert(KEY_ISLOADING, isLoading());
519  jsonObj.insert(KEY_HASERROR, hasError());
520 
521  QJsonArray jsonArr;
522  for (Article *article : asConst(m_articles))
523  jsonArr << article->toJsonObject();
524  jsonObj.insert(KEY_ARTICLES, jsonArr);
525  }
526 
527  return jsonObj;
528 }
529 
531 {
532  if (enabled)
533  {
534  downloadIcon();
537  }
538 }
539 
541 {
542  article->disconnect(this);
544  emit articleRead(article);
545  // will be stored deferred
546  m_dirty = true;
547  storeDeferred();
548 }
549 
551 {
554 }
555 
556 void Feed::timerEvent(QTimerEvent *event)
557 {
558  Q_UNUSED(event);
559  store();
560 }
QDir storageDir() const
void store(const QString &fileName, const QByteArray &data)
virtual void cancel()=0
void finished(const DownloadResult &result)
void registerSequentialService(const ServiceID &serviceID)
DownloadHandler * download(const DownloadRequest &downloadRequest)
static DownloadManager * instance()
SettingsPtr applicationSettings(const QString &name) const
Definition: profile.cpp:109
static const Profile * instance()
Definition: profile.cpp:67
Definition: rss_article.h:43
static const QString KeyIsRead
Definition: rss_article.h:60
static const QString KeyDate
Definition: rss_article.h:54
QString guid() const
Definition: rss_article.cpp:82
void read(Article *article=nullptr)
static bool articleDateRecentThan(const Article *article, const QDateTime &date)
static const QString KeyLink
Definition: rss_article.h:59
static const QString KeyId
Definition: rss_article.h:53
static const QString KeyTorrentURL
Definition: rss_article.h:58
QDateTime date() const
Definition: rss_article.cpp:87
bool isRead() const
Article * articleByGUID(const QString &guid) const
Definition: rss_feed.cpp:179
QJsonValue toJsonValue(bool withData=false) const override
Definition: rss_feed.cpp:508
QString m_dataFileName
Definition: rss_feed.h:126
QList< Article * > articles() const override
Definition: rss_feed.cpp:105
QHash< QString, Article * > m_articles
Definition: rss_feed.h:122
void removeOldestArticle()
Definition: rss_feed.cpp:396
void markAsRead() override
Definition: rss_feed.cpp:110
void handleIconDownloadFinished(const Net::DownloadResult &result)
Definition: rss_feed.cpp:191
void iconLoaded(Feed *feed=nullptr)
QList< Article * > m_articlesByDate
Definition: rss_feed.h:123
QString title() const
Definition: rss_feed.cpp:159
void loadArticles(const QByteArray &data)
Definition: rss_feed.cpp:286
QString m_lastBuildDate
Definition: rss_feed.h:119
void downloadIcon()
Definition: rss_feed.cpp:424
void loadArticlesLegacy()
Definition: rss_feed.cpp:325
void increaseUnreadCount()
Definition: rss_feed.cpp:410
int unreadCount() const override
Definition: rss_feed.cpp:174
~Feed() override
Definition: rss_feed.cpp:100
QString lastBuildDate() const
Definition: rss_feed.cpp:169
bool m_hasError
Definition: rss_feed.h:120
void load()
Definition: rss_feed.cpp:263
bool isLoading() const
Definition: rss_feed.cpp:164
void refresh() override
Definition: rss_feed.cpp:132
void titleChanged(Feed *feed=nullptr)
int m_unreadCount
Definition: rss_feed.h:124
Private::Parser * m_parser
Definition: rss_feed.h:115
QUuid uid() const
Definition: rss_feed.cpp:149
void handleParsingFinished(const Private::ParsingResult &result)
Definition: rss_feed.cpp:227
Net::DownloadHandler * m_downloadHandler
Definition: rss_feed.h:129
QBasicTimer m_savingTimer
Definition: rss_feed.h:127
const QString m_url
Definition: rss_feed.h:117
void handleDownloadFinished(const Net::DownloadResult &result)
Definition: rss_feed.cpp:204
bool hasError() const
Definition: rss_feed.cpp:199
void handleMaxArticlesPerFeedChanged(int n)
Definition: rss_feed.cpp:184
void timerEvent(QTimerEvent *event) override
Definition: rss_feed.cpp:556
void cleanup() override
Definition: rss_feed.cpp:550
int updateArticles(const QList< QVariantHash > &loadedArticles)
Definition: rss_feed.cpp:435
void store()
Definition: rss_feed.cpp:347
bool m_dirty
Definition: rss_feed.h:128
const QUuid m_uid
Definition: rss_feed.h:116
void handleSessionProcessingEnabledChanged(bool enabled)
Definition: rss_feed.cpp:530
QString m_title
Definition: rss_feed.h:118
QString url() const
Definition: rss_feed.cpp:154
void decreaseUnreadCount()
Definition: rss_feed.cpp:416
QString m_iconPath
Definition: rss_feed.h:125
bool addArticle(Article *article)
Definition: rss_feed.cpp:367
void stateChanged(Feed *feed=nullptr)
bool m_isLoading
Definition: rss_feed.h:121
Feed(const QUuid &uid, const QString &url, const QString &path, Session *session)
Definition: rss_feed.cpp:64
void storeDeferred()
Definition: rss_feed.cpp:361
Session * m_session
Definition: rss_feed.h:114
void handleArticleRead(Article *article)
Definition: rss_feed.cpp:540
void newArticle(Article *article)
void articleAboutToBeRemoved(Article *article)
void aboutToBeDestroyed(Item *item=nullptr)
void unreadCountChanged(Item *item=nullptr)
void articleRead(Article *article)
void parse(const QByteArray &feedData)
Definition: rss_parser.cpp:550
void finished(const RSS::Private::ParsingResult &result)
bool isProcessingEnabled() const
QThread * workingThread() const
AsyncFileStorage * dataFileStorage() const
void maxArticlesPerFeedChanged(int n)
int maxArticlesPerFeed() const
void processingStateChanged(bool enabled)
constexpr std::add_const_t< T > & asConst(T &t) noexcept
Definition: global.h:42
void LogMsg(const QString &message, const Log::MsgType &type)
Definition: logger.cpp:125
@ WARNING
Definition: logger.h:47
QString toValidFileSystemName(const QString &name, bool allowSeparators=false, const QString &pad=QLatin1String(" "))
Definition: fs.cpp:237
QString toUniformPath(const QString &path)
Definition: fs.cpp:69
bool forceRemove(const QString &filePath)
Definition: fs.cpp:173
nonstd::expected< void, QString > saveToFile(const QString &path, const QByteArray &data)
Definition: io.cpp:69
QString toString(const lt::socket_type_t socketType)
Definition: session.cpp:183
std::unique_ptr< QSettings > SettingsPtr
Definition: profile.h:44
const QString KEY_LASTBUILDDATE(QStringLiteral("lastBuildDate"))
const QString KEY_URL(QStringLiteral("url"))
const QString KEY_UID(QStringLiteral("uid"))
const QString KEY_TITLE(QStringLiteral("title"))
const QString KEY_ISLOADING(QStringLiteral("isLoading"))
const QString KEY_ARTICLES(QStringLiteral("articles"))
const QString KEY_HASERROR(QStringLiteral("hasError"))
file(GLOB QBT_TS_FILES "${qBittorrent_SOURCE_DIR}/src/lang/*.ts") set_source_files_properties($
Definition: CMakeLists.txt:5
DownloadStatus status
static ServiceID fromURL(const QUrl &url)
QList< QVariantHash > articles
Definition: rss_parser.h:49