qBittorrent
rss_autodownloader.cpp
Go to the documentation of this file.
1 /*
2  * Bittorrent Client using Qt and libtorrent.
3  * Copyright (C) 2017 Vladimir Golovnev <glassez@yandex.ru>
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License
7  * as published by the Free Software Foundation; either version 2
8  * of the License, or (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18  *
19  * In addition, as a special exception, the copyright holders give permission to
20  * link this program with the OpenSSL project's "OpenSSL" library (or with
21  * modified versions of it that use the same license as the "OpenSSL" library),
22  * and distribute the linked executables. You must obey the GNU General Public
23  * License in all respects for all of the code used other than "OpenSSL". If you
24  * modify file(s), you may extend this exception to your version of the file(s),
25  * but you are not obligated to do so. If you do not wish to do so, delete this
26  * exception statement from your version.
27  */
28 
29 #include "rss_autodownloader.h"
30 
31 #include <QDataStream>
32 #include <QDebug>
33 #include <QJsonDocument>
34 #include <QJsonObject>
35 #include <QJsonValue>
36 #include <QThread>
37 #include <QTimer>
38 #include <QUrl>
39 #include <QVariant>
40 #include <QVector>
41 
42 #include "../bittorrent/magneturi.h"
43 #include "../bittorrent/session.h"
44 #include "../asyncfilestorage.h"
45 #include "../global.h"
46 #include "../logger.h"
47 #include "../profile.h"
48 #include "../utils/fs.h"
49 #include "rss_article.h"
50 #include "rss_autodownloadrule.h"
51 #include "rss_feed.h"
52 #include "rss_folder.h"
53 #include "rss_session.h"
54 
56 {
57  QString feedURL;
58  QVariantHash articleData;
59 };
60 
61 const QString ConfFolderName(QStringLiteral("rss"));
62 const QString RulesFileName(QStringLiteral("download_rules.json"));
63 
64 namespace
65 {
66  QVector<RSS::AutoDownloadRule> rulesFromJSON(const QByteArray &jsonData)
67  {
68  QJsonParseError jsonError;
69  const QJsonDocument jsonDoc = QJsonDocument::fromJson(jsonData, &jsonError);
70  if (jsonError.error != QJsonParseError::NoError)
71  throw RSS::ParsingError(jsonError.errorString());
72 
73  if (!jsonDoc.isObject())
74  throw RSS::ParsingError(RSS::AutoDownloader::tr("Invalid data format."));
75 
76  const QJsonObject jsonObj {jsonDoc.object()};
77  QVector<RSS::AutoDownloadRule> rules;
78  for (auto it = jsonObj.begin(); it != jsonObj.end(); ++it)
79  {
80  const QJsonValue jsonVal {it.value()};
81  if (!jsonVal.isObject())
82  throw RSS::ParsingError(RSS::AutoDownloader::tr("Invalid data format."));
83 
84  rules.append(RSS::AutoDownloadRule::fromJsonObject(jsonVal.toObject(), it.key()));
85  }
86 
87  return rules;
88  }
89 }
90 
91 using namespace RSS;
92 
93 QPointer<AutoDownloader> AutoDownloader::m_instance = nullptr;
94 
95 QString computeSmartFilterRegex(const QStringList &filters)
96 {
97  return QString::fromLatin1("(?:_|\\b)(?:%1)(?:_|\\b)").arg(filters.join(QString(")|(?:")));
98 }
99 
101  : m_storeProcessingEnabled("RSS/AutoDownloader/EnableProcessing", false)
102  , m_storeSmartEpisodeFilter("RSS/AutoDownloader/SmartEpisodeFilter")
103  , m_storeDownloadRepacks("RSS/AutoDownloader/DownloadRepacks")
104  , m_processingTimer(new QTimer(this))
105  , m_ioThread(new QThread(this))
106 {
107  Q_ASSERT(!m_instance); // only one instance is allowed
108  m_instance = this;
109 
112  if (!m_fileStorage)
113  throw RuntimeError(tr("Directory for RSS AutoDownloader data is unavailable."));
114 
115  m_fileStorage->moveToThread(m_ioThread);
116  connect(m_ioThread, &QThread::finished, m_fileStorage, &AsyncFileStorage::deleteLater);
117  connect(m_fileStorage, &AsyncFileStorage::failed, [](const QString &fileName, const QString &errorString)
118  {
119  LogMsg(tr("Couldn't save RSS AutoDownloader data in %1. Error: %2")
120  .arg(fileName, errorString), Log::CRITICAL);
121  });
122 
123  m_ioThread->start();
124 
129 
130  // initialise the smart episode regex
131  const QString regex = computeSmartFilterRegex(smartEpisodeFilters());
132  m_smartEpisodeRegex = QRegularExpression(regex,
133  QRegularExpression::CaseInsensitiveOption
134  | QRegularExpression::ExtendedPatternSyntaxOption
135  | QRegularExpression::UseUnicodePropertiesOption);
136 
137  load();
138 
139  m_processingTimer->setSingleShot(true);
140  connect(m_processingTimer, &QTimer::timeout, this, &AutoDownloader::process);
141 
142  if (isProcessingEnabled())
143  startProcessing();
144 }
145 
147 {
148  store();
149 
150  m_ioThread->quit();
151  m_ioThread->wait();
152 }
153 
155 {
156  return m_instance;
157 }
158 
159 bool AutoDownloader::hasRule(const QString &ruleName) const
160 {
161  return m_rules.contains(ruleName);
162 }
163 
164 AutoDownloadRule AutoDownloader::ruleByName(const QString &ruleName) const
165 {
166  return m_rules.value(ruleName, AutoDownloadRule("Unknown Rule"));
167 }
168 
169 QList<AutoDownloadRule> AutoDownloader::rules() const
170 {
171  return m_rules.values();
172 }
173 
175 {
176  if (!hasRule(rule.name()))
177  {
178  // Insert new rule
179  setRule_impl(rule);
180  m_dirty = true;
181  store();
182  emit ruleAdded(rule.name());
184  }
185  else if (ruleByName(rule.name()) != rule)
186  {
187  // Update existing rule
188  setRule_impl(rule);
189  m_dirty = true;
190  storeDeferred();
191  emit ruleChanged(rule.name());
193  }
194 }
195 
196 bool AutoDownloader::renameRule(const QString &ruleName, const QString &newRuleName)
197 {
198  if (!hasRule(ruleName)) return false;
199  if (hasRule(newRuleName)) return false;
200 
201  AutoDownloadRule rule = m_rules.take(ruleName);
202  rule.setName(newRuleName);
203  m_rules.insert(newRuleName, rule);
204  m_dirty = true;
205  store();
206  emit ruleRenamed(newRuleName, ruleName);
207  return true;
208 }
209 
210 void AutoDownloader::removeRule(const QString &ruleName)
211 {
212  if (m_rules.contains(ruleName))
213  {
214  emit ruleAboutToBeRemoved(ruleName);
215  m_rules.remove(ruleName);
216  m_dirty = true;
217  store();
218  }
219 }
220 
222 {
223  switch (format)
224  {
226  return exportRulesToLegacyFormat();
227  default:
228  return exportRulesToJSONFormat();
229  }
230 }
231 
232 void AutoDownloader::importRules(const QByteArray &data, const AutoDownloader::RulesFileFormat format)
233 {
234  switch (format)
235  {
238  break;
239  default:
241  }
242 }
243 
245 {
246  QJsonObject jsonObj;
247  for (const auto &rule : asConst(rules()))
248  jsonObj.insert(rule.name(), rule.toJsonObject());
249 
250  return QJsonDocument(jsonObj).toJson();
251 }
252 
253 void AutoDownloader::importRulesFromJSONFormat(const QByteArray &data)
254 {
255  for (const auto &rule : asConst(rulesFromJSON(data)))
256  insertRule(rule);
257 }
258 
260 {
261  QVariantHash dict;
262  for (const auto &rule : asConst(rules()))
263  dict[rule.name()] = rule.toLegacyDict();
264 
265  QByteArray data;
266  QDataStream out(&data, QIODevice::WriteOnly);
267  out.setVersion(QDataStream::Qt_4_5);
268  out << dict;
269 
270  return data;
271 }
272 
274 {
275  QDataStream in(data);
276  in.setVersion(QDataStream::Qt_4_5);
277  QVariantHash dict;
278  in >> dict;
279  if (in.status() != QDataStream::Ok)
280  throw ParsingError(tr("Invalid data format"));
281 
282  for (const QVariant &val : asConst(dict))
284 }
285 
287 {
288  const QVariant filter = m_storeSmartEpisodeFilter.get();
289  if (filter.isNull())
290  {
291  const QStringList defaultFilters =
292  {
293  "s(\\d+)e(\\d+)", // Format 1: s01e01
294  "(\\d+)x(\\d+)", // Format 2: 01x01
295  "(\\d{4}[.\\-]\\d{1,2}[.\\-]\\d{1,2})", // Format 3: 2017.01.01
296  "(\\d{1,2}[.\\-]\\d{1,2}[.\\-]\\d{4})" // Format 4: 01.01.2017
297  };
298  return defaultFilters;
299  }
300  return filter.toStringList();
301 }
302 
303 QRegularExpression AutoDownloader::smartEpisodeRegex() const
304 {
305  return m_smartEpisodeRegex;
306 }
307 
308 void AutoDownloader::setSmartEpisodeFilters(const QStringList &filters)
309 {
310  m_storeSmartEpisodeFilter = filters;
311 
312  const QString regex = computeSmartFilterRegex(filters);
313  m_smartEpisodeRegex.setPattern(regex);
314 }
315 
317 {
318  return m_storeDownloadRepacks.get(true);
319 }
320 
321 void AutoDownloader::setDownloadRepacks(const bool enabled)
322 {
323  m_storeDownloadRepacks = enabled;
324 }
325 
327 {
328  if (m_processingQueue.isEmpty()) return; // processing was disabled
329 
330  processJob(m_processingQueue.takeFirst());
331  if (!m_processingQueue.isEmpty())
332  // Schedule to process the next torrent (if any)
333  m_processingTimer->start();
334 }
335 
337 {
338  const auto job = m_waitingJobs.take(url);
339  if (!job) return;
340 
341  if (Feed *feed = Session::instance()->feedByURL(job->feedURL))
342  if (Article *article = feed->articleByGUID(job->articleData.value(Article::KeyId).toString()))
343  article->markAsRead();
344 }
345 
347 {
348  m_waitingJobs.remove(url);
349  // TODO: Re-schedule job here.
350 }
351 
353 {
354  if (!article->isRead() && !article->torrentUrl().isEmpty())
355  addJobForArticle(article);
356 }
357 
359 {
360  m_rules.insert(rule.name(), rule);
361 }
362 
364 {
365  const QString torrentURL = article->torrentUrl();
366  if (m_waitingJobs.contains(torrentURL)) return;
367 
368  QSharedPointer<ProcessingJob> job(new ProcessingJob);
369  job->feedURL = article->feed()->url();
370  job->articleData = article->data();
371  m_processingQueue.append(job);
372  if (!m_processingTimer->isActive())
373  m_processingTimer->start();
374 }
375 
376 void AutoDownloader::processJob(const QSharedPointer<ProcessingJob> &job)
377 {
378  for (AutoDownloadRule &rule : m_rules)
379  {
380  if (!rule.isEnabled()) continue;
381  if (!rule.feedURLs().contains(job->feedURL)) continue;
382  if (!rule.accepts(job->articleData)) continue;
383 
384  m_dirty = true;
385  storeDeferred();
386 
388  params.savePath = rule.savePath();
389  params.category = rule.assignedCategory();
390  params.addPaused = rule.addPaused();
391  params.contentLayout = rule.torrentContentLayout();
392  if (!rule.savePath().isEmpty())
393  params.useAutoTMM = false;
394  const auto torrentURL = job->articleData.value(Article::KeyTorrentURL).toString();
395  BitTorrent::Session::instance()->addTorrent(torrentURL, params);
396 
397  if (BitTorrent::MagnetUri(torrentURL).isValid())
398  {
399  if (Feed *feed = Session::instance()->feedByURL(job->feedURL))
400  {
401  if (Article *article = feed->articleByGUID(job->articleData.value(Article::KeyId).toString()))
402  article->markAsRead();
403  }
404  }
405  else
406  {
407  // waiting for torrent file downloading
408  m_waitingJobs.insert(torrentURL, job);
409  }
410 
411  return;
412  }
413 }
414 
416 {
417  QFile rulesFile(m_fileStorage->storageDir().absoluteFilePath(RulesFileName));
418 
419  if (!rulesFile.exists())
420  loadRulesLegacy();
421  else if (rulesFile.open(QFile::ReadOnly))
422  loadRules(rulesFile.readAll());
423  else
424  LogMsg(tr("Couldn't read RSS AutoDownloader rules from %1. Error: %2")
425  .arg(rulesFile.fileName(), rulesFile.errorString()), Log::CRITICAL);
426 }
427 
428 void AutoDownloader::loadRules(const QByteArray &data)
429 {
430  try
431  {
432  const auto rules = rulesFromJSON(data);
433  for (const auto &rule : rules)
434  setRule_impl(rule);
435  }
436  catch (const ParsingError &error)
437  {
438  LogMsg(tr("Couldn't load RSS AutoDownloader rules. Reason: %1")
439  .arg(error.message()), Log::CRITICAL);
440  }
441 }
442 
444 {
445  const SettingsPtr settings = Profile::instance()->applicationSettings(QStringLiteral("qBittorrent-rss"));
446  const QVariantHash rules = settings->value(QStringLiteral("download_rules")).toHash();
447  for (const QVariant &ruleVar : rules)
448  {
449  const auto rule = AutoDownloadRule::fromLegacyDict(ruleVar.toHash());
450  if (!rule.name().isEmpty())
451  insertRule(rule);
452  }
453 }
454 
456 {
457  if (!m_dirty) return;
458 
459  m_dirty = false;
460  m_savingTimer.stop();
461 
462  QJsonObject jsonObj;
463  for (const auto &rule : asConst(m_rules))
464  jsonObj.insert(rule.name(), rule.toJsonObject());
465 
466  m_fileStorage->store(RulesFileName, QJsonDocument(jsonObj).toJson());
467 }
468 
470 {
471  if (!m_savingTimer.isActive())
472  m_savingTimer.start(5 * 1000, this);
473 }
474 
476 {
478 }
479 
481 {
482  m_processingQueue.clear();
483  if (!isProcessingEnabled()) return;
484 
485  for (Article *article : asConst(Session::instance()->rootFolder()->articles()))
486  {
487  if (!article->isRead() && !article->torrentUrl().isEmpty())
488  addJobForArticle(article);
489  }
490 }
491 
493 {
495  connect(Session::instance()->rootFolder(), &Folder::newArticle, this, &AutoDownloader::handleNewArticle);
496 }
497 
498 void AutoDownloader::setProcessingEnabled(const bool enabled)
499 {
500  if (m_storeProcessingEnabled != enabled)
501  {
502  m_storeProcessingEnabled = enabled;
503  if (enabled)
504  {
505  startProcessing();
506  }
507  else
508  {
509  m_processingQueue.clear();
510  disconnect(Session::instance()->rootFolder(), &Folder::newArticle, this, &AutoDownloader::handleNewArticle);
511  }
512 
513  emit processingStateChanged(enabled);
514  }
515 }
516 
517 void AutoDownloader::timerEvent(QTimerEvent *event)
518 {
519  Q_UNUSED(event);
520  store();
521 }
void failed(const QString &fileName, const QString &errorString)
QDir storageDir() const
void store(const QString &fileName, const QByteArray &data)
static Session * instance()
Definition: session.cpp:997
void downloadFromUrlFinished(const QString &url)
bool addTorrent(const QString &source, const AddTorrentParams &params=AddTorrentParams())
Definition: session.cpp:2007
void downloadFromUrlFailed(const QString &url, const QString &reason)
QString message() const noexcept
Definition: exceptions.cpp:36
SettingsPtr applicationSettings(const QString &name) const
Definition: profile.cpp:109
static const Profile * instance()
Definition: profile.cpp:67
Definition: rss_article.h:43
QString torrentUrl() const
static const QString KeyId
Definition: rss_article.h:53
static const QString KeyTorrentURL
Definition: rss_article.h:58
QVariantHash data() const
Feed * feed() const
bool isRead() const
static AutoDownloadRule fromJsonObject(const QJsonObject &jsonObj, const QString &name="")
static AutoDownloadRule fromLegacyDict(const QVariantHash &dict)
void setName(const QString &name)
QByteArray exportRulesToJSONFormat() const
void importRulesFromLegacyFormat(const QByteArray &data)
CachedSettingValue< bool > m_storeProcessingEnabled
void ruleChanged(const QString &ruleName)
void setDownloadRepacks(bool enabled)
bool isProcessingEnabled() const
bool hasRule(const QString &ruleName) const
QList< QSharedPointer< ProcessingJob > > m_processingQueue
QList< AutoDownloadRule > rules() const
QByteArray exportRulesToLegacyFormat() const
void setRule_impl(const AutoDownloadRule &rule)
QHash< QString, AutoDownloadRule > m_rules
AsyncFileStorage * m_fileStorage
SettingValue< bool > m_storeDownloadRepacks
void setSmartEpisodeFilters(const QStringList &filters)
void removeRule(const QString &ruleName)
static QPointer< AutoDownloader > m_instance
void processingStateChanged(bool enabled)
SettingValue< QVariant > m_storeSmartEpisodeFilter
void handleTorrentDownloadFinished(const QString &url)
bool renameRule(const QString &ruleName, const QString &newRuleName)
void processJob(const QSharedPointer< ProcessingJob > &job)
static AutoDownloader * instance()
void ruleAdded(const QString &ruleName)
void timerEvent(QTimerEvent *event) override
void ruleAboutToBeRemoved(const QString &ruleName)
QHash< QString, QSharedPointer< ProcessingJob > > m_waitingJobs
void loadRules(const QByteArray &data)
void ruleRenamed(const QString &ruleName, const QString &oldRuleName)
AutoDownloadRule ruleByName(const QString &ruleName) const
QByteArray exportRules(RulesFileFormat format=RulesFileFormat::JSON) const
void handleNewArticle(const Article *article)
QRegularExpression smartEpisodeRegex() const
QStringList smartEpisodeFilters() const
void addJobForArticle(const Article *article)
void insertRule(const AutoDownloadRule &rule)
void setProcessingEnabled(bool enabled)
QRegularExpression m_smartEpisodeRegex
void handleTorrentDownloadFailed(const QString &url)
void importRules(const QByteArray &data, RulesFileFormat format=RulesFileFormat::JSON)
void importRulesFromJSONFormat(const QByteArray &data)
QString url() const
Definition: rss_feed.cpp:154
void newArticle(Article *article)
static Session * instance()
T get(const T &defaultValue={}) const
Definition: settingvalue.h:46
constexpr std::add_const_t< T > & asConst(T &t) noexcept
Definition: global.h:42
void LogMsg(const QString &message, const Log::MsgType &type)
Definition: logger.cpp:125
@ CRITICAL
Definition: logger.h:48
QString fileName(const QString &filePath)
Definition: fs.cpp:87
QString expandPathAbs(const QString &path)
Definition: fs.cpp:309
QVector< RSS::AutoDownloadRule > rulesFromJSON(const QByteArray &jsonData)
QString specialFolderLocation(const SpecialFolder folder)
Definition: profile.cpp:131
std::unique_ptr< QSettings > SettingsPtr
Definition: profile.h:44
const QString ConfFolderName(QStringLiteral("rss"))
const QString RulesFileName(QStringLiteral("download_rules.json"))
QString computeSmartFilterRegex(const QStringList &filters)
std::optional< bool > useAutoTMM
std::optional< bool > addPaused
std::optional< BitTorrent::TorrentContentLayout > contentLayout
QVariantHash articleData