qBittorrent
geoipdatabase.cpp
Go to the documentation of this file.
1 /*
2  * Bittorrent Client using Qt and libtorrent.
3  * Copyright (C) 2015 Vladimir Golovnev <glassez@yandex.ru>
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License
7  * as published by the Free Software Foundation; either version 2
8  * of the License, or (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18  *
19  * In addition, as a special exception, the copyright holders give permission to
20  * link this program with the OpenSSL project's "OpenSSL" library (or with
21  * modified versions of it that use the same license as the "OpenSSL" library),
22  * and distribute the linked executables. You must obey the GNU General Public
23  * License in all respects for all of the code used other than "OpenSSL". If you
24  * modify file(s), you may extend this exception to your version of the file(s),
25  * but you are not obligated to do so. If you do not wish to do so, delete this
26  * exception statement from your version.
27  */
28 
29 #include <QDateTime>
30 #include <QDebug>
31 #include <QFile>
32 #include <QHostAddress>
33 #include <QVariant>
34 
35 #include "geoipdatabase.h"
36 
37 namespace
38 {
39  const qint32 MAX_FILE_SIZE = 67108864; // 64MB
40  const quint32 MAX_METADATA_SIZE = 131072; // 128KB
41  const char METADATA_BEGIN_MARK[] = "\xab\xcd\xefMaxMind.com";
42  const char DATA_SECTION_SEPARATOR[16] = {0};
43 
44  enum class DataType
45  {
46  Unknown = 0,
47  Pointer = 1,
48  String = 2,
49  Double = 3,
50  Bytes = 4,
51  Integer16 = 5,
52  Integer32 = 6,
53  Map = 7,
54  SignedInteger32 = 8,
55  Integer64 = 9,
56  Integer128 = 10,
57  Array = 11,
58  DataCacheContainer = 12,
59  EndMarker = 13,
60  Boolean = 14,
61  Float = 15
62  };
63 }
64 
66 {
67  DataType fieldType {DataType::Unknown};
68  union
69  {
70  quint32 fieldSize = 0;
71  quint32 offset; // Pointer
72  };
73 };
74 
75 GeoIPDatabase::GeoIPDatabase(const quint32 size)
76  : m_ipVersion(0)
77  , m_recordSize(0)
78  , m_nodeCount(0)
79  , m_nodeSize(0)
80  , m_indexSize(0)
81  , m_recordBytes(0)
82  , m_size(size)
83  , m_data(new uchar[size])
84 {
85 }
86 
87 GeoIPDatabase *GeoIPDatabase::load(const QString &filename, QString &error)
88 {
89  GeoIPDatabase *db = nullptr;
90  QFile file(filename);
91  if (file.size() > MAX_FILE_SIZE)
92  {
93  error = tr("Unsupported database file size.");
94  return nullptr;
95  }
96 
97  if (!file.open(QFile::ReadOnly))
98  {
99  error = file.errorString();
100  return nullptr;
101  }
102 
103  db = new GeoIPDatabase(file.size());
104 
105  if (file.read(reinterpret_cast<char *>(db->m_data), db->m_size) != db->m_size)
106  {
107  error = file.errorString();
108  delete db;
109  return nullptr;
110  }
111 
112 
113  if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error))
114  {
115  delete db;
116  return nullptr;
117  }
118 
119  return db;
120 }
121 
122 GeoIPDatabase *GeoIPDatabase::load(const QByteArray &data, QString &error)
123 {
124  GeoIPDatabase *db = nullptr;
125  if (data.size() > MAX_FILE_SIZE)
126  {
127  error = tr("Unsupported database file size.");
128  return nullptr;
129  }
130 
131  db = new GeoIPDatabase(data.size());
132 
133  memcpy(reinterpret_cast<char *>(db->m_data), data.constData(), db->m_size);
134 
135  if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error))
136  {
137  delete db;
138  return nullptr;
139  }
140 
141  return db;
142 }
143 
145 {
146  delete [] m_data;
147 }
148 
149 QString GeoIPDatabase::type() const
150 {
151  return m_dbType;
152 }
153 
155 {
156  return m_ipVersion;
157 }
158 
159 QDateTime GeoIPDatabase::buildEpoch() const
160 {
161  return m_buildEpoch;
162 }
163 
164 QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const
165 {
166  Q_IPV6ADDR addr = hostAddr.toIPv6Address();
167 
168  const uchar *ptr = m_data;
169 
170  for (int i = 0; i < 16; ++i)
171  {
172  for (int j = 0; j < 8; ++j)
173  {
174  const bool right = static_cast<bool>((addr[i] >> (7 - j)) & 1);
175  // Interpret the left/right record as number
176  if (right)
177  ptr += m_recordBytes;
178 
179  quint32 id = 0;
180  auto *idPtr = reinterpret_cast<uchar *>(&id);
181  memcpy(&idPtr[4 - m_recordBytes], ptr, m_recordBytes);
182  fromBigEndian(idPtr, 4);
183 
184  if (id == m_nodeCount)
185  {
186  return {};
187  }
188  if (id > m_nodeCount)
189  {
190  QString country = m_countries.value(id);
191  if (country.isEmpty())
192  {
193  const quint32 offset = id - m_nodeCount - sizeof(DATA_SECTION_SEPARATOR);
194  quint32 tmp = offset + m_indexSize + sizeof(DATA_SECTION_SEPARATOR);
195  const QVariant val = readDataField(tmp);
196  if (val.userType() == QMetaType::QVariantHash)
197  {
198  country = val.toHash()["country"].toHash()["iso_code"].toString();
199  m_countries[id] = country;
200  }
201  }
202  return country;
203  }
204 
205  ptr = m_data + (id * m_nodeSize);
206  }
207  }
208 
209  return {};
210 }
211 
212 #define CHECK_METADATA_REQ(key, type) \
213 if (!metadata.contains(#key)) \
214 { \
215  error = errMsgNotFound.arg(#key); \
216  return false; \
217 } \
218 if (metadata.value(#key).userType() != QMetaType::type) \
219 { \
220  error = errMsgInvalid.arg(#key); \
221  return false; \
222 }
223 
224 #define CHECK_METADATA_OPT(key, type) \
225 if (metadata.contains(#key)) \
226 { \
227  if (metadata.value(#key).userType() != QMetaType::type) \
228  { \
229  error = errMsgInvalid.arg(#key); \
230  return false; \
231  } \
232 }
233 
234 bool GeoIPDatabase::parseMetadata(const QVariantHash &metadata, QString &error)
235 {
236  const QString errMsgNotFound = tr("Metadata error: '%1' entry not found.");
237  const QString errMsgInvalid = tr("Metadata error: '%1' entry has invalid type.");
238 
239  qDebug() << "Parsing MaxMindDB metadata...";
240 
241  CHECK_METADATA_REQ(binary_format_major_version, UShort);
242  CHECK_METADATA_REQ(binary_format_minor_version, UShort);
243  const uint versionMajor = metadata.value("binary_format_major_version").toUInt();
244  const uint versionMinor = metadata.value("binary_format_minor_version").toUInt();
245  if (versionMajor != 2)
246  {
247  error = tr("Unsupported database version: %1.%2").arg(versionMajor).arg(versionMinor);
248  return false;
249  }
250 
251  CHECK_METADATA_REQ(ip_version, UShort);
252  m_ipVersion = metadata.value("ip_version").value<quint16>();
253  if (m_ipVersion != 6)
254  {
255  error = tr("Unsupported IP version: %1").arg(m_ipVersion);
256  return false;
257  }
258 
259  CHECK_METADATA_REQ(record_size, UShort);
260  m_recordSize = metadata.value("record_size").value<quint16>();
261  if (m_recordSize != 24)
262  {
263  error = tr("Unsupported record size: %1").arg(m_recordSize);
264  return false;
265  }
266  m_nodeSize = m_recordSize / 4;
268 
269  CHECK_METADATA_REQ(node_count, UInt);
270  m_nodeCount = metadata.value("node_count").value<quint32>();
272 
273  CHECK_METADATA_REQ(database_type, QString);
274  m_dbType = metadata.value("database_type").toString();
275 
276  CHECK_METADATA_REQ(build_epoch, ULongLong);
277  m_buildEpoch = QDateTime::fromSecsSinceEpoch(metadata.value("build_epoch").toULongLong());
278 
279  CHECK_METADATA_OPT(languages, QVariantList);
280  CHECK_METADATA_OPT(description, QVariantHash);
281 
282  return true;
283 }
284 
285 bool GeoIPDatabase::loadDB(QString &error) const
286 {
287  qDebug() << "Parsing IP geolocation database index tree...";
288 
289  const int nodeSize = m_recordSize / 4; // in bytes
290  const int indexSize = m_nodeCount * nodeSize;
291  if ((m_size < (indexSize + sizeof(DATA_SECTION_SEPARATOR)))
292  || (memcmp(m_data + indexSize, DATA_SECTION_SEPARATOR, sizeof(DATA_SECTION_SEPARATOR)) != 0))
293  {
294  error = tr("Database corrupted: no data section found.");
295  return false;
296  }
297 
298  return true;
299 }
300 
301 QVariantHash GeoIPDatabase::readMetadata() const
302 {
303  const char *ptr = reinterpret_cast<const char *>(m_data);
304  quint32 size = m_size;
306  {
307  ptr += m_size - MAX_METADATA_SIZE;
308  size = MAX_METADATA_SIZE;
309  }
310 
311  const QByteArray data = QByteArray::fromRawData(ptr, size);
312  int index = data.lastIndexOf(METADATA_BEGIN_MARK);
313  if (index >= 0)
314  {
316  index += (m_size - MAX_METADATA_SIZE); // from begin of all data
317  auto offset = static_cast<quint32>(index + strlen(METADATA_BEGIN_MARK));
318  const QVariant metadata = readDataField(offset);
319  if (metadata.userType() == QMetaType::QVariantHash)
320  return metadata.toHash();
321  }
322 
323  return {};
324 }
325 
326 QVariant GeoIPDatabase::readDataField(quint32 &offset) const
327 {
328  DataFieldDescriptor descr;
329  if (!readDataFieldDescriptor(offset, descr))
330  return {};
331 
332  quint32 locOffset = offset;
333  bool usePointer = false;
334  if (descr.fieldType == DataType::Pointer)
335  {
336  usePointer = true;
337  // convert offset from data section to global
338  locOffset = descr.offset + (m_nodeCount * m_recordSize / 4) + sizeof(DATA_SECTION_SEPARATOR);
339  if (!readDataFieldDescriptor(locOffset, descr))
340  return {};
341  }
342 
343  QVariant fieldValue;
344  switch (descr.fieldType)
345  {
346  case DataType::Pointer:
347  qDebug() << "* Illegal Pointer using";
348  break;
349  case DataType::String:
350  fieldValue = QString::fromUtf8(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
351  locOffset += descr.fieldSize;
352  break;
353  case DataType::Double:
354  if (descr.fieldSize == 8)
355  fieldValue = readPlainValue<double>(locOffset, descr.fieldSize);
356  else
357  qDebug() << "* Invalid field size for type: Double";
358  break;
359  case DataType::Bytes:
360  fieldValue = QByteArray(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
361  locOffset += descr.fieldSize;
362  break;
363  case DataType::Integer16:
364  fieldValue = readPlainValue<quint16>(locOffset, descr.fieldSize);
365  break;
366  case DataType::Integer32:
367  fieldValue = readPlainValue<quint32>(locOffset, descr.fieldSize);
368  break;
369  case DataType::Map:
370  fieldValue = readMapValue(locOffset, descr.fieldSize);
371  break;
372  case DataType::SignedInteger32:
373  fieldValue = readPlainValue<qint32>(locOffset, descr.fieldSize);
374  break;
375  case DataType::Integer64:
376  fieldValue = readPlainValue<quint64>(locOffset, descr.fieldSize);
377  break;
378  case DataType::Integer128:
379  qDebug() << "* Unsupported data type: Integer128";
380  break;
381  case DataType::Array:
382  fieldValue = readArrayValue(locOffset, descr.fieldSize);
383  break;
384  case DataType::DataCacheContainer:
385  qDebug() << "* Unsupported data type: DataCacheContainer";
386  break;
387  case DataType::EndMarker:
388  qDebug() << "* Unsupported data type: EndMarker";
389  break;
390  case DataType::Boolean:
391  fieldValue = QVariant::fromValue(static_cast<bool>(descr.fieldSize));
392  break;
393  case DataType::Float:
394  if (descr.fieldSize == 4)
395  fieldValue = readPlainValue<float>(locOffset, descr.fieldSize);
396  else
397  qDebug() << "* Invalid field size for type: Float";
398  break;
399  default:
400  qDebug() << "* Unsupported data type: Unknown";
401  }
402 
403  if (!usePointer)
404  offset = locOffset;
405  return fieldValue;
406 }
407 
409 {
410  const uchar *dataPtr = m_data + offset;
411  const int availSize = m_size - offset;
412  if (availSize < 1) return false;
413 
414  out.fieldType = static_cast<DataType>((dataPtr[0] & 0xE0) >> 5);
415  if (out.fieldType == DataType::Pointer)
416  {
417  const int size = ((dataPtr[0] & 0x18) >> 3);
418  if (availSize < (size + 2)) return false;
419 
420  if (size == 0)
421  out.offset = ((dataPtr[0] & 0x07) << 8) + dataPtr[1];
422  else if (size == 1)
423  out.offset = ((dataPtr[0] & 0x07) << 16) + (dataPtr[1] << 8) + dataPtr[2] + 2048;
424  else if (size == 2)
425  out.offset = ((dataPtr[0] & 0x07) << 24) + (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 526336;
426  else if (size == 3)
427  out.offset = (dataPtr[1] << 24) + (dataPtr[2] << 16) + (dataPtr[3] << 8) + dataPtr[4];
428 
429  offset += size + 2;
430  return true;
431  }
432 
433  out.fieldSize = dataPtr[0] & 0x1F;
434  if (out.fieldSize <= 28)
435  {
436  if (out.fieldType == DataType::Unknown)
437  {
438  out.fieldType = static_cast<DataType>(dataPtr[1] + 7);
439  if ((out.fieldType <= DataType::Map) || (out.fieldType > DataType::Float) || (availSize < 3))
440  return false;
441  offset += 2;
442  }
443  else
444  {
445  offset += 1;
446  }
447  }
448  else if (out.fieldSize == 29)
449  {
450  if (availSize < 2) return false;
451  out.fieldSize = dataPtr[1] + 29;
452  offset += 2;
453  }
454  else if (out.fieldSize == 30)
455  {
456  if (availSize < 3) return false;
457  out.fieldSize = (dataPtr[1] << 8) + dataPtr[2] + 285;
458  offset += 3;
459  }
460  else if (out.fieldSize == 31)
461  {
462  if (availSize < 4) return false;
463  out.fieldSize = (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 65821;
464  offset += 4;
465  }
466 
467  return true;
468 }
469 
470 void GeoIPDatabase::fromBigEndian(uchar *buf, const quint32 len) const
471 {
472 #if (Q_BYTE_ORDER == Q_LITTLE_ENDIAN)
473  std::reverse(buf, buf + len);
474 #else
475  Q_UNUSED(buf);
476  Q_UNUSED(len);
477 #endif
478 }
479 
480 QVariant GeoIPDatabase::readMapValue(quint32 &offset, const quint32 count) const
481 {
482  QVariantHash map;
483 
484  for (quint32 i = 0; i < count; ++i)
485  {
486  QVariant field = readDataField(offset);
487  if (field.userType() != QMetaType::QString)
488  return {};
489 
490  const QString key = field.toString();
491  field = readDataField(offset);
492  if (field.userType() == QVariant::Invalid)
493  return {};
494 
495  map[key] = field;
496  }
497 
498  return map;
499 }
500 
501 QVariant GeoIPDatabase::readArrayValue(quint32 &offset, const quint32 count) const
502 {
503  QVariantList array;
504 
505  for (quint32 i = 0; i < count; ++i)
506  {
507  const QVariant field = readDataField(offset);
508  if (field.userType() == QVariant::Invalid)
509  return {};
510 
511  array.append(field);
512  }
513 
514  return array;
515 }
bool loadDB(QString &error) const
quint16 ipVersion() const
void fromBigEndian(uchar *buf, quint32 len) const
QString m_dbType
Definition: geoipdatabase.h:96
quint16 m_recordSize
Definition: geoipdatabase.h:90
QVariant readArrayValue(quint32 &offset, quint32 count) const
QDateTime buildEpoch() const
QVariant readDataField(quint32 &offset) const
quint32 m_size
Definition: geoipdatabase.h:99
bool readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out) const
QString lookup(const QHostAddress &hostAddr) const
quint16 m_ipVersion
Definition: geoipdatabase.h:89
QHash< quint32, QString > m_countries
Definition: geoipdatabase.h:98
GeoIPDatabase(quint32 size)
QVariantHash readMetadata() const
static GeoIPDatabase * load(const QString &filename, QString &error)
QDateTime m_buildEpoch
Definition: geoipdatabase.h:95
bool parseMetadata(const QVariantHash &metadata, QString &error)
quint32 m_nodeCount
Definition: geoipdatabase.h:91
QString type() const
QVariant readMapValue(quint32 &offset, quint32 count) const
#define CHECK_METADATA_OPT(key, type)
#define CHECK_METADATA_REQ(key, type)
file(GLOB QBT_TS_FILES "${qBittorrent_SOURCE_DIR}/src/lang/*.ts") set_source_files_properties($
Definition: CMakeLists.txt:5