diff --git a/BrickStoreXML.rnc b/BrickStoreXML.rnc index d612c39a..f8560f5c 100755 --- a/BrickStoreXML.rnc +++ b/BrickStoreXML.rnc @@ -18,10 +18,14 @@ BrickStoreXML = element BrickStoreXML | BrickStockXML { Inventory, GuiState? } -# Currency is optional. The default currency is USD: +# Currency is optional. The default currency is USD. +# +# BrickLinkChangelogId is optional, but quite helpful when automatically applying item and color +# renames and merges (see also https://www.bricklink.com/btchglog.asp?viewHelp=Y) Inventory = element Inventory { attribute Currency { xsd:string { length="3" } }?, + attribute BrickLinkChangelogId { xsd:integer } }?, Item* } diff --git a/src/bricklink/changelogentry.h b/src/bricklink/changelogentry.h index d61c8770..e621e549 100755 --- a/src/bricklink/changelogentry.h +++ b/src/bricklink/changelogentry.h @@ -15,62 +15,87 @@ namespace BrickLink { class ColorChangeLogEntry { public: - uint fromColorId() const { return m_fromColorId; } - uint toColorId() const { return m_toColorId; } + uint id() const { return m_id; } + QDate date() const { return QDate::fromJulianDay(m_julianDay); } + uint fromColorId() const { return m_fromColorId; } + uint toColorId() const { return m_toColorId; } - explicit ColorChangeLogEntry(uint fromColorId = Color::InvalidId, uint toColorId = Color::InvalidId, - QDate date = { }) - - : m_fromColorId(fromColorId) + explicit ColorChangeLogEntry() = default; + explicit ColorChangeLogEntry(uint id, const QDate &date, uint fromColorId, uint toColorId) + : m_id(id) + , m_julianDay(uint(date.toJulianDay())) + , m_fromColorId(fromColorId) , m_toColorId(toColorId) - , m_date(date) { } - constexpr std::weak_ordering operator<=>(uint colorId) const { return m_fromColorId <=> colorId; } - constexpr std::weak_ordering operator<=>(const ColorChangeLogEntry &other) const { return *this <=> other.m_fromColorId; } - constexpr bool operator==(uint colorId) const { return (*this <=> colorId == 0); } - constexpr bool operator==(const ColorChangeLogEntry &other) const { return *this == other.m_fromColorId; } + std::weak_ordering operator<=>(uint colorId) const; + std::weak_ordering operator<=>(const ColorChangeLogEntry &other) const; private: - uint m_fromColorId; - uint m_toColorId; - QDate m_date; // only used when rebuilding the DB + uint m_id = 0; + uint m_julianDay = 0; + uint m_fromColorId = Color::InvalidId; + uint m_toColorId = Color::InvalidId; friend class Core; friend class Database; }; +inline std::weak_ordering ColorChangeLogEntry::operator<=>(uint colorId) const +{ + return m_fromColorId <=> colorId; +} + +inline std::weak_ordering ColorChangeLogEntry::operator<=>(const ColorChangeLogEntry &other) const +{ + auto cmp = (m_fromColorId <=> other.m_fromColorId); + return (cmp == 0) ? (m_id <=> other.m_id) : cmp; +} + + class ItemChangeLogEntry { public: - char fromItemTypeId() const { return m_fromTypeAndId.at(0); } - QByteArray fromItemId() const { return m_fromTypeAndId.mid(1); } + uint id() const { return m_id; } + QDate date() const { return QDate::fromJulianDay(m_julianDay); } + char fromItemTypeId() const { return m_fromTypeAndId.at(0); } + QByteArray fromItemId() const { return m_fromTypeAndId.mid(1); } + char toItemTypeId() const { return m_toTypeAndId.at(0); } + QByteArray toItemId() const { return m_toTypeAndId.mid(1); } + QByteArray toItemTypeAndId() const { return m_toTypeAndId; } - char toItemTypeId() const { return m_toTypeAndId.at(0); } - QByteArray toItemId() const { return m_toTypeAndId.mid(1); } - - QDate date() const { return m_date; } - - explicit ItemChangeLogEntry(const QByteArray &fromTypeAndId = { }, const QByteArray &toTypeAndId = { }, - QDate date = { }) - : m_fromTypeAndId(fromTypeAndId) + explicit ItemChangeLogEntry() = default; + explicit ItemChangeLogEntry(uint id, const QDate &date, const QByteArray &fromTypeAndId, + const QByteArray &toTypeAndId) + : m_id(id) + , m_julianDay(uint(date.toJulianDay())) + , m_fromTypeAndId(fromTypeAndId) , m_toTypeAndId(toTypeAndId) - , m_date(date) { } - std::weak_ordering operator<=>(const QByteArray &typeAndId) const { return m_fromTypeAndId.compare(typeAndId) <=> 0; } - std::weak_ordering operator<=>(const ItemChangeLogEntry &other) const { return *this <=> other.m_fromTypeAndId; } - bool operator==(const QByteArray &typeAndId) const { return (*this <=> typeAndId == 0); } - bool operator==(const ItemChangeLogEntry &other) const { return *this == other.m_fromTypeAndId; } + std::weak_ordering operator<=>(const QByteArray &typeAndId) const; + std::weak_ordering operator<=>(const ItemChangeLogEntry &other) const; private: + uint m_id = 0; + uint m_julianDay = 0; QByteArray m_fromTypeAndId; QByteArray m_toTypeAndId; - QDate m_date; // only used when rebuilding the DB friend class Core; friend class Database; }; +inline std::weak_ordering ItemChangeLogEntry::operator<=>(const QByteArray &typeAndId) const +{ + return m_fromTypeAndId.compare(typeAndId) <=> 0; +} + +inline std::weak_ordering ItemChangeLogEntry::operator<=>(const ItemChangeLogEntry &other) const +{ + auto cmp = (m_fromTypeAndId.compare(other.m_fromTypeAndId) <=> 0); + return (cmp == 0) ? (m_id <=> other.m_id) : cmp; +} + } // namespace BrickLink diff --git a/src/bricklink/core.cpp b/src/bricklink/core.cpp index 5715da74..6995d409 100755 --- a/src/bricklink/core.cpp +++ b/src/bricklink/core.cpp @@ -816,81 +816,125 @@ void Core::cancelTransfers() m_authenticatedTransfer->abortAllJobs(); } -bool Core::applyChangeLog(const Item *&item, const Color *&color, const Incomplete *inc) +QByteArray Core::applyItemChangeLog(QByteArray itemTypeAndId, uint startAtChangelogId, const QDate &creationDate) { - if (!inc) - return false; + uint changelogId = startAtChangelogId; + bool foundChangelogEntry; - // there are a items that changed their name multiple times, so we have to loop (e.g. 3069bpb78) + constexpr bool dbg = false; + //bool dbg = itemTypeAndId.startsWith("P72206"); - if (!item) { - QByteArray itemTypeAndId = inc->m_itemtype_id + inc->m_item_id; - if (!inc->m_itemtype_name.isEmpty()) - itemTypeAndId[0] = inc->m_itemtype_name.at(0).toUpper().toLatin1(); + do { + foundChangelogEntry = false; + auto [lit, uit] = std::equal_range(itemChangelog().cbegin(), itemChangelog().cend(), itemTypeAndId); - while (!item) { - auto it = std::lower_bound(itemChangelog().cbegin(), itemChangelog().cend(), itemTypeAndId); - if ((it == itemChangelog().cend()) || (*it != itemTypeAndId)) - break; - item = core()->item(it->toItemTypeId(), it->toItemId()); - if (!item) - itemTypeAndId = it->toItemTypeId() + it->toItemId(); + if (dbg) + qDebug(LogResolver) << "SEARCH:" << itemTypeAndId << "has" << std::distance(lit, uit) << "changes"; + + // apply strictly from older to newer, starting at 'startAtChangelogId' or 'creationTime' + // in order to avoid infinite loops + for (auto it = lit; it != uit; ++it) { + if (dbg) + qDebug(LogResolver) << "CHANGE:" << it->toItemTypeAndId() << it->id() << changelogId << it->date() << creationDate; + + if (changelogId) { + if (it->id() <= changelogId) + continue; + changelogId = it->id(); + } else { + if (it->date() <= creationDate) + continue; + changelogId = it->id(); + } + qCInfo(LogResolver).noquote() << "item:" << itemTypeAndId << "->" << it->toItemTypeAndId(); + + itemTypeAndId = it->toItemTypeAndId(); + foundChangelogEntry = true; + break; } - } - if (!color) { - uint colorId = inc->m_color_id; + } while (foundChangelogEntry); - while (!color) { - auto it = std::lower_bound(colorChangelog().cbegin(), colorChangelog().cend(), colorId); - if ((it == colorChangelog().cend()) || (*it != colorId)) - break; - color = core()->color(it->toColorId()); - if (!color) - colorId = it->toColorId(); - } - } - - return (item && color); + return itemTypeAndId; } -Core::ResolveResult Core::resolveIncomplete(Lot *lot) +uint Core::applyColorChangeLog(uint colorId, uint startAtChangelogId, const QDate &creationDate) { - Incomplete *inc = lot->isIncomplete(); + uint changelogId = startAtChangelogId; + bool foundChangelogEntry; + do { + foundChangelogEntry = false; + auto [lit, uit] = std::equal_range(colorChangelog().cbegin(), colorChangelog().cend(), colorId); + + // apply strictly from older to newer, starting at 'startAtChangelogId' or 'creationTime' + // in order to avoid infinite loops + for (auto it = lit; it != uit; ++it) { + if (changelogId) { + if (it->id() <= changelogId) + continue; + changelogId = it->id(); + } else { + if (it->date() <= creationDate) + continue; + changelogId = it->id(); + } + qCInfo(LogResolver) << "color:" << colorId << "->" << it->toColorId(); + + colorId = it->toColorId(); + foundChangelogEntry = true; + break; + } + } while (foundChangelogEntry); + + return colorId; +} + +Core::ResolveResult Core::resolveIncomplete(Lot *lot, uint startAtChangelogId, const QDateTime &creationTime) +{ + // How to apply changelog entries: + // * if startAtChangelogId > 0, use it + // * else if creationTime is valid, use that + // * else do not apply the changelog at all + + Incomplete *inc = lot->isIncomplete(); if (!inc) return ResolveResult::Direct; + QByteArray itemTypeAndId = inc->m_itemtype_id + inc->m_item_id; + if (!inc->m_itemtype_name.isEmpty()) + itemTypeAndId[0] = inc->m_itemtype_name.at(0).toUpper().toLatin1(); + QByteArray resolvedItemTypeAndId = itemTypeAndId; + + uint colorId = inc->m_color_id; + uint resolvedColorId = colorId; + + bool tryToResolveItem = (itemTypeAndId.size() > 1) && itemTypeAndId.at(0); + bool tryToResolveColor = (colorId != Color::InvalidId); + + if (startAtChangelogId || creationTime.isValid()) { + if (tryToResolveItem) + resolvedItemTypeAndId = applyItemChangeLog(itemTypeAndId, startAtChangelogId, creationTime.date()); + if (tryToResolveColor) + resolvedColorId = applyColorChangeLog(colorId, startAtChangelogId, creationTime.date()); + } + const Item *item = nullptr; const Color *color = nullptr; - if ((inc->m_itemtype_id != ItemType::InvalidId) && !inc->m_item_id.isEmpty()) - item = core()->item(inc->m_itemtype_id, inc->m_item_id); - - if (inc->m_color_id != Color::InvalidId) - color = core()->color(inc->m_color_id); + if (tryToResolveItem) + item = core()->item(resolvedItemTypeAndId.at(0), resolvedItemTypeAndId.mid(1)); + if (tryToResolveColor) + color = core()->color(resolvedColorId); if (item) lot->setItem(item); if (color) lot->setColor(color); - if (lot->item() && lot->color()) - return ResolveResult::Direct; - bool ok = applyChangeLog(item, color, inc); - - if (ok) { - qCInfo(LogResolver).nospace() << " [ OK ] " - << (inc->m_itemtype_id ? inc->m_itemtype_id : '?') - << '-' << inc->m_item_id.constData() << " (" << int(inc->m_color_id) << ')' - << " -> " - << item->itemTypeId() - << '-' << item->id().constData() << " (" << color->id() << ')'; - } else { - qCWarning(LogResolver).nospace() << " [FAIL] " - << (inc->m_itemtype_id ? inc->m_itemtype_id : '?') - << '-' << inc->m_item_id.constData() << " (" << int(inc->m_color_id) << ')'; - - if (item) { + if (!lot->item() || !lot->color()) { + if (!lot->item()) { + qCWarning(LogResolver).noquote() << "item:" << resolvedItemTypeAndId << "[failed]"; + } else { inc->m_item_id.clear(); inc->m_item_name.clear(); inc->m_itemtype_id = ItemType::InvalidId; @@ -898,18 +942,21 @@ Core::ResolveResult Core::resolveIncomplete(Lot *lot) inc->m_category_id = Category::InvalidId; inc->m_category_name.clear(); } - if (color) { + if (!lot->color()) { + qCWarning(LogResolver) << "color:" << resolvedColorId << "[failed]"; + } else { inc->m_color_id = Color::InvalidId; inc->m_color_name.clear(); } - } - if (item) - lot->setItem(item); - if (color) - lot->setColor(color); + return ResolveResult::Fail; - Q_ASSERT(ok == !lot->isIncomplete()); - return ok ? ResolveResult::ChangeLog : ResolveResult::Fail; + } else if ((resolvedItemTypeAndId != itemTypeAndId) || (resolvedColorId != colorId)) { + lot->setIncomplete(nullptr); + return ResolveResult::ChangeLog; + } else { + lot->setIncomplete(nullptr); + return ResolveResult::Direct; + } } QSize Core::standardPictureSize() const diff --git a/src/bricklink/core.h b/src/bricklink/core.h index 60b777dd..390583ff 100755 --- a/src/bricklink/core.h +++ b/src/bricklink/core.h @@ -85,6 +85,8 @@ public: inline const std::vector &itemChangelog() const { return database()->m_itemChangelog; } inline const std::vector &colorChangelog() const { return database()->m_colorChangelog; } + inline uint latestChangelogId() const { return database()->m_latestChangelogId; } + const QImage noImage(const QSize &s) const; const Color *color(uint id) const; @@ -99,14 +101,16 @@ public: QSize standardPictureSize() const; - bool applyChangeLog(const Item *&item, const Color *&color, const Incomplete *inc); + QByteArray applyItemChangeLog(QByteArray itemTypeAndId, uint startAtChangelogId, + const QDate &creationDate); + uint applyColorChangeLog(uint colorId, uint startAtChangelogId, const QDate &creationDate); QString countryIdFromName(const QString &name) const; static QString itemHtmlDescription(const Item *item, const Color *color, const QColor &highlight); enum class ResolveResult { Fail, Direct, ChangeLog }; - ResolveResult resolveIncomplete(Lot *lot); + ResolveResult resolveIncomplete(Lot *lot, uint startAtChangelogId, const QDateTime &creationTime); public slots: void setUpdateIntervals(const QMap &intervals); diff --git a/src/bricklink/database.cpp b/src/bricklink/database.cpp index d6eee26a..e9a317ff 100755 --- a/src/bricklink/database.cpp +++ b/src/bricklink/database.cpp @@ -215,7 +215,7 @@ void Database::read(const QString &fileName) } bool gotColors = false, gotCategories = false, gotItemTypes = false, gotItems = false; - bool gotItemChangeLog = false, gotColorChangeLog = false, gotPccs = false; + bool gotChangeLog = false, gotPccs = false; auto check = [&ds, &f]() { if (ds.status() != QDataStream::Ok) @@ -238,6 +238,7 @@ void Database::read(const QString &fileName) std::vector itemChangelog; std::vector colorChangelog; std::vector pccs; + uint latestChangelogId = 0; while (cr.startChunk()) { switch (cr.chunkId() | quint64(cr.chunkVersion()) << 32) { @@ -315,32 +316,25 @@ void Database::read(const QString &fileName) gotItems = true; break; } - case ChunkId('I','C','H','G') | 1ULL << 32: { - quint32 clc = 0; - ds >> clc; + case ChunkId('C','H','G','L') | 2ULL << 32: { + quint32 clid = 0, clic = 0, clcc = 0; + ds >> clid >> clic >> clcc; check(); - sizeCheck(clc, 1'000'000); + sizeCheck(clic, 1'000'000); + sizeCheck(clcc, 1'000); - itemChangelog.resize(clc); - for (quint32 i = 0; i < clc; ++i) { + itemChangelog.resize(clic); + for (quint32 i = 0; i < clic; ++i) { readItemChangeLogFromDatabase(itemChangelog[i], ds, Version::Latest); check(); } - gotItemChangeLog = true; - break; - } - case ChunkId('C','C','H','G') | 1ULL << 32: { - quint32 clc = 0; - ds >> clc; - check(); - sizeCheck(clc, 1'000); - - colorChangelog.resize(clc); - for (quint32 i = 0; i < clc; ++i) { + colorChangelog.resize(clcc); + for (quint32 i = 0; i < clcc; ++i) { readColorChangeLogFromDatabase(colorChangelog[i], ds, Version::Latest); check(); } - gotColorChangeLog = true; + latestChangelogId = clid; + gotChangeLog = true; break; } case ChunkId('P','C','C',' ') | 1ULL << 32: { @@ -375,33 +369,35 @@ void Database::read(const QString &fileName) delete sw; - if (!gotColors || !gotCategories || !gotItemTypes || !gotItems || !gotItemChangeLog - || !gotColorChangeLog || !gotPccs) { + if (!gotColors || !gotCategories || !gotItemTypes || !gotItems || !gotChangeLog || !gotPccs) { throw Exception("not all required data chunks were found in the database (%1)") .arg(f.fileName()); } - if (true) { + { QString out = u"Loaded database from " + f.fileName(); QLocale loc = QLocale(QLocale::Swedish); // space as number group separator QVector> log = { { u"Generated at"_qs, generationDate.toString(Qt::RFC2822Date) }, - { u"ChangeLog I"_qs, loc.toString(itemChangelog.size()).rightJustified(10) }, - { u"ChangeLog C"_qs, loc.toString(colorChangelog.size()).rightJustified(10) }, - { u"PCCs"_qs, loc.toString(pccs.size()).rightJustified(10) }, - { u"Colors"_qs, loc.toString(colors.size()).rightJustified(10) }, - { u"LDraw Colors"_qs, loc.toString(ldrawExtraColors.size()).rightJustified(10) }, + { u"Changelog Id"_qs, QString::number(latestChangelogId) }, + { u" Items"_qs, loc.toString(itemChangelog.size()).rightJustified(10) }, + { u" Colors"_qs, loc.toString(colorChangelog.size()).rightJustified(10) }, { u"Item Types"_qs, loc.toString(itemTypes.size()).rightJustified(10) }, { u"Categories"_qs, loc.toString(categories.size()).rightJustified(10) }, + { u"Colors"_qs, loc.toString(colors.size()).rightJustified(10) }, + { u"LDraw Colors"_qs, loc.toString(ldrawExtraColors.size()).rightJustified(10) }, + { u"PCCs"_qs, loc.toString(pccs.size()).rightJustified(10) }, { u"Items"_qs, loc.toString(items.size()).rightJustified(10) }, }; +#if defined(QT_DEBUG) std::vector itemCount(itemTypes.size()); for (const auto &item : std::as_const(items)) ++itemCount[item.m_itemTypeIndex]; for (size_t i = 0; i < itemTypes.size(); ++i) { log.append({ u" "_qs + itemTypes.at(i).name(), - loc.toString(itemCount.at(i)).rightJustified(10) }); + loc.toString(itemCount.at(i)).rightJustified(10) }); } +#endif qsizetype leftSize = 0; for (const auto &logPair : std::as_const(log)) leftSize = std::max(leftSize, logPair.first.length()); @@ -418,6 +414,7 @@ void Database::read(const QString &fileName) m_itemChangelog = itemChangelog; m_colorChangelog = colorChangelog; m_pccs = pccs; + m_latestChangelogId = latestChangelogId; Color::s_colorImageCache.clear(); @@ -497,7 +494,17 @@ void Database::write(const QString &filename, Version version) const writeItemToDatabase(item, ds, version); check(cw.endChunk()); - if (version >= Version::V5) { + if (version >= Version::V9) { + check(cw.startChunk(ChunkId('C','H','G','L'), 2)); + ds << quint32(m_latestChangelogId) + << quint32(m_itemChangelog.size()) + << quint32(m_colorChangelog.size()); + for (const ItemChangeLogEntry &e : m_itemChangelog) + writeItemChangeLogToDatabase(e, ds, version); + for (const ColorChangeLogEntry &e : m_colorChangelog) + writeColorChangeLogToDatabase(e, ds, version); + check(cw.endChunk()); + } else if (version >= Version::V5) { check(cw.startChunk(ChunkId('I','C','H','G'), 1)); ds << quint32(m_itemChangelog.size()); for (const ItemChangeLogEntry &e : m_itemChangelog) @@ -719,21 +726,25 @@ void Database::writePCCToDatabase(const PartColorCode &pcc, QDataStream &dataStr void Database::readItemChangeLogFromDatabase(ItemChangeLogEntry &e, QDataStream &dataStream, Version) { - dataStream >> e.m_fromTypeAndId >> e.m_toTypeAndId; + dataStream >> e.m_id >> e.m_julianDay >> e.m_fromTypeAndId >> e.m_toTypeAndId; } -void Database::writeItemChangeLogToDatabase(const ItemChangeLogEntry &e, QDataStream &dataStream, Version) const +void Database::writeItemChangeLogToDatabase(const ItemChangeLogEntry &e, QDataStream &dataStream, Version v) const { + if (v >= Version::V9) + dataStream << e.m_id << e.m_julianDay; dataStream << e.m_fromTypeAndId << e.m_toTypeAndId; } void Database::readColorChangeLogFromDatabase(ColorChangeLogEntry &e, QDataStream &dataStream, Version) { - dataStream >> e.m_fromColorId >> e.m_toColorId; + dataStream >> e.m_id >> e.m_julianDay >> e.m_fromColorId >> e.m_toColorId; } -void Database::writeColorChangeLogToDatabase(const ColorChangeLogEntry &e, QDataStream &dataStream, Version) const +void Database::writeColorChangeLogToDatabase(const ColorChangeLogEntry &e, QDataStream &dataStream, Version v) const { + if (v >= Version::V9) + dataStream << e.m_id << e.m_julianDay; dataStream << e.m_fromColorId << e.m_toColorId; } diff --git a/src/bricklink/database.h b/src/bricklink/database.h index 78692b5c..7db17082 100755 --- a/src/bricklink/database.h +++ b/src/bricklink/database.h @@ -43,10 +43,11 @@ public: V6, // 2022.2.1 V7, // 2022.6.1 (not released) V8, // 2022.6.2 + V9, // 2023.3.1 OldestStillSupported = V4, - Latest = V8 + Latest = V9 }; void setUpdateInterval(int interval); @@ -100,6 +101,8 @@ private: std::vector m_colorChangelog; std::vector m_pccs; + uint m_latestChangelogId = 0; + friend class Core; friend class TextImport; diff --git a/src/bricklink/io.cpp b/src/bricklink/io.cpp index 4d81bfad..557c9848 100755 --- a/src/bricklink/io.cpp +++ b/src/bricklink/io.cpp @@ -103,7 +103,7 @@ QString IO::toBrickLinkXML(const LotList &lots) } -IO::ParseResult IO::fromBrickLinkXML(const QByteArray &data, Hint hint) +IO::ParseResult IO::fromBrickLinkXML(const QByteArray &data, Hint hint, const QDateTime &creationTime) { //stopwatch loadXMLWatch("Load XML"); @@ -213,7 +213,7 @@ IO::ParseResult IO::fromBrickLinkXML(const QByteArray &data, Hint hint) xml.skipCurrentElement(); } - switch (core()->resolveIncomplete(lot)) { + switch (core()->resolveIncomplete(lot, 0, creationTime)) { case Core::ResolveResult::Fail: pr.incInvalidLotCount(); break; case Core::ResolveResult::ChangeLog: pr.incFixedLotCount(); break; default: break; diff --git a/src/bricklink/io.h b/src/bricklink/io.h index ca4124cf..603bed88 100755 --- a/src/bricklink/io.h +++ b/src/bricklink/io.h @@ -63,7 +63,7 @@ enum class Hint { }; QString toBrickLinkXML(const LotList &lots); -ParseResult fromBrickLinkXML(const QByteArray &xml, Hint hint = Hint::Plain); +ParseResult fromBrickLinkXML(const QByteArray &xml, Hint hint, const QDateTime &creationTime = { }); ParseResult fromPartInventory(const Item *item, const Color *color = nullptr, int quantity = 1, Condition condition = Condition::New, Status extraParts = Status::Extra, diff --git a/src/bricklink/lot.cpp b/src/bricklink/lot.cpp index 0dc7cf8d..f8307801 100755 --- a/src/bricklink/lot.cpp +++ b/src/bricklink/lot.cpp @@ -116,10 +116,10 @@ Lot::~Lot() void Lot::save(QDataStream &ds) const { - ds << QByteArray("II") << qint32(4) - << QString::fromLatin1(itemId()) - << qint8(itemType() ? itemType()->id() : ItemType::InvalidId) - << uint(color() ? color()->id() : Color::InvalidId) + ds << QByteArray("II") << qint32(5) + << itemId() + << (itemType() ? itemType()->id() : ItemType::InvalidId) + << (color() ? color()->id() : Color::InvalidId) << qint8(m_status) << qint8(m_condition) << qint8(m_scondition) << qint8(m_retain ? 1 : 0) << qint8(m_stockroom) << m_lot_id << m_reserved << m_comments << m_remarks << m_quantity << m_bulk_quantity @@ -131,61 +131,63 @@ void Lot::save(QDataStream &ds) const << m_dateAdded << m_dateLastSold; } -Lot *Lot::restore(QDataStream &ds) +Lot *Lot::restore(QDataStream &ds, uint startChangelogAt) { std::unique_ptr lot; QByteArray tag; qint32 version; ds >> tag >> version; - if ((ds.status() != QDataStream::Ok) || (tag != "II") || (version < 2) || (version > 4)) + if ((ds.status() != QDataStream::Ok) || (tag != "II") || (version != 5)) return nullptr; - QString itemid; - uint colorid = 0; - qint8 itemtypeid = 0; + QByteArray itemId; + uint colorId = 0; + char itemTypeId = 0; - ds >> itemid >> itemtypeid >> colorid; + ds >> itemId >> itemTypeId >> colorId; if (ds.status() != QDataStream::Ok) return nullptr; - auto item = core()->item(itemtypeid, itemid.toLatin1()); - auto color = core()->color(colorid); - std::unique_ptr inc; + if (startChangelogAt) { + const QByteArray itemTypeAndId = itemTypeId + itemId; + QByteArray newId = core()->applyItemChangeLog(itemTypeAndId, startChangelogAt, { }); + if (newId != itemTypeAndId) { + itemTypeId = newId.at(0); + itemId = newId.mid(1); + } + colorId = core()->applyColorChangeLog(colorId, startChangelogAt, { }); + } + auto item = core()->item(itemTypeId, itemId); + auto color = core()->color(colorId); + + lot = std::make_unique(item, color); if (!item || !color) { - inc = std::make_unique(); + auto *inc = new Incomplete; if (!item) { - inc->m_item_id = itemid.toLatin1(); - inc->m_itemtype_id = itemtypeid; + inc->m_item_id = itemId; + inc->m_itemtype_id = itemTypeId; } if (!color) { - inc->m_color_id = colorid; - inc->m_color_name = QString::number(colorid); + inc->m_color_id = colorId; + inc->m_color_name = u"BL #"_qs + QString::number(colorId); } - - if (core()->applyChangeLog(item, color, inc.get())) - inc.reset(); + lot->setIncomplete(inc); } - lot = std::make_unique(item, color); - if (inc) - lot->setIncomplete(inc.release()); // alternate, cpart and altid are left out on purpose! qint8 status = 0, cond = 0, scond = 0, retain = 0, stockroom = 0; ds >> status >> cond >> scond >> retain >> stockroom - >> lot->m_lot_id >> lot->m_reserved >> lot->m_comments >> lot->m_remarks - >> lot->m_quantity >> lot->m_bulk_quantity - >> lot->m_tier_quantity[0] >> lot->m_tier_quantity[1] >> lot->m_tier_quantity[2] - >> lot->m_sale >> lot->m_price >> lot->m_cost - >> lot->m_tier_price[0] >> lot->m_tier_price[1] >> lot->m_tier_price[2] - >> lot->m_weight; - if (version >= 3) - ds >> lot->m_markerText >> lot->m_markerColor; - if (version >= 4) - ds >> lot->m_dateAdded >> lot->m_dateLastSold; + >> lot->m_lot_id >> lot->m_reserved >> lot->m_comments >> lot->m_remarks + >> lot->m_quantity >> lot->m_bulk_quantity + >> lot->m_tier_quantity[0] >> lot->m_tier_quantity[1] >> lot->m_tier_quantity[2] + >> lot->m_sale >> lot->m_price >> lot->m_cost + >> lot->m_tier_price[0] >> lot->m_tier_price[1] >> lot->m_tier_price[2] + >> lot->m_weight >> lot->m_markerText >> lot->m_markerColor + >> lot->m_dateAdded >> lot->m_dateLastSold; if (ds.status() != QDataStream::Ok) return nullptr; diff --git a/src/bricklink/lot.h b/src/bricklink/lot.h index 517626e3..919bf116 100755 --- a/src/bricklink/lot.h +++ b/src/bricklink/lot.h @@ -141,7 +141,7 @@ public: void setIncomplete(Incomplete *inc) { m_incomplete.reset(inc); } void save(QDataStream &ds) const; - static Lot *restore(QDataStream &ds); + static Lot *restore(QDataStream &ds, uint startChangelogAt); private: const Item * m_item; diff --git a/src/bricklink/textimport.cpp b/src/bricklink/textimport.cpp index 4fdf3bc0..9e364e0c 100644 --- a/src/bricklink/textimport.cpp +++ b/src/bricklink/textimport.cpp @@ -390,12 +390,14 @@ bool BrickLink::TextImport::readInventory(const Item *item, ImportInventoriesSte // if this itemid was involved in a changelog entry after the last time we downloaded // the inventory, we need to reload QByteArray itemTypeAndId = itemTypeId + itemId; - auto it = std::lower_bound(m_itemChangelog.cbegin(), m_itemChangelog.cend(), itemTypeAndId); - if ((it != m_itemChangelog.cend()) && (*it == itemTypeAndId) && (it->date() > fileDate)) { - throw Exception("Item id %1 changed on %2 (last download: %3)") - .arg(QString::fromLatin1(itemTypeAndId)) - .arg(it->date().toString(u"yyyy/MM/dd")) - .arg(fileDate.toString(u"yyyy/MM/dd")); + auto [lit, uit] = std::equal_range(m_itemChangelog.cbegin(), m_itemChangelog.cend(), itemTypeAndId); + for (auto it = lit; it != uit; ++it) { + if (it->date() > fileDate) { + throw Exception("Item id %1 changed on %2 (last download: %3)") + .arg(QString::fromLatin1(itemTypeAndId)) + .arg(it->date().toString(u"yyyy/MM/dd")) + .arg(fileDate.toString(u"yyyy/MM/dd")); + } } inventory.append(co); @@ -752,6 +754,8 @@ void BrickLink::TextImport::readChangeLog(const QString &path) if (!f.open(QFile::ReadOnly)) throw ParseException(&f, "could not open file"); + m_latestChangelogId = 0; + QTextStream ts(&f); int lineNumber = 0; while (!ts.atEnd()) { @@ -764,21 +768,23 @@ void BrickLink::TextImport::readChangeLog(const QString &path) if (strs.count() < 7) throw ParseException(&f, "expected at least 7 fields in line %1").arg(lineNumber); + uint id = strs.at(0).toUInt(); + QDate date = QDate::fromString(strs.at(1), u"M/d/yyyy"_qs); char c = ItemType::idFromFirstCharInString(strs.at(2)); - QDate date = QDate::fromString(strs.at(1), u"M/d/yyyy"_qs); // not stored in the database switch (c) { case 'I': // ItemId case 'T': // ItemType case 'M': { // ItemMerge - QString fromType = strs.at(3); - QString fromId = strs.at(4); - QString toType = strs.at(5); - QString toId = strs.at(6); + const QString &fromType = strs.at(3); + const QString &fromId = strs.at(4); + const QString &toType = strs.at(5); + const QString &toId = strs.at(6); if ((fromType.length() == 1) && (toType.length() == 1) && !fromId.isEmpty() && !toId.isEmpty()) { - m_itemChangelog.emplace_back((fromType + fromId).toLatin1(), - (toType + toId).toLatin1(), date); + m_itemChangelog.emplace_back(id, date, (fromType + fromId).toLatin1(), + (toType + toId).toLatin1()); + m_latestChangelogId = std::max(m_latestChangelogId, id); } break; } @@ -786,8 +792,10 @@ void BrickLink::TextImport::readChangeLog(const QString &path) bool fromOk = false, toOk = false; uint fromId = strs.at(3).toUInt(&fromOk); uint toId = strs.at(5).toUInt(&toOk); - if (fromOk && toOk) - m_colorChangelog.emplace_back(fromId, toId, date); + if (fromOk && toOk) { + m_colorChangelog.emplace_back(id, date, fromId, toId); + m_latestChangelogId = std::max(m_latestChangelogId, id); + } break; } case 'E': // CategoryName @@ -813,6 +821,7 @@ void BrickLink::TextImport::exportTo(Database *db) std::swap(db->m_pccs, m_pccs); std::swap(db->m_itemChangelog, m_itemChangelog); std::swap(db->m_colorChangelog, m_colorChangelog); + db->m_latestChangelogId = m_latestChangelogId; for (auto it = m_consists_of_hash.cbegin(); it != m_consists_of_hash.cend(); ++it) { Item &item = db->m_items[it.key()]; diff --git a/src/bricklink/textimport.h b/src/bricklink/textimport.h index 6522c9ad..55d6e4cf 100755 --- a/src/bricklink/textimport.h +++ b/src/bricklink/textimport.h @@ -72,6 +72,8 @@ private: QHash>>> m_appears_in_hash; // item-idx -> { vector < consists-of > } QHash> m_consists_of_hash; + + uint m_latestChangelogId = 0; }; } // namespace BrickLink diff --git a/src/common/document.cpp b/src/common/document.cpp index 4ac35a96..47bfabcd 100755 --- a/src/common/document.cpp +++ b/src/common/document.cpp @@ -2444,12 +2444,13 @@ void Document::autosave() const QByteArray ba; QDataStream ds(&ba, QIODevice::WriteOnly); ds << QByteArray(autosaveMagic) - << qint32(5) // version + << qint32(6) // version << title() << filePath() << m_model->currencyCode() << saveColumnsState() << model()->saveSortFilterState() + << BrickLink::core()->latestChangelogId() << qint32(lots.count()); for (auto lot : lots) { @@ -2488,24 +2489,25 @@ int Document::processAutosaves(AutosaveAction action) QByteArray columnState; QByteArray savedSortFilterState; qint32 count = 0; + uint startChangelogAt = 0; QDataStream ds(&f); ds >> magic >> version; - if ((magic != QByteArray(autosaveMagic)) || (version != 5)) + if ((magic != QByteArray(autosaveMagic)) || (version != 6)) continue; ds >> savedTitle >> savedFileName >> savedCurrencyCode >> columnState - >> savedSortFilterState >> count; + >> savedSortFilterState >> startChangelogAt >> count; BrickLink::IO::ParseResult pr; pr.setCurrencyCode(savedCurrencyCode); if (count > 0) { for (int i = 0; i < count; ++i) { - if (auto lot = Lot::restore(ds)) { + if (auto lot = Lot::restore(ds, startChangelogAt)) { bool hasBase = false; ds >> hasBase; if (hasBase) { - if (auto base = Lot::restore(ds)) { + if (auto base = Lot::restore(ds, startChangelogAt)) { pr.addToDifferenceModeBase(lot, *base); delete base; } else { diff --git a/src/common/documentio.cpp b/src/common/documentio.cpp index 9d043bb7..4dc7c3bf 100755 --- a/src/common/documentio.cpp +++ b/src/common/documentio.cpp @@ -141,7 +141,9 @@ QCoro::Task DocumentIO::importBrickLinkXML(QString fileName) QFile f(fn); if (f.open(QIODevice::ReadOnly)) { try { - auto result = BrickLink::IO::fromBrickLinkXML(f.readAll(), BrickLink::IO::Hint::PlainOrWanted); + auto result = BrickLink::IO::fromBrickLinkXML(f.readAll(), + BrickLink::IO::Hint::PlainOrWanted, + f.fileTime(QFile::FileModificationTime)); auto *document = new Document(new DocumentModel(std::move(result))); // Document owns the items now document->setTitle(tr("Import of %1").arg(QFileInfo(fn).fileName())); co_return document; @@ -423,13 +425,15 @@ bool DocumentIO::parseLDrawModelInternal(QFile *f, bool isStudio, const QString -Document *DocumentIO::parseBsxInventory(QIODevice *in) +Document *DocumentIO::parseBsxInventory(QFile *in) { //stopwatch loadBsxWatch("Load BSX"); Q_ASSERT(in); QXmlStreamReader xml(in); BsxContents bsx; + QDateTime creationTime = in->fileTime(QFile::FileModificationTime); + uint startAtChangelogId = 0; try { bsx.setCurrencyCode(u"$$$"_qs); // flag as legacy currency @@ -550,7 +554,7 @@ Document *DocumentIO::parseBsxInventory(QIODevice *in) } } - switch (BrickLink::core()->resolveIncomplete(lot)) { + switch (BrickLink::core()->resolveIncomplete(lot, startAtChangelogId, creationTime)) { case BrickLink::Core::ResolveResult::Fail: bsx.incInvalidLotCount(); break; case BrickLink::Core::ResolveResult::ChangeLog: bsx.incFixedLotCount(); break; default: break; @@ -575,7 +579,8 @@ Document *DocumentIO::parseBsxInventory(QIODevice *in) (*it)(&base, attr.value().toString()); } } - if (BrickLink::core()->resolveIncomplete(&base) == BrickLink::Core::ResolveResult::Fail) { + if (BrickLink::core()->resolveIncomplete(&base, startAtChangelogId, creationTime) + == BrickLink::Core::ResolveResult::Fail) { if (!base.item() && lot->item()) base.setItem(lot->item()); if (!base.color() && lot->color()) @@ -618,6 +623,7 @@ Document *DocumentIO::parseBsxInventory(QIODevice *in) if (xml.name() == u"Inventory") { foundInventory = true; bsx.setCurrencyCode(xml.attributes().value(u"Currency"_qs).toString()); + startAtChangelogId = xml.attributes().value(u"BrickLinkChangelogId"_qs).toUInt(); parseInventory(); } else if ((xml.name() == u"GuiState") && (xml.attributes().value(u"Application"_qs) == u"BrickStore") @@ -668,6 +674,7 @@ bool DocumentIO::createBsxInventory(QIODevice *out, const Document *doc) xml.writeStartElement(u"BrickStoreXML"_qs); xml.writeStartElement(u"Inventory"_qs); xml.writeAttribute(u"Currency"_qs, doc->model()->currencyCode()); + xml.writeAttribute(u"BrickLinkChangelogId"_qs, QString::number(BrickLink::core()->latestChangelogId())); const Lot *lot; const Lot *base; diff --git a/src/common/documentio.h b/src/common/documentio.h index 47cf3414..d97229a4 100755 --- a/src/common/documentio.h +++ b/src/common/documentio.h @@ -51,7 +51,7 @@ public: static QString exportBrickLinkUpdateClipboard(const DocumentModel *doc, const LotList &lots); - static Document *parseBsxInventory(QIODevice *in); + static Document *parseBsxInventory(QFile *in); static bool createBsxInventory(QIODevice *out, const Document *doc); private: diff --git a/src/common/documentmodel.cpp b/src/common/documentmodel.cpp index 48adc6f4..30ba592b 100755 --- a/src/common/documentmodel.cpp +++ b/src/common/documentmodel.cpp @@ -2758,9 +2758,9 @@ void DocumentLotsMimeData::setLots(const LotList &lots, const QString ¤cyC QString text; QDataStream ds(&data, QIODevice::WriteOnly); - ds << QByteArray("LOTS") << qint32(1); + ds << QByteArray("LOTS") << qint32(2); - ds << currencyCode << quint32(lots.count()); + ds << currencyCode << BrickLink::core()->latestChangelogId() << quint32(lots.count()); for (const Lot *lot : lots) { lot->save(ds); if (!text.isEmpty()) @@ -2779,22 +2779,23 @@ std::tuple DocumentLotsMimeData::lots(const QMimeData *md) if (md) { QByteArray data = md->data(s_mimetype); QDataStream ds(data); + uint startChangelogAt = 0; QByteArray tag; qint32 version; ds >> tag >> version; - if ((ds.status() != QDataStream::Ok) || (tag != "LOTS") || (version != 1)) + if ((ds.status() != QDataStream::Ok) || (tag != "LOTS") || (version != 2)) return { }; quint32 count = 0; - ds >> currencyCode >> count; + ds >> currencyCode >> startChangelogAt >> count; if ((ds.status() != QDataStream::Ok) || (currencyCode.size() != 3) || (count > 1000000)) return { }; lots.reserve(count); for (; count && !ds.atEnd(); count--) { - if (auto lot = Lot::restore(ds)) + if (auto lot = Lot::restore(ds, startChangelogAt)) lots << lot; } }