From 8048974b838e2f6fd811d0c7a9ad5351ce5d34eb Mon Sep 17 00:00:00 2001 From: Robert Griebl Date: Tue, 14 Mar 2023 13:14:37 +0100 Subject: [PATCH] Correctly implement ChangeLog handling for item and color ids The old changelog handling did not take into account that the changelog had to be applied from oldest to newest to correctly catch id swaps. This was also due to the fact, that we didn't know where to start in the changelog history. Adding the recursive resolve to this broken setup ended in infinte loops on some item renames (e.g. 72206) Newly saved BSX files now carry the latest BL changelog id to correctly resolve any future renames. For old BSX files, as well as BrickLink XML files, the file modification time will be used as a starting point in the changelog. This is just a best guess though, because BrickLink's temporal precision for changelog entries is a day, plus the BrickStore database is typically a few days older than the file being saved. It does work out quite well in practice though. Closes: #686 --- BrickStoreXML.rnc | 6 +- src/bricklink/changelogentry.h | 85 +++++++++++------ src/bricklink/core.cpp | 169 +++++++++++++++++++++------------ src/bricklink/core.h | 8 +- src/bricklink/database.cpp | 77 ++++++++------- src/bricklink/database.h | 5 +- src/bricklink/io.cpp | 4 +- src/bricklink/io.h | 2 +- src/bricklink/lot.cpp | 70 +++++++------- src/bricklink/lot.h | 2 +- src/bricklink/textimport.cpp | 39 +++++--- src/bricklink/textimport.h | 2 + src/common/document.cpp | 12 ++- src/common/documentio.cpp | 15 ++- src/common/documentio.h | 2 +- src/common/documentmodel.cpp | 11 ++- 16 files changed, 313 insertions(+), 196 deletions(-) diff --git a/BrickStoreXML.rnc b/BrickStoreXML.rnc index d612c39a..f8560f5c 100755 --- a/BrickStoreXML.rnc +++ b/BrickStoreXML.rnc @@ -18,10 +18,14 @@ BrickStoreXML = element BrickStoreXML | BrickStockXML { Inventory, GuiState? } -# Currency is optional. The default currency is USD: +# Currency is optional. The default currency is USD. +# +# BrickLinkChangelogId is optional, but quite helpful when automatically applying item and color +# renames and merges (see also https://www.bricklink.com/btchglog.asp?viewHelp=Y) Inventory = element Inventory { attribute Currency { xsd:string { length="3" } }?, + attribute BrickLinkChangelogId { xsd:integer } }?, Item* } diff --git a/src/bricklink/changelogentry.h b/src/bricklink/changelogentry.h index d61c8770..e621e549 100755 --- a/src/bricklink/changelogentry.h +++ b/src/bricklink/changelogentry.h @@ -15,62 +15,87 @@ namespace BrickLink { class ColorChangeLogEntry { public: - uint fromColorId() const { return m_fromColorId; } - uint toColorId() const { return m_toColorId; } + uint id() const { return m_id; } + QDate date() const { return QDate::fromJulianDay(m_julianDay); } + uint fromColorId() const { return m_fromColorId; } + uint toColorId() const { return m_toColorId; } - explicit ColorChangeLogEntry(uint fromColorId = Color::InvalidId, uint toColorId = Color::InvalidId, - QDate date = { }) - - : m_fromColorId(fromColorId) + explicit ColorChangeLogEntry() = default; + explicit ColorChangeLogEntry(uint id, const QDate &date, uint fromColorId, uint toColorId) + : m_id(id) + , m_julianDay(uint(date.toJulianDay())) + , m_fromColorId(fromColorId) , m_toColorId(toColorId) - , m_date(date) { } - constexpr std::weak_ordering operator<=>(uint colorId) const { return m_fromColorId <=> colorId; } - constexpr std::weak_ordering operator<=>(const ColorChangeLogEntry &other) const { return *this <=> other.m_fromColorId; } - constexpr bool operator==(uint colorId) const { return (*this <=> colorId == 0); } - constexpr bool operator==(const ColorChangeLogEntry &other) const { return *this == other.m_fromColorId; } + std::weak_ordering operator<=>(uint colorId) const; + std::weak_ordering operator<=>(const ColorChangeLogEntry &other) const; private: - uint m_fromColorId; - uint m_toColorId; - QDate m_date; // only used when rebuilding the DB + uint m_id = 0; + uint m_julianDay = 0; + uint m_fromColorId = Color::InvalidId; + uint m_toColorId = Color::InvalidId; friend class Core; friend class Database; }; +inline std::weak_ordering ColorChangeLogEntry::operator<=>(uint colorId) const +{ + return m_fromColorId <=> colorId; +} + +inline std::weak_ordering ColorChangeLogEntry::operator<=>(const ColorChangeLogEntry &other) const +{ + auto cmp = (m_fromColorId <=> other.m_fromColorId); + return (cmp == 0) ? (m_id <=> other.m_id) : cmp; +} + + class ItemChangeLogEntry { public: - char fromItemTypeId() const { return m_fromTypeAndId.at(0); } - QByteArray fromItemId() const { return m_fromTypeAndId.mid(1); } + uint id() const { return m_id; } + QDate date() const { return QDate::fromJulianDay(m_julianDay); } + char fromItemTypeId() const { return m_fromTypeAndId.at(0); } + QByteArray fromItemId() const { return m_fromTypeAndId.mid(1); } + char toItemTypeId() const { return m_toTypeAndId.at(0); } + QByteArray toItemId() const { return m_toTypeAndId.mid(1); } + QByteArray toItemTypeAndId() const { return m_toTypeAndId; } - char toItemTypeId() const { return m_toTypeAndId.at(0); } - QByteArray toItemId() const { return m_toTypeAndId.mid(1); } - - QDate date() const { return m_date; } - - explicit ItemChangeLogEntry(const QByteArray &fromTypeAndId = { }, const QByteArray &toTypeAndId = { }, - QDate date = { }) - : m_fromTypeAndId(fromTypeAndId) + explicit ItemChangeLogEntry() = default; + explicit ItemChangeLogEntry(uint id, const QDate &date, const QByteArray &fromTypeAndId, + const QByteArray &toTypeAndId) + : m_id(id) + , m_julianDay(uint(date.toJulianDay())) + , m_fromTypeAndId(fromTypeAndId) , m_toTypeAndId(toTypeAndId) - , m_date(date) { } - std::weak_ordering operator<=>(const QByteArray &typeAndId) const { return m_fromTypeAndId.compare(typeAndId) <=> 0; } - std::weak_ordering operator<=>(const ItemChangeLogEntry &other) const { return *this <=> other.m_fromTypeAndId; } - bool operator==(const QByteArray &typeAndId) const { return (*this <=> typeAndId == 0); } - bool operator==(const ItemChangeLogEntry &other) const { return *this == other.m_fromTypeAndId; } + std::weak_ordering operator<=>(const QByteArray &typeAndId) const; + std::weak_ordering operator<=>(const ItemChangeLogEntry &other) const; private: + uint m_id = 0; + uint m_julianDay = 0; QByteArray m_fromTypeAndId; QByteArray m_toTypeAndId; - QDate m_date; // only used when rebuilding the DB friend class Core; friend class Database; }; +inline std::weak_ordering ItemChangeLogEntry::operator<=>(const QByteArray &typeAndId) const +{ + return m_fromTypeAndId.compare(typeAndId) <=> 0; +} + +inline std::weak_ordering ItemChangeLogEntry::operator<=>(const ItemChangeLogEntry &other) const +{ + auto cmp = (m_fromTypeAndId.compare(other.m_fromTypeAndId) <=> 0); + return (cmp == 0) ? (m_id <=> other.m_id) : cmp; +} + } // namespace BrickLink diff --git a/src/bricklink/core.cpp b/src/bricklink/core.cpp index 5715da74..6995d409 100755 --- a/src/bricklink/core.cpp +++ b/src/bricklink/core.cpp @@ -816,81 +816,125 @@ void Core::cancelTransfers() m_authenticatedTransfer->abortAllJobs(); } -bool Core::applyChangeLog(const Item *&item, const Color *&color, const Incomplete *inc) +QByteArray Core::applyItemChangeLog(QByteArray itemTypeAndId, uint startAtChangelogId, const QDate &creationDate) { - if (!inc) - return false; + uint changelogId = startAtChangelogId; + bool foundChangelogEntry; - // there are a items that changed their name multiple times, so we have to loop (e.g. 3069bpb78) + constexpr bool dbg = false; + //bool dbg = itemTypeAndId.startsWith("P72206"); - if (!item) { - QByteArray itemTypeAndId = inc->m_itemtype_id + inc->m_item_id; - if (!inc->m_itemtype_name.isEmpty()) - itemTypeAndId[0] = inc->m_itemtype_name.at(0).toUpper().toLatin1(); + do { + foundChangelogEntry = false; + auto [lit, uit] = std::equal_range(itemChangelog().cbegin(), itemChangelog().cend(), itemTypeAndId); - while (!item) { - auto it = std::lower_bound(itemChangelog().cbegin(), itemChangelog().cend(), itemTypeAndId); - if ((it == itemChangelog().cend()) || (*it != itemTypeAndId)) - break; - item = core()->item(it->toItemTypeId(), it->toItemId()); - if (!item) - itemTypeAndId = it->toItemTypeId() + it->toItemId(); + if (dbg) + qDebug(LogResolver) << "SEARCH:" << itemTypeAndId << "has" << std::distance(lit, uit) << "changes"; + + // apply strictly from older to newer, starting at 'startAtChangelogId' or 'creationTime' + // in order to avoid infinite loops + for (auto it = lit; it != uit; ++it) { + if (dbg) + qDebug(LogResolver) << "CHANGE:" << it->toItemTypeAndId() << it->id() << changelogId << it->date() << creationDate; + + if (changelogId) { + if (it->id() <= changelogId) + continue; + changelogId = it->id(); + } else { + if (it->date() <= creationDate) + continue; + changelogId = it->id(); + } + qCInfo(LogResolver).noquote() << "item:" << itemTypeAndId << "->" << it->toItemTypeAndId(); + + itemTypeAndId = it->toItemTypeAndId(); + foundChangelogEntry = true; + break; } - } - if (!color) { - uint colorId = inc->m_color_id; + } while (foundChangelogEntry); - while (!color) { - auto it = std::lower_bound(colorChangelog().cbegin(), colorChangelog().cend(), colorId); - if ((it == colorChangelog().cend()) || (*it != colorId)) - break; - color = core()->color(it->toColorId()); - if (!color) - colorId = it->toColorId(); - } - } - - return (item && color); + return itemTypeAndId; } -Core::ResolveResult Core::resolveIncomplete(Lot *lot) +uint Core::applyColorChangeLog(uint colorId, uint startAtChangelogId, const QDate &creationDate) { - Incomplete *inc = lot->isIncomplete(); + uint changelogId = startAtChangelogId; + bool foundChangelogEntry; + do { + foundChangelogEntry = false; + auto [lit, uit] = std::equal_range(colorChangelog().cbegin(), colorChangelog().cend(), colorId); + + // apply strictly from older to newer, starting at 'startAtChangelogId' or 'creationTime' + // in order to avoid infinite loops + for (auto it = lit; it != uit; ++it) { + if (changelogId) { + if (it->id() <= changelogId) + continue; + changelogId = it->id(); + } else { + if (it->date() <= creationDate) + continue; + changelogId = it->id(); + } + qCInfo(LogResolver) << "color:" << colorId << "->" << it->toColorId(); + + colorId = it->toColorId(); + foundChangelogEntry = true; + break; + } + } while (foundChangelogEntry); + + return colorId; +} + +Core::ResolveResult Core::resolveIncomplete(Lot *lot, uint startAtChangelogId, const QDateTime &creationTime) +{ + // How to apply changelog entries: + // * if startAtChangelogId > 0, use it + // * else if creationTime is valid, use that + // * else do not apply the changelog at all + + Incomplete *inc = lot->isIncomplete(); if (!inc) return ResolveResult::Direct; + QByteArray itemTypeAndId = inc->m_itemtype_id + inc->m_item_id; + if (!inc->m_itemtype_name.isEmpty()) + itemTypeAndId[0] = inc->m_itemtype_name.at(0).toUpper().toLatin1(); + QByteArray resolvedItemTypeAndId = itemTypeAndId; + + uint colorId = inc->m_color_id; + uint resolvedColorId = colorId; + + bool tryToResolveItem = (itemTypeAndId.size() > 1) && itemTypeAndId.at(0); + bool tryToResolveColor = (colorId != Color::InvalidId); + + if (startAtChangelogId || creationTime.isValid()) { + if (tryToResolveItem) + resolvedItemTypeAndId = applyItemChangeLog(itemTypeAndId, startAtChangelogId, creationTime.date()); + if (tryToResolveColor) + resolvedColorId = applyColorChangeLog(colorId, startAtChangelogId, creationTime.date()); + } + const Item *item = nullptr; const Color *color = nullptr; - if ((inc->m_itemtype_id != ItemType::InvalidId) && !inc->m_item_id.isEmpty()) - item = core()->item(inc->m_itemtype_id, inc->m_item_id); - - if (inc->m_color_id != Color::InvalidId) - color = core()->color(inc->m_color_id); + if (tryToResolveItem) + item = core()->item(resolvedItemTypeAndId.at(0), resolvedItemTypeAndId.mid(1)); + if (tryToResolveColor) + color = core()->color(resolvedColorId); if (item) lot->setItem(item); if (color) lot->setColor(color); - if (lot->item() && lot->color()) - return ResolveResult::Direct; - bool ok = applyChangeLog(item, color, inc); - - if (ok) { - qCInfo(LogResolver).nospace() << " [ OK ] " - << (inc->m_itemtype_id ? inc->m_itemtype_id : '?') - << '-' << inc->m_item_id.constData() << " (" << int(inc->m_color_id) << ')' - << " -> " - << item->itemTypeId() - << '-' << item->id().constData() << " (" << color->id() << ')'; - } else { - qCWarning(LogResolver).nospace() << " [FAIL] " - << (inc->m_itemtype_id ? inc->m_itemtype_id : '?') - << '-' << inc->m_item_id.constData() << " (" << int(inc->m_color_id) << ')'; - - if (item) { + if (!lot->item() || !lot->color()) { + if (!lot->item()) { + qCWarning(LogResolver).noquote() << "item:" << resolvedItemTypeAndId << "[failed]"; + } else { inc->m_item_id.clear(); inc->m_item_name.clear(); inc->m_itemtype_id = ItemType::InvalidId; @@ -898,18 +942,21 @@ Core::ResolveResult Core::resolveIncomplete(Lot *lot) inc->m_category_id = Category::InvalidId; inc->m_category_name.clear(); } - if (color) { + if (!lot->color()) { + qCWarning(LogResolver) << "color:" << resolvedColorId << "[failed]"; + } else { inc->m_color_id = Color::InvalidId; inc->m_color_name.clear(); } - } - if (item) - lot->setItem(item); - if (color) - lot->setColor(color); + return ResolveResult::Fail; - Q_ASSERT(ok == !lot->isIncomplete()); - return ok ? ResolveResult::ChangeLog : ResolveResult::Fail; + } else if ((resolvedItemTypeAndId != itemTypeAndId) || (resolvedColorId != colorId)) { + lot->setIncomplete(nullptr); + return ResolveResult::ChangeLog; + } else { + lot->setIncomplete(nullptr); + return ResolveResult::Direct; + } } QSize Core::standardPictureSize() const diff --git a/src/bricklink/core.h b/src/bricklink/core.h index 60b777dd..390583ff 100755 --- a/src/bricklink/core.h +++ b/src/bricklink/core.h @@ -85,6 +85,8 @@ public: inline const std::vector &itemChangelog() const { return database()->m_itemChangelog; } inline const std::vector &colorChangelog() const { return database()->m_colorChangelog; } + inline uint latestChangelogId() const { return database()->m_latestChangelogId; } + const QImage noImage(const QSize &s) const; const Color *color(uint id) const; @@ -99,14 +101,16 @@ public: QSize standardPictureSize() const; - bool applyChangeLog(const Item *&item, const Color *&color, const Incomplete *inc); + QByteArray applyItemChangeLog(QByteArray itemTypeAndId, uint startAtChangelogId, + const QDate &creationDate); + uint applyColorChangeLog(uint colorId, uint startAtChangelogId, const QDate &creationDate); QString countryIdFromName(const QString &name) const; static QString itemHtmlDescription(const Item *item, const Color *color, const QColor &highlight); enum class ResolveResult { Fail, Direct, ChangeLog }; - ResolveResult resolveIncomplete(Lot *lot); + ResolveResult resolveIncomplete(Lot *lot, uint startAtChangelogId, const QDateTime &creationTime); public slots: void setUpdateIntervals(const QMap &intervals); diff --git a/src/bricklink/database.cpp b/src/bricklink/database.cpp index d6eee26a..e9a317ff 100755 --- a/src/bricklink/database.cpp +++ b/src/bricklink/database.cpp @@ -215,7 +215,7 @@ void Database::read(const QString &fileName) } bool gotColors = false, gotCategories = false, gotItemTypes = false, gotItems = false; - bool gotItemChangeLog = false, gotColorChangeLog = false, gotPccs = false; + bool gotChangeLog = false, gotPccs = false; auto check = [&ds, &f]() { if (ds.status() != QDataStream::Ok) @@ -238,6 +238,7 @@ void Database::read(const QString &fileName) std::vector itemChangelog; std::vector colorChangelog; std::vector pccs; + uint latestChangelogId = 0; while (cr.startChunk()) { switch (cr.chunkId() | quint64(cr.chunkVersion()) << 32) { @@ -315,32 +316,25 @@ void Database::read(const QString &fileName) gotItems = true; break; } - case ChunkId('I','C','H','G') | 1ULL << 32: { - quint32 clc = 0; - ds >> clc; + case ChunkId('C','H','G','L') | 2ULL << 32: { + quint32 clid = 0, clic = 0, clcc = 0; + ds >> clid >> clic >> clcc; check(); - sizeCheck(clc, 1'000'000); + sizeCheck(clic, 1'000'000); + sizeCheck(clcc, 1'000); - itemChangelog.resize(clc); - for (quint32 i = 0; i < clc; ++i) { + itemChangelog.resize(clic); + for (quint32 i = 0; i < clic; ++i) { readItemChangeLogFromDatabase(itemChangelog[i], ds, Version::Latest); check(); } - gotItemChangeLog = true; - break; - } - case ChunkId('C','C','H','G') | 1ULL << 32: { - quint32 clc = 0; - ds >> clc; - check(); - sizeCheck(clc, 1'000); - - colorChangelog.resize(clc); - for (quint32 i = 0; i < clc; ++i) { + colorChangelog.resize(clcc); + for (quint32 i = 0; i < clcc; ++i) { readColorChangeLogFromDatabase(colorChangelog[i], ds, Version::Latest); check(); } - gotColorChangeLog = true; + latestChangelogId = clid; + gotChangeLog = true; break; } case ChunkId('P','C','C',' ') | 1ULL << 32: { @@ -375,33 +369,35 @@ void Database::read(const QString &fileName) delete sw; - if (!gotColors || !gotCategories || !gotItemTypes || !gotItems || !gotItemChangeLog - || !gotColorChangeLog || !gotPccs) { + if (!gotColors || !gotCategories || !gotItemTypes || !gotItems || !gotChangeLog || !gotPccs) { throw Exception("not all required data chunks were found in the database (%1)") .arg(f.fileName()); } - if (true) { + { QString out = u"Loaded database from " + f.fileName(); QLocale loc = QLocale(QLocale::Swedish); // space as number group separator QVector> log = { { u"Generated at"_qs, generationDate.toString(Qt::RFC2822Date) }, - { u"ChangeLog I"_qs, loc.toString(itemChangelog.size()).rightJustified(10) }, - { u"ChangeLog C"_qs, loc.toString(colorChangelog.size()).rightJustified(10) }, - { u"PCCs"_qs, loc.toString(pccs.size()).rightJustified(10) }, - { u"Colors"_qs, loc.toString(colors.size()).rightJustified(10) }, - { u"LDraw Colors"_qs, loc.toString(ldrawExtraColors.size()).rightJustified(10) }, + { u"Changelog Id"_qs, QString::number(latestChangelogId) }, + { u" Items"_qs, loc.toString(itemChangelog.size()).rightJustified(10) }, + { u" Colors"_qs, loc.toString(colorChangelog.size()).rightJustified(10) }, { u"Item Types"_qs, loc.toString(itemTypes.size()).rightJustified(10) }, { u"Categories"_qs, loc.toString(categories.size()).rightJustified(10) }, + { u"Colors"_qs, loc.toString(colors.size()).rightJustified(10) }, + { u"LDraw Colors"_qs, loc.toString(ldrawExtraColors.size()).rightJustified(10) }, + { u"PCCs"_qs, loc.toString(pccs.size()).rightJustified(10) }, { u"Items"_qs, loc.toString(items.size()).rightJustified(10) }, }; +#if defined(QT_DEBUG) std::vector itemCount(itemTypes.size()); for (const auto &item : std::as_const(items)) ++itemCount[item.m_itemTypeIndex]; for (size_t i = 0; i < itemTypes.size(); ++i) { log.append({ u" "_qs + itemTypes.at(i).name(), - loc.toString(itemCount.at(i)).rightJustified(10) }); + loc.toString(itemCount.at(i)).rightJustified(10) }); } +#endif qsizetype leftSize = 0; for (const auto &logPair : std::as_const(log)) leftSize = std::max(leftSize, logPair.first.length()); @@ -418,6 +414,7 @@ void Database::read(const QString &fileName) m_itemChangelog = itemChangelog; m_colorChangelog = colorChangelog; m_pccs = pccs; + m_latestChangelogId = latestChangelogId; Color::s_colorImageCache.clear(); @@ -497,7 +494,17 @@ void Database::write(const QString &filename, Version version) const writeItemToDatabase(item, ds, version); check(cw.endChunk()); - if (version >= Version::V5) { + if (version >= Version::V9) { + check(cw.startChunk(ChunkId('C','H','G','L'), 2)); + ds << quint32(m_latestChangelogId) + << quint32(m_itemChangelog.size()) + << quint32(m_colorChangelog.size()); + for (const ItemChangeLogEntry &e : m_itemChangelog) + writeItemChangeLogToDatabase(e, ds, version); + for (const ColorChangeLogEntry &e : m_colorChangelog) + writeColorChangeLogToDatabase(e, ds, version); + check(cw.endChunk()); + } else if (version >= Version::V5) { check(cw.startChunk(ChunkId('I','C','H','G'), 1)); ds << quint32(m_itemChangelog.size()); for (const ItemChangeLogEntry &e : m_itemChangelog) @@ -719,21 +726,25 @@ void Database::writePCCToDatabase(const PartColorCode &pcc, QDataStream &dataStr void Database::readItemChangeLogFromDatabase(ItemChangeLogEntry &e, QDataStream &dataStream, Version) { - dataStream >> e.m_fromTypeAndId >> e.m_toTypeAndId; + dataStream >> e.m_id >> e.m_julianDay >> e.m_fromTypeAndId >> e.m_toTypeAndId; } -void Database::writeItemChangeLogToDatabase(const ItemChangeLogEntry &e, QDataStream &dataStream, Version) const +void Database::writeItemChangeLogToDatabase(const ItemChangeLogEntry &e, QDataStream &dataStream, Version v) const { + if (v >= Version::V9) + dataStream << e.m_id << e.m_julianDay; dataStream << e.m_fromTypeAndId << e.m_toTypeAndId; } void Database::readColorChangeLogFromDatabase(ColorChangeLogEntry &e, QDataStream &dataStream, Version) { - dataStream >> e.m_fromColorId >> e.m_toColorId; + dataStream >> e.m_id >> e.m_julianDay >> e.m_fromColorId >> e.m_toColorId; } -void Database::writeColorChangeLogToDatabase(const ColorChangeLogEntry &e, QDataStream &dataStream, Version) const +void Database::writeColorChangeLogToDatabase(const ColorChangeLogEntry &e, QDataStream &dataStream, Version v) const { + if (v >= Version::V9) + dataStream << e.m_id << e.m_julianDay; dataStream << e.m_fromColorId << e.m_toColorId; } diff --git a/src/bricklink/database.h b/src/bricklink/database.h index 78692b5c..7db17082 100755 --- a/src/bricklink/database.h +++ b/src/bricklink/database.h @@ -43,10 +43,11 @@ public: V6, // 2022.2.1 V7, // 2022.6.1 (not released) V8, // 2022.6.2 + V9, // 2023.3.1 OldestStillSupported = V4, - Latest = V8 + Latest = V9 }; void setUpdateInterval(int interval); @@ -100,6 +101,8 @@ private: std::vector m_colorChangelog; std::vector m_pccs; + uint m_latestChangelogId = 0; + friend class Core; friend class TextImport; diff --git a/src/bricklink/io.cpp b/src/bricklink/io.cpp index 4d81bfad..557c9848 100755 --- a/src/bricklink/io.cpp +++ b/src/bricklink/io.cpp @@ -103,7 +103,7 @@ QString IO::toBrickLinkXML(const LotList &lots) } -IO::ParseResult IO::fromBrickLinkXML(const QByteArray &data, Hint hint) +IO::ParseResult IO::fromBrickLinkXML(const QByteArray &data, Hint hint, const QDateTime &creationTime) { //stopwatch loadXMLWatch("Load XML"); @@ -213,7 +213,7 @@ IO::ParseResult IO::fromBrickLinkXML(const QByteArray &data, Hint hint) xml.skipCurrentElement(); } - switch (core()->resolveIncomplete(lot)) { + switch (core()->resolveIncomplete(lot, 0, creationTime)) { case Core::ResolveResult::Fail: pr.incInvalidLotCount(); break; case Core::ResolveResult::ChangeLog: pr.incFixedLotCount(); break; default: break; diff --git a/src/bricklink/io.h b/src/bricklink/io.h index ca4124cf..603bed88 100755 --- a/src/bricklink/io.h +++ b/src/bricklink/io.h @@ -63,7 +63,7 @@ enum class Hint { }; QString toBrickLinkXML(const LotList &lots); -ParseResult fromBrickLinkXML(const QByteArray &xml, Hint hint = Hint::Plain); +ParseResult fromBrickLinkXML(const QByteArray &xml, Hint hint, const QDateTime &creationTime = { }); ParseResult fromPartInventory(const Item *item, const Color *color = nullptr, int quantity = 1, Condition condition = Condition::New, Status extraParts = Status::Extra, diff --git a/src/bricklink/lot.cpp b/src/bricklink/lot.cpp index 0dc7cf8d..f8307801 100755 --- a/src/bricklink/lot.cpp +++ b/src/bricklink/lot.cpp @@ -116,10 +116,10 @@ Lot::~Lot() void Lot::save(QDataStream &ds) const { - ds << QByteArray("II") << qint32(4) - << QString::fromLatin1(itemId()) - << qint8(itemType() ? itemType()->id() : ItemType::InvalidId) - << uint(color() ? color()->id() : Color::InvalidId) + ds << QByteArray("II") << qint32(5) + << itemId() + << (itemType() ? itemType()->id() : ItemType::InvalidId) + << (color() ? color()->id() : Color::InvalidId) << qint8(m_status) << qint8(m_condition) << qint8(m_scondition) << qint8(m_retain ? 1 : 0) << qint8(m_stockroom) << m_lot_id << m_reserved << m_comments << m_remarks << m_quantity << m_bulk_quantity @@ -131,61 +131,63 @@ void Lot::save(QDataStream &ds) const << m_dateAdded << m_dateLastSold; } -Lot *Lot::restore(QDataStream &ds) +Lot *Lot::restore(QDataStream &ds, uint startChangelogAt) { std::unique_ptr lot; QByteArray tag; qint32 version; ds >> tag >> version; - if ((ds.status() != QDataStream::Ok) || (tag != "II") || (version < 2) || (version > 4)) + if ((ds.status() != QDataStream::Ok) || (tag != "II") || (version != 5)) return nullptr; - QString itemid; - uint colorid = 0; - qint8 itemtypeid = 0; + QByteArray itemId; + uint colorId = 0; + char itemTypeId = 0; - ds >> itemid >> itemtypeid >> colorid; + ds >> itemId >> itemTypeId >> colorId; if (ds.status() != QDataStream::Ok) return nullptr; - auto item = core()->item(itemtypeid, itemid.toLatin1()); - auto color = core()->color(colorid); - std::unique_ptr inc; + if (startChangelogAt) { + const QByteArray itemTypeAndId = itemTypeId + itemId; + QByteArray newId = core()->applyItemChangeLog(itemTypeAndId, startChangelogAt, { }); + if (newId != itemTypeAndId) { + itemTypeId = newId.at(0); + itemId = newId.mid(1); + } + colorId = core()->applyColorChangeLog(colorId, startChangelogAt, { }); + } + auto item = core()->item(itemTypeId, itemId); + auto color = core()->color(colorId); + + lot = std::make_unique(item, color); if (!item || !color) { - inc = std::make_unique(); + auto *inc = new Incomplete; if (!item) { - inc->m_item_id = itemid.toLatin1(); - inc->m_itemtype_id = itemtypeid; + inc->m_item_id = itemId; + inc->m_itemtype_id = itemTypeId; } if (!color) { - inc->m_color_id = colorid; - inc->m_color_name = QString::number(colorid); + inc->m_color_id = colorId; + inc->m_color_name = u"BL #"_qs + QString::number(colorId); } - - if (core()->applyChangeLog(item, color, inc.get())) - inc.reset(); + lot->setIncomplete(inc); } - lot = std::make_unique(item, color); - if (inc) - lot->setIncomplete(inc.release()); // alternate, cpart and altid are left out on purpose! qint8 status = 0, cond = 0, scond = 0, retain = 0, stockroom = 0; ds >> status >> cond >> scond >> retain >> stockroom - >> lot->m_lot_id >> lot->m_reserved >> lot->m_comments >> lot->m_remarks - >> lot->m_quantity >> lot->m_bulk_quantity - >> lot->m_tier_quantity[0] >> lot->m_tier_quantity[1] >> lot->m_tier_quantity[2] - >> lot->m_sale >> lot->m_price >> lot->m_cost - >> lot->m_tier_price[0] >> lot->m_tier_price[1] >> lot->m_tier_price[2] - >> lot->m_weight; - if (version >= 3) - ds >> lot->m_markerText >> lot->m_markerColor; - if (version >= 4) - ds >> lot->m_dateAdded >> lot->m_dateLastSold; + >> lot->m_lot_id >> lot->m_reserved >> lot->m_comments >> lot->m_remarks + >> lot->m_quantity >> lot->m_bulk_quantity + >> lot->m_tier_quantity[0] >> lot->m_tier_quantity[1] >> lot->m_tier_quantity[2] + >> lot->m_sale >> lot->m_price >> lot->m_cost + >> lot->m_tier_price[0] >> lot->m_tier_price[1] >> lot->m_tier_price[2] + >> lot->m_weight >> lot->m_markerText >> lot->m_markerColor + >> lot->m_dateAdded >> lot->m_dateLastSold; if (ds.status() != QDataStream::Ok) return nullptr; diff --git a/src/bricklink/lot.h b/src/bricklink/lot.h index 517626e3..919bf116 100755 --- a/src/bricklink/lot.h +++ b/src/bricklink/lot.h @@ -141,7 +141,7 @@ public: void setIncomplete(Incomplete *inc) { m_incomplete.reset(inc); } void save(QDataStream &ds) const; - static Lot *restore(QDataStream &ds); + static Lot *restore(QDataStream &ds, uint startChangelogAt); private: const Item * m_item; diff --git a/src/bricklink/textimport.cpp b/src/bricklink/textimport.cpp index 4fdf3bc0..9e364e0c 100644 --- a/src/bricklink/textimport.cpp +++ b/src/bricklink/textimport.cpp @@ -390,12 +390,14 @@ bool BrickLink::TextImport::readInventory(const Item *item, ImportInventoriesSte // if this itemid was involved in a changelog entry after the last time we downloaded // the inventory, we need to reload QByteArray itemTypeAndId = itemTypeId + itemId; - auto it = std::lower_bound(m_itemChangelog.cbegin(), m_itemChangelog.cend(), itemTypeAndId); - if ((it != m_itemChangelog.cend()) && (*it == itemTypeAndId) && (it->date() > fileDate)) { - throw Exception("Item id %1 changed on %2 (last download: %3)") - .arg(QString::fromLatin1(itemTypeAndId)) - .arg(it->date().toString(u"yyyy/MM/dd")) - .arg(fileDate.toString(u"yyyy/MM/dd")); + auto [lit, uit] = std::equal_range(m_itemChangelog.cbegin(), m_itemChangelog.cend(), itemTypeAndId); + for (auto it = lit; it != uit; ++it) { + if (it->date() > fileDate) { + throw Exception("Item id %1 changed on %2 (last download: %3)") + .arg(QString::fromLatin1(itemTypeAndId)) + .arg(it->date().toString(u"yyyy/MM/dd")) + .arg(fileDate.toString(u"yyyy/MM/dd")); + } } inventory.append(co); @@ -752,6 +754,8 @@ void BrickLink::TextImport::readChangeLog(const QString &path) if (!f.open(QFile::ReadOnly)) throw ParseException(&f, "could not open file"); + m_latestChangelogId = 0; + QTextStream ts(&f); int lineNumber = 0; while (!ts.atEnd()) { @@ -764,21 +768,23 @@ void BrickLink::TextImport::readChangeLog(const QString &path) if (strs.count() < 7) throw ParseException(&f, "expected at least 7 fields in line %1").arg(lineNumber); + uint id = strs.at(0).toUInt(); + QDate date = QDate::fromString(strs.at(1), u"M/d/yyyy"_qs); char c = ItemType::idFromFirstCharInString(strs.at(2)); - QDate date = QDate::fromString(strs.at(1), u"M/d/yyyy"_qs); // not stored in the database switch (c) { case 'I': // ItemId case 'T': // ItemType case 'M': { // ItemMerge - QString fromType = strs.at(3); - QString fromId = strs.at(4); - QString toType = strs.at(5); - QString toId = strs.at(6); + const QString &fromType = strs.at(3); + const QString &fromId = strs.at(4); + const QString &toType = strs.at(5); + const QString &toId = strs.at(6); if ((fromType.length() == 1) && (toType.length() == 1) && !fromId.isEmpty() && !toId.isEmpty()) { - m_itemChangelog.emplace_back((fromType + fromId).toLatin1(), - (toType + toId).toLatin1(), date); + m_itemChangelog.emplace_back(id, date, (fromType + fromId).toLatin1(), + (toType + toId).toLatin1()); + m_latestChangelogId = std::max(m_latestChangelogId, id); } break; } @@ -786,8 +792,10 @@ void BrickLink::TextImport::readChangeLog(const QString &path) bool fromOk = false, toOk = false; uint fromId = strs.at(3).toUInt(&fromOk); uint toId = strs.at(5).toUInt(&toOk); - if (fromOk && toOk) - m_colorChangelog.emplace_back(fromId, toId, date); + if (fromOk && toOk) { + m_colorChangelog.emplace_back(id, date, fromId, toId); + m_latestChangelogId = std::max(m_latestChangelogId, id); + } break; } case 'E': // CategoryName @@ -813,6 +821,7 @@ void BrickLink::TextImport::exportTo(Database *db) std::swap(db->m_pccs, m_pccs); std::swap(db->m_itemChangelog, m_itemChangelog); std::swap(db->m_colorChangelog, m_colorChangelog); + db->m_latestChangelogId = m_latestChangelogId; for (auto it = m_consists_of_hash.cbegin(); it != m_consists_of_hash.cend(); ++it) { Item &item = db->m_items[it.key()]; diff --git a/src/bricklink/textimport.h b/src/bricklink/textimport.h index 6522c9ad..55d6e4cf 100755 --- a/src/bricklink/textimport.h +++ b/src/bricklink/textimport.h @@ -72,6 +72,8 @@ private: QHash>>> m_appears_in_hash; // item-idx -> { vector < consists-of > } QHash> m_consists_of_hash; + + uint m_latestChangelogId = 0; }; } // namespace BrickLink diff --git a/src/common/document.cpp b/src/common/document.cpp index 4ac35a96..47bfabcd 100755 --- a/src/common/document.cpp +++ b/src/common/document.cpp @@ -2444,12 +2444,13 @@ void Document::autosave() const QByteArray ba; QDataStream ds(&ba, QIODevice::WriteOnly); ds << QByteArray(autosaveMagic) - << qint32(5) // version + << qint32(6) // version << title() << filePath() << m_model->currencyCode() << saveColumnsState() << model()->saveSortFilterState() + << BrickLink::core()->latestChangelogId() << qint32(lots.count()); for (auto lot : lots) { @@ -2488,24 +2489,25 @@ int Document::processAutosaves(AutosaveAction action) QByteArray columnState; QByteArray savedSortFilterState; qint32 count = 0; + uint startChangelogAt = 0; QDataStream ds(&f); ds >> magic >> version; - if ((magic != QByteArray(autosaveMagic)) || (version != 5)) + if ((magic != QByteArray(autosaveMagic)) || (version != 6)) continue; ds >> savedTitle >> savedFileName >> savedCurrencyCode >> columnState - >> savedSortFilterState >> count; + >> savedSortFilterState >> startChangelogAt >> count; BrickLink::IO::ParseResult pr; pr.setCurrencyCode(savedCurrencyCode); if (count > 0) { for (int i = 0; i < count; ++i) { - if (auto lot = Lot::restore(ds)) { + if (auto lot = Lot::restore(ds, startChangelogAt)) { bool hasBase = false; ds >> hasBase; if (hasBase) { - if (auto base = Lot::restore(ds)) { + if (auto base = Lot::restore(ds, startChangelogAt)) { pr.addToDifferenceModeBase(lot, *base); delete base; } else { diff --git a/src/common/documentio.cpp b/src/common/documentio.cpp index 9d043bb7..4dc7c3bf 100755 --- a/src/common/documentio.cpp +++ b/src/common/documentio.cpp @@ -141,7 +141,9 @@ QCoro::Task DocumentIO::importBrickLinkXML(QString fileName) QFile f(fn); if (f.open(QIODevice::ReadOnly)) { try { - auto result = BrickLink::IO::fromBrickLinkXML(f.readAll(), BrickLink::IO::Hint::PlainOrWanted); + auto result = BrickLink::IO::fromBrickLinkXML(f.readAll(), + BrickLink::IO::Hint::PlainOrWanted, + f.fileTime(QFile::FileModificationTime)); auto *document = new Document(new DocumentModel(std::move(result))); // Document owns the items now document->setTitle(tr("Import of %1").arg(QFileInfo(fn).fileName())); co_return document; @@ -423,13 +425,15 @@ bool DocumentIO::parseLDrawModelInternal(QFile *f, bool isStudio, const QString -Document *DocumentIO::parseBsxInventory(QIODevice *in) +Document *DocumentIO::parseBsxInventory(QFile *in) { //stopwatch loadBsxWatch("Load BSX"); Q_ASSERT(in); QXmlStreamReader xml(in); BsxContents bsx; + QDateTime creationTime = in->fileTime(QFile::FileModificationTime); + uint startAtChangelogId = 0; try { bsx.setCurrencyCode(u"$$$"_qs); // flag as legacy currency @@ -550,7 +554,7 @@ Document *DocumentIO::parseBsxInventory(QIODevice *in) } } - switch (BrickLink::core()->resolveIncomplete(lot)) { + switch (BrickLink::core()->resolveIncomplete(lot, startAtChangelogId, creationTime)) { case BrickLink::Core::ResolveResult::Fail: bsx.incInvalidLotCount(); break; case BrickLink::Core::ResolveResult::ChangeLog: bsx.incFixedLotCount(); break; default: break; @@ -575,7 +579,8 @@ Document *DocumentIO::parseBsxInventory(QIODevice *in) (*it)(&base, attr.value().toString()); } } - if (BrickLink::core()->resolveIncomplete(&base) == BrickLink::Core::ResolveResult::Fail) { + if (BrickLink::core()->resolveIncomplete(&base, startAtChangelogId, creationTime) + == BrickLink::Core::ResolveResult::Fail) { if (!base.item() && lot->item()) base.setItem(lot->item()); if (!base.color() && lot->color()) @@ -618,6 +623,7 @@ Document *DocumentIO::parseBsxInventory(QIODevice *in) if (xml.name() == u"Inventory") { foundInventory = true; bsx.setCurrencyCode(xml.attributes().value(u"Currency"_qs).toString()); + startAtChangelogId = xml.attributes().value(u"BrickLinkChangelogId"_qs).toUInt(); parseInventory(); } else if ((xml.name() == u"GuiState") && (xml.attributes().value(u"Application"_qs) == u"BrickStore") @@ -668,6 +674,7 @@ bool DocumentIO::createBsxInventory(QIODevice *out, const Document *doc) xml.writeStartElement(u"BrickStoreXML"_qs); xml.writeStartElement(u"Inventory"_qs); xml.writeAttribute(u"Currency"_qs, doc->model()->currencyCode()); + xml.writeAttribute(u"BrickLinkChangelogId"_qs, QString::number(BrickLink::core()->latestChangelogId())); const Lot *lot; const Lot *base; diff --git a/src/common/documentio.h b/src/common/documentio.h index 47cf3414..d97229a4 100755 --- a/src/common/documentio.h +++ b/src/common/documentio.h @@ -51,7 +51,7 @@ public: static QString exportBrickLinkUpdateClipboard(const DocumentModel *doc, const LotList &lots); - static Document *parseBsxInventory(QIODevice *in); + static Document *parseBsxInventory(QFile *in); static bool createBsxInventory(QIODevice *out, const Document *doc); private: diff --git a/src/common/documentmodel.cpp b/src/common/documentmodel.cpp index 48adc6f4..30ba592b 100755 --- a/src/common/documentmodel.cpp +++ b/src/common/documentmodel.cpp @@ -2758,9 +2758,9 @@ void DocumentLotsMimeData::setLots(const LotList &lots, const QString ¤cyC QString text; QDataStream ds(&data, QIODevice::WriteOnly); - ds << QByteArray("LOTS") << qint32(1); + ds << QByteArray("LOTS") << qint32(2); - ds << currencyCode << quint32(lots.count()); + ds << currencyCode << BrickLink::core()->latestChangelogId() << quint32(lots.count()); for (const Lot *lot : lots) { lot->save(ds); if (!text.isEmpty()) @@ -2779,22 +2779,23 @@ std::tuple DocumentLotsMimeData::lots(const QMimeData *md) if (md) { QByteArray data = md->data(s_mimetype); QDataStream ds(data); + uint startChangelogAt = 0; QByteArray tag; qint32 version; ds >> tag >> version; - if ((ds.status() != QDataStream::Ok) || (tag != "LOTS") || (version != 1)) + if ((ds.status() != QDataStream::Ok) || (tag != "LOTS") || (version != 2)) return { }; quint32 count = 0; - ds >> currencyCode >> count; + ds >> currencyCode >> startChangelogAt >> count; if ((ds.status() != QDataStream::Ok) || (currencyCode.size() != 3) || (count > 1000000)) return { }; lots.reserve(count); for (; count && !ds.atEnd(); count--) { - if (auto lot = Lot::restore(ds)) + if (auto lot = Lot::restore(ds, startChangelogAt)) lots << lot; } }