Correctly implement ChangeLog handling for item and color ids

The old changelog handling did not take into account that the changelog
had to be applied from oldest to newest to correctly catch id swaps.
This was also due to the fact, that we didn't know where to start in
the changelog history. Adding the recursive resolve to this broken
setup ended in infinte loops on some item renames (e.g. 72206)

Newly saved BSX files now carry the latest BL changelog id to correctly
resolve any future renames. For old BSX files, as well as BrickLink XML
files, the file modification time will be used as a starting point in
the changelog. This is just a best guess though, because BrickLink's
temporal precision for changelog entries is a day, plus the BrickStore
database is typically a few days older than the file being saved.
It does work out quite well in practice though.

Closes: #686

BrickStoreXML.rnc

@@ -18,10 +18,14 @@ BrickStoreXML = element BrickStoreXML | BrickStockXML {
     Inventory, GuiState? 
 }
 
-# Currency is optional. The default currency is USD:
+# Currency is optional. The default currency is USD.
+#
+# BrickLinkChangelogId is optional, but quite helpful when automatically applying item and color
+# renames and merges (see also https://www.bricklink.com/btchglog.asp?viewHelp=Y)
 
 Inventory = element Inventory {
     attribute Currency { xsd:string { length="3" } }?,
+    attribute BrickLinkChangelogId { xsd:integer } }?,
     Item*
 }
 

src/bricklink/changelogentry.h

@@ -15,62 +15,87 @@ namespace BrickLink {
 class ColorChangeLogEntry
 {
 public:
-    uint fromColorId() const  { return m_fromColorId; }
-    uint toColorId() const  { return m_toColorId; }
+    uint id() const          { return m_id; }
+    QDate date() const       { return QDate::fromJulianDay(m_julianDay); }
+    uint fromColorId() const { return m_fromColorId; }
+    uint toColorId() const   { return m_toColorId; }
 
-    explicit ColorChangeLogEntry(uint fromColorId = Color::InvalidId, uint toColorId = Color::InvalidId,
-                                 QDate date = { })
-
-        : m_fromColorId(fromColorId)
+    explicit ColorChangeLogEntry() = default;
+    explicit ColorChangeLogEntry(uint id, const QDate &date, uint fromColorId, uint toColorId)
+        : m_id(id)
+        , m_julianDay(uint(date.toJulianDay()))
+        , m_fromColorId(fromColorId)
         , m_toColorId(toColorId)
-        , m_date(date)
     { }
 
-    constexpr std::weak_ordering operator<=>(uint colorId) const { return m_fromColorId <=> colorId; }
-    constexpr std::weak_ordering operator<=>(const ColorChangeLogEntry &other) const { return *this <=> other.m_fromColorId; }
-    constexpr bool operator==(uint colorId) const { return (*this <=> colorId == 0); }
-    constexpr bool operator==(const ColorChangeLogEntry &other) const { return *this == other.m_fromColorId; }
+    std::weak_ordering operator<=>(uint colorId) const;
+    std::weak_ordering operator<=>(const ColorChangeLogEntry &other) const;
 
 private:
-    uint m_fromColorId;
-    uint m_toColorId;
-    QDate m_date; // only used when rebuilding the DB
+    uint m_id = 0;
+    uint m_julianDay = 0;
+    uint m_fromColorId = Color::InvalidId;
+    uint m_toColorId = Color::InvalidId;
 
     friend class Core;
     friend class Database;
 };
 
 
+inline std::weak_ordering ColorChangeLogEntry::operator<=>(uint colorId) const
+{
+    return m_fromColorId <=> colorId;
+}
+
+inline std::weak_ordering ColorChangeLogEntry::operator<=>(const ColorChangeLogEntry &other) const
+{
+    auto cmp = (m_fromColorId <=> other.m_fromColorId);
+    return (cmp == 0) ? (m_id <=> other.m_id) : cmp;
+}
+
+
 class ItemChangeLogEntry
 {
 public:
-    char fromItemTypeId() const    { return m_fromTypeAndId.at(0); }
-    QByteArray fromItemId() const  { return m_fromTypeAndId.mid(1); }
+    uint id() const                     { return m_id; }
+    QDate date() const                  { return QDate::fromJulianDay(m_julianDay); }
+    char fromItemTypeId() const         { return m_fromTypeAndId.at(0); }
+    QByteArray fromItemId() const       { return m_fromTypeAndId.mid(1); }
+    char toItemTypeId() const           { return m_toTypeAndId.at(0); }
+    QByteArray toItemId() const         { return m_toTypeAndId.mid(1); }
+    QByteArray toItemTypeAndId() const  { return m_toTypeAndId; }
 
-    char toItemTypeId() const    { return m_toTypeAndId.at(0); }
-    QByteArray toItemId() const  { return m_toTypeAndId.mid(1); }
-
-    QDate date() const  { return m_date; }
-
-    explicit ItemChangeLogEntry(const QByteArray &fromTypeAndId = { }, const QByteArray &toTypeAndId = { },
-                                QDate date = { })
-        : m_fromTypeAndId(fromTypeAndId)
+    explicit ItemChangeLogEntry() = default;
+    explicit ItemChangeLogEntry(uint id, const QDate &date, const QByteArray &fromTypeAndId,
+                                const QByteArray &toTypeAndId)
+        : m_id(id)
+        , m_julianDay(uint(date.toJulianDay()))
+        , m_fromTypeAndId(fromTypeAndId)
         , m_toTypeAndId(toTypeAndId)
-        , m_date(date)
     { }
 
-    std::weak_ordering operator<=>(const QByteArray &typeAndId) const { return m_fromTypeAndId.compare(typeAndId) <=> 0; }
-    std::weak_ordering operator<=>(const ItemChangeLogEntry &other) const { return *this <=> other.m_fromTypeAndId; }
-    bool operator==(const QByteArray &typeAndId) const { return (*this <=> typeAndId == 0); }
-    bool operator==(const ItemChangeLogEntry &other) const { return *this == other.m_fromTypeAndId; }
+    std::weak_ordering operator<=>(const QByteArray &typeAndId) const;
+    std::weak_ordering operator<=>(const ItemChangeLogEntry &other) const;
 
 private:
+    uint m_id = 0;
+    uint m_julianDay = 0;
     QByteArray m_fromTypeAndId;
     QByteArray m_toTypeAndId;
-    QDate m_date; // only used when rebuilding the DB
 
     friend class Core;
     friend class Database;
 };
 
+inline std::weak_ordering ItemChangeLogEntry::operator<=>(const QByteArray &typeAndId) const
+{
+    return m_fromTypeAndId.compare(typeAndId) <=> 0;
+}
+
+inline std::weak_ordering ItemChangeLogEntry::operator<=>(const ItemChangeLogEntry &other) const
+{
+    auto cmp = (m_fromTypeAndId.compare(other.m_fromTypeAndId) <=> 0);
+    return (cmp == 0) ? (m_id <=> other.m_id) : cmp;
+}
+
 } // namespace BrickLink

src/bricklink/core.cpp

@@ -816,81 +816,125 @@ void Core::cancelTransfers()
         m_authenticatedTransfer->abortAllJobs();
 }
 
-bool Core::applyChangeLog(const Item *&item, const Color *&color, const Incomplete *inc)
+QByteArray Core::applyItemChangeLog(QByteArray itemTypeAndId, uint startAtChangelogId, const QDate &creationDate)
 {
-    if (!inc)
-        return false;
+    uint changelogId = startAtChangelogId;
+    bool foundChangelogEntry;
 
-    // there are a items that changed their name multiple times, so we have to loop (e.g. 3069bpb78)
+    constexpr bool dbg = false;
+    //bool dbg = itemTypeAndId.startsWith("P72206");
 
-    if (!item) {
-        QByteArray itemTypeAndId = inc->m_itemtype_id + inc->m_item_id;
-        if (!inc->m_itemtype_name.isEmpty())
-            itemTypeAndId[0] = inc->m_itemtype_name.at(0).toUpper().toLatin1();
+    do {
+        foundChangelogEntry = false;
+        auto [lit, uit] = std::equal_range(itemChangelog().cbegin(), itemChangelog().cend(), itemTypeAndId);
 
-        while (!item) {
-            auto it = std::lower_bound(itemChangelog().cbegin(), itemChangelog().cend(), itemTypeAndId);
-            if ((it == itemChangelog().cend()) || (*it != itemTypeAndId))
-                break;
-            item = core()->item(it->toItemTypeId(), it->toItemId());
-            if (!item)
-                itemTypeAndId = it->toItemTypeId() + it->toItemId();
+        if (dbg)
+            qDebug(LogResolver) << "SEARCH:" << itemTypeAndId << "has" << std::distance(lit, uit) << "changes";
+
+        // apply strictly from older to newer, starting at 'startAtChangelogId' or 'creationTime'
+        // in order to avoid infinite loops
+        for (auto it = lit; it != uit; ++it) {
+            if (dbg)
+                qDebug(LogResolver) << "CHANGE:" << it->toItemTypeAndId() << it->id() << changelogId << it->date() << creationDate;
+
+            if (changelogId) {
+                if (it->id() <= changelogId)
+                    continue;
+                changelogId = it->id();
+            } else {
+                if (it->date() <= creationDate)
+                    continue;
+                changelogId = it->id();
+            }
+            qCInfo(LogResolver).noquote() << "item:" << itemTypeAndId << "->" << it->toItemTypeAndId();
+
+            itemTypeAndId = it->toItemTypeAndId();
+            foundChangelogEntry = true;
+            break;
         }
-    }
-    if (!color) {
-        uint colorId = inc->m_color_id;
+    } while (foundChangelogEntry);
 
-        while (!color) {
-            auto it = std::lower_bound(colorChangelog().cbegin(), colorChangelog().cend(), colorId);
-            if ((it == colorChangelog().cend()) || (*it != colorId))
-                break;
-            color = core()->color(it->toColorId());
-            if (!color)
-                colorId = it->toColorId();
-        }
-    }
-
-    return (item && color);
+    return itemTypeAndId;
 }
 
-Core::ResolveResult Core::resolveIncomplete(Lot *lot)
+uint Core::applyColorChangeLog(uint colorId, uint startAtChangelogId, const QDate &creationDate)
 {
-    Incomplete *inc = lot->isIncomplete();
+    uint changelogId = startAtChangelogId;
+    bool foundChangelogEntry;
 
+    do {
+        foundChangelogEntry = false;
+        auto [lit, uit] = std::equal_range(colorChangelog().cbegin(), colorChangelog().cend(), colorId);
+
+        // apply strictly from older to newer, starting at 'startAtChangelogId' or 'creationTime'
+        // in order to avoid infinite loops
+        for (auto it = lit; it != uit; ++it) {
+            if (changelogId) {
+                if (it->id() <= changelogId)
+                    continue;
+                changelogId = it->id();
+            } else {
+                if (it->date() <= creationDate)
+                    continue;
+                changelogId = it->id();
+            }
+            qCInfo(LogResolver) << "color:" << colorId << "->" << it->toColorId();
+
+            colorId = it->toColorId();
+            foundChangelogEntry = true;
+            break;
+        }
+    } while (foundChangelogEntry);
+
+    return colorId;
+}
+
+Core::ResolveResult Core::resolveIncomplete(Lot *lot, uint startAtChangelogId, const QDateTime &creationTime)
+{
+    // How to apply changelog entries:
+    //  * if startAtChangelogId > 0, use it
+    //  * else if creationTime is valid, use that
+    //  * else do not apply the changelog at all
+
+    Incomplete *inc = lot->isIncomplete();
     if (!inc)
         return ResolveResult::Direct;
 
+    QByteArray itemTypeAndId = inc->m_itemtype_id + inc->m_item_id;
+    if (!inc->m_itemtype_name.isEmpty())
+        itemTypeAndId[0] = inc->m_itemtype_name.at(0).toUpper().toLatin1();
+    QByteArray resolvedItemTypeAndId = itemTypeAndId;
+
+    uint colorId = inc->m_color_id;
+    uint resolvedColorId = colorId;
+
+    bool tryToResolveItem = (itemTypeAndId.size() > 1) && itemTypeAndId.at(0);
+    bool tryToResolveColor = (colorId != Color::InvalidId);
+
+    if (startAtChangelogId || creationTime.isValid()) {
+        if (tryToResolveItem)
+            resolvedItemTypeAndId = applyItemChangeLog(itemTypeAndId, startAtChangelogId, creationTime.date());
+        if (tryToResolveColor)
+            resolvedColorId = applyColorChangeLog(colorId, startAtChangelogId, creationTime.date());
+    }
+
     const Item *item = nullptr;
     const Color *color = nullptr;
 
-    if ((inc->m_itemtype_id != ItemType::InvalidId) && !inc->m_item_id.isEmpty())
-        item = core()->item(inc->m_itemtype_id, inc->m_item_id);
-
-    if (inc->m_color_id != Color::InvalidId)
-        color = core()->color(inc->m_color_id);
+    if (tryToResolveItem)
+        item = core()->item(resolvedItemTypeAndId.at(0), resolvedItemTypeAndId.mid(1));
+    if (tryToResolveColor)
+        color = core()->color(resolvedColorId);
 
     if (item)
         lot->setItem(item);
     if (color)
         lot->setColor(color);
-    if (lot->item() && lot->color())
-        return ResolveResult::Direct;
 
-    bool ok = applyChangeLog(item, color, inc);
-
-    if (ok) {
-        qCInfo(LogResolver).nospace() << " [ OK ] "
-                                      << (inc->m_itemtype_id ? inc->m_itemtype_id : '?')
-                                      << '-' << inc->m_item_id.constData() << " (" << int(inc->m_color_id) << ')'
-                                      << " -> "
-                                      << item->itemTypeId()
-                                      << '-' << item->id().constData() << " (" << color->id() << ')';
-    } else {
-        qCWarning(LogResolver).nospace() << " [FAIL] "
-                                         << (inc->m_itemtype_id ? inc->m_itemtype_id : '?')
-                                         << '-' << inc->m_item_id.constData() << " (" << int(inc->m_color_id) << ')';
-
-        if (item) {
+    if (!lot->item() || !lot->color()) {
+        if (!lot->item()) {
+            qCWarning(LogResolver).noquote() << "item:" << resolvedItemTypeAndId << "[failed]";
+        } else {
             inc->m_item_id.clear();
             inc->m_item_name.clear();
             inc->m_itemtype_id = ItemType::InvalidId;
@@ -898,18 +942,21 @@ Core::ResolveResult Core::resolveIncomplete(Lot *lot)
             inc->m_category_id = Category::InvalidId;
             inc->m_category_name.clear();
         }
-        if (color) {
+        if (!lot->color()) {
+            qCWarning(LogResolver) << "color:" << resolvedColorId << "[failed]";
+        } else {
             inc->m_color_id = Color::InvalidId;
             inc->m_color_name.clear();
         }
-    }
-    if (item)
-        lot->setItem(item);
-    if (color)
-        lot->setColor(color);
+        return ResolveResult::Fail;
 
-    Q_ASSERT(ok == !lot->isIncomplete());
-    return ok ? ResolveResult::ChangeLog : ResolveResult::Fail;
+    } else if ((resolvedItemTypeAndId != itemTypeAndId) || (resolvedColorId != colorId)) {
+        lot->setIncomplete(nullptr);
+        return ResolveResult::ChangeLog;
+    } else {
+        lot->setIncomplete(nullptr);
+        return ResolveResult::Direct;
+    }
 }
 
 QSize Core::standardPictureSize() const

src/bricklink/core.h

@@ -85,6 +85,8 @@ public:
     inline const std::vector<ItemChangeLogEntry>  &itemChangelog() const  { return database()->m_itemChangelog; }
     inline const std::vector<ColorChangeLogEntry> &colorChangelog() const { return database()->m_colorChangelog; }
 
+    inline uint latestChangelogId() const { return database()->m_latestChangelogId; }
+
     const QImage noImage(const QSize &s) const;
 
     const Color *color(uint id) const;
@@ -99,14 +101,16 @@ public:
 
     QSize standardPictureSize() const;
 
-    bool applyChangeLog(const Item *&item, const Color *&color, const Incomplete *inc);
+    QByteArray applyItemChangeLog(QByteArray itemTypeAndId, uint startAtChangelogId,
+                                  const QDate &creationDate);
+    uint applyColorChangeLog(uint colorId, uint startAtChangelogId, const QDate &creationDate);
 
     QString countryIdFromName(const QString &name) const;
 
     static QString itemHtmlDescription(const Item *item, const Color *color, const QColor &highlight);
 
     enum class ResolveResult { Fail, Direct, ChangeLog };
-    ResolveResult resolveIncomplete(Lot *lot);
+    ResolveResult resolveIncomplete(Lot *lot, uint startAtChangelogId, const QDateTime &creationTime);
 
 public slots:
     void setUpdateIntervals(const QMap<QByteArray, int> &intervals);

src/bricklink/database.cpp

@@ -215,7 +215,7 @@ void Database::read(const QString &fileName)
         }
 
         bool gotColors = false, gotCategories = false, gotItemTypes = false, gotItems = false;
-        bool gotItemChangeLog = false, gotColorChangeLog = false, gotPccs = false;
+        bool gotChangeLog = false, gotPccs = false;
 
         auto check = [&ds, &f]() {
             if (ds.status() != QDataStream::Ok)
@@ -238,6 +238,7 @@ void Database::read(const QString &fileName)
         std::vector<ItemChangeLogEntry>  itemChangelog;
         std::vector<ColorChangeLogEntry> colorChangelog;
         std::vector<PartColorCode>       pccs;
+        uint                             latestChangelogId = 0;
 
         while (cr.startChunk()) {
             switch (cr.chunkId() | quint64(cr.chunkVersion()) << 32) {
@@ -315,32 +316,25 @@ void Database::read(const QString &fileName)
                 gotItems = true;
                 break;
             }
-            case ChunkId('I','C','H','G') | 1ULL << 32: {
-                quint32 clc = 0;
-                ds >> clc;
+            case ChunkId('C','H','G','L') | 2ULL << 32: {
+                quint32 clid = 0, clic = 0, clcc = 0;
+                ds >> clid >> clic >> clcc;
                 check();
-                sizeCheck(clc, 1'000'000);
+                sizeCheck(clic, 1'000'000);
+                sizeCheck(clcc, 1'000);
 
-                itemChangelog.resize(clc);
-                for (quint32 i = 0; i < clc; ++i) {
+                itemChangelog.resize(clic);
+                for (quint32 i = 0; i < clic; ++i) {
                     readItemChangeLogFromDatabase(itemChangelog[i], ds, Version::Latest);
                     check();
                 }
-                gotItemChangeLog = true;
-                break;
-            }
-            case ChunkId('C','C','H','G') | 1ULL << 32: {
-                quint32 clc = 0;
-                ds >> clc;
-                check();
-                sizeCheck(clc, 1'000);
-
-                colorChangelog.resize(clc);
-                for (quint32 i = 0; i < clc; ++i) {
+                colorChangelog.resize(clcc);
+                for (quint32 i = 0; i < clcc; ++i) {
                     readColorChangeLogFromDatabase(colorChangelog[i], ds, Version::Latest);
                     check();
                 }
-                gotColorChangeLog = true;
+                latestChangelogId = clid;
+                gotChangeLog = true;
                 break;
             }
             case ChunkId('P','C','C',' ') | 1ULL << 32: {
@@ -375,33 +369,35 @@ void Database::read(const QString &fileName)
 
         delete sw;
 
-        if (!gotColors || !gotCategories || !gotItemTypes || !gotItems || !gotItemChangeLog
-                || !gotColorChangeLog || !gotPccs) {
+        if (!gotColors || !gotCategories || !gotItemTypes || !gotItems || !gotChangeLog || !gotPccs) {
             throw Exception("not all required data chunks were found in the database (%1)")
                 .arg(f.fileName());
         }
 
-        if (true) {
+        {
             QString out = u"Loaded database from " + f.fileName();
             QLocale loc = QLocale(QLocale::Swedish); // space as number group separator
             QVector<std::pair<QString, QString>> log = {
                 { u"Generated at"_qs, generationDate.toString(Qt::RFC2822Date) },
-                { u"ChangeLog I"_qs,  loc.toString(itemChangelog.size()).rightJustified(10) },
-                { u"ChangeLog C"_qs,  loc.toString(colorChangelog.size()).rightJustified(10) },
-                { u"PCCs"_qs,         loc.toString(pccs.size()).rightJustified(10) },
-                { u"Colors"_qs,       loc.toString(colors.size()).rightJustified(10) },
-                { u"LDraw Colors"_qs, loc.toString(ldrawExtraColors.size()).rightJustified(10) },
+                { u"Changelog Id"_qs, QString::number(latestChangelogId) },
+                { u"  Items"_qs,      loc.toString(itemChangelog.size()).rightJustified(10) },
+                { u"  Colors"_qs,     loc.toString(colorChangelog.size()).rightJustified(10) },
                 { u"Item Types"_qs,   loc.toString(itemTypes.size()).rightJustified(10) },
                 { u"Categories"_qs,   loc.toString(categories.size()).rightJustified(10) },
+                { u"Colors"_qs,       loc.toString(colors.size()).rightJustified(10) },
+                { u"LDraw Colors"_qs, loc.toString(ldrawExtraColors.size()).rightJustified(10) },
+                { u"PCCs"_qs,         loc.toString(pccs.size()).rightJustified(10) },
                 { u"Items"_qs,        loc.toString(items.size()).rightJustified(10) },
             };
+#if defined(QT_DEBUG)
             std::vector<int> itemCount(itemTypes.size());
             for (const auto &item : std::as_const(items))
                 ++itemCount[item.m_itemTypeIndex];
             for (size_t i = 0; i < itemTypes.size(); ++i) {
                 log.append({ u"  "_qs + itemTypes.at(i).name(),
-                             loc.toString(itemCount.at(i)).rightJustified(10) });
+                            loc.toString(itemCount.at(i)).rightJustified(10) });
             }
+#endif
             qsizetype leftSize = 0;
             for (const auto &logPair : std::as_const(log))
                 leftSize = std::max(leftSize, logPair.first.length());
@@ -418,6 +414,7 @@ void Database::read(const QString &fileName)
         m_itemChangelog = itemChangelog;
         m_colorChangelog = colorChangelog;
         m_pccs = pccs;
+        m_latestChangelogId = latestChangelogId;
 
         Color::s_colorImageCache.clear();
 
@@ -497,7 +494,17 @@ void Database::write(const QString &filename, Version version) const
         writeItemToDatabase(item, ds, version);
     check(cw.endChunk());
 
-    if (version >= Version::V5) {
+    if (version >= Version::V9) {
+        check(cw.startChunk(ChunkId('C','H','G','L'), 2));
+        ds << quint32(m_latestChangelogId)
+           << quint32(m_itemChangelog.size())
+           << quint32(m_colorChangelog.size());
+        for (const ItemChangeLogEntry &e : m_itemChangelog)
+            writeItemChangeLogToDatabase(e, ds, version);
+        for (const ColorChangeLogEntry &e : m_colorChangelog)
+            writeColorChangeLogToDatabase(e, ds, version);
+        check(cw.endChunk());
+    } else if (version >= Version::V5) {
         check(cw.startChunk(ChunkId('I','C','H','G'), 1));
         ds << quint32(m_itemChangelog.size());
         for (const ItemChangeLogEntry &e : m_itemChangelog)
@@ -719,21 +726,25 @@ void Database::writePCCToDatabase(const PartColorCode &pcc, QDataStream &dataStr
 
 void Database::readItemChangeLogFromDatabase(ItemChangeLogEntry &e, QDataStream &dataStream, Version)
 {
-    dataStream >> e.m_fromTypeAndId >> e.m_toTypeAndId;
+    dataStream >> e.m_id >> e.m_julianDay >> e.m_fromTypeAndId >> e.m_toTypeAndId;
 }
 
-void Database::writeItemChangeLogToDatabase(const ItemChangeLogEntry &e, QDataStream &dataStream, Version) const
+void Database::writeItemChangeLogToDatabase(const ItemChangeLogEntry &e, QDataStream &dataStream, Version v) const
 {
+    if (v >= Version::V9)
+        dataStream << e.m_id << e.m_julianDay;
     dataStream << e.m_fromTypeAndId << e.m_toTypeAndId;
 }
 
 void Database::readColorChangeLogFromDatabase(ColorChangeLogEntry &e, QDataStream &dataStream, Version)
 {
-    dataStream >> e.m_fromColorId >> e.m_toColorId;
+    dataStream >> e.m_id >> e.m_julianDay >> e.m_fromColorId >> e.m_toColorId;
 }
 
-void Database::writeColorChangeLogToDatabase(const ColorChangeLogEntry &e, QDataStream &dataStream, Version) const
+void Database::writeColorChangeLogToDatabase(const ColorChangeLogEntry &e, QDataStream &dataStream, Version v) const
 {
+    if (v >= Version::V9)
+        dataStream << e.m_id << e.m_julianDay;
     dataStream << e.m_fromColorId << e.m_toColorId;
 }
 

src/bricklink/database.h

@@ -43,10 +43,11 @@ public:
         V6, // 2022.2.1
         V7, // 2022.6.1 (not released)
         V8, // 2022.6.2
+        V9, // 2023.3.1
 
         OldestStillSupported = V4,
 
-        Latest = V8
+        Latest = V9
     };
 
     void setUpdateInterval(int interval);
@@ -100,6 +101,8 @@ private:
     std::vector<ColorChangeLogEntry> m_colorChangelog;
     std::vector<PartColorCode>       m_pccs;
 
+    uint m_latestChangelogId = 0;
+
     friend class Core;
     friend class TextImport;
 

src/bricklink/io.cpp

@@ -103,7 +103,7 @@ QString IO::toBrickLinkXML(const LotList &lots)
 }
 
 
-IO::ParseResult IO::fromBrickLinkXML(const QByteArray &data, Hint hint)
+IO::ParseResult IO::fromBrickLinkXML(const QByteArray &data, Hint hint, const QDateTime &creationTime)
 {
     //stopwatch loadXMLWatch("Load XML");
 
@@ -213,7 +213,7 @@ IO::ParseResult IO::fromBrickLinkXML(const QByteArray &data, Hint hint)
                             xml.skipCurrentElement();
                     }
 
-                    switch (core()->resolveIncomplete(lot)) {
+                    switch (core()->resolveIncomplete(lot, 0, creationTime)) {
                     case Core::ResolveResult::Fail: pr.incInvalidLotCount(); break;
                     case Core::ResolveResult::ChangeLog: pr.incFixedLotCount(); break;
                     default: break;

src/bricklink/io.h

@@ -63,7 +63,7 @@ enum class Hint {
 };
 
 QString toBrickLinkXML(const LotList &lots);
-ParseResult fromBrickLinkXML(const QByteArray &xml, Hint hint = Hint::Plain);
+ParseResult fromBrickLinkXML(const QByteArray &xml, Hint hint, const QDateTime &creationTime = { });
 
 ParseResult fromPartInventory(const Item *item, const Color *color = nullptr, int quantity = 1,
                               Condition condition = Condition::New, Status extraParts = Status::Extra,

src/bricklink/lot.cpp

@@ -116,10 +116,10 @@ Lot::~Lot()
 
 void Lot::save(QDataStream &ds) const
 {
-    ds << QByteArray("II") << qint32(4)
-       << QString::fromLatin1(itemId())
-       << qint8(itemType() ? itemType()->id() : ItemType::InvalidId)
-       << uint(color() ? color()->id() : Color::InvalidId)
+    ds << QByteArray("II") << qint32(5)
+       << itemId()
+       << (itemType() ? itemType()->id() : ItemType::InvalidId)
+       << (color() ? color()->id() : Color::InvalidId)
        << qint8(m_status) << qint8(m_condition) << qint8(m_scondition) << qint8(m_retain ? 1 : 0)
        << qint8(m_stockroom) << m_lot_id << m_reserved << m_comments << m_remarks
        << m_quantity << m_bulk_quantity
@@ -131,61 +131,63 @@ void Lot::save(QDataStream &ds) const
        << m_dateAdded << m_dateLastSold;
 }
 
-Lot *Lot::restore(QDataStream &ds)
+Lot *Lot::restore(QDataStream &ds, uint startChangelogAt)
 {
     std::unique_ptr<Lot> lot;
 
     QByteArray tag;
     qint32 version;
     ds >> tag >> version;
-    if ((ds.status() != QDataStream::Ok) || (tag != "II") || (version < 2) || (version > 4))
+    if ((ds.status() != QDataStream::Ok) || (tag != "II") || (version != 5))
         return nullptr;
 
-    QString itemid;
-    uint colorid = 0;
-    qint8 itemtypeid = 0;
+    QByteArray itemId;
+    uint colorId = 0;
+    char itemTypeId = 0;
 
-    ds >> itemid >> itemtypeid >> colorid;
+    ds >> itemId >> itemTypeId >> colorId;
 
     if (ds.status() != QDataStream::Ok)
         return nullptr;
 
-    auto item = core()->item(itemtypeid, itemid.toLatin1());
-    auto color = core()->color(colorid);
-    std::unique_ptr<Incomplete> inc;
+    if (startChangelogAt) {
+        const QByteArray itemTypeAndId = itemTypeId + itemId;
+        QByteArray newId = core()->applyItemChangeLog(itemTypeAndId, startChangelogAt, { });
+        if (newId != itemTypeAndId) {
+            itemTypeId = newId.at(0);
+            itemId = newId.mid(1);
+        }
+        colorId = core()->applyColorChangeLog(colorId, startChangelogAt, { });
+    }
 
+    auto item = core()->item(itemTypeId, itemId);
+    auto color = core()->color(colorId);
+
+    lot = std::make_unique<Lot>(item, color);
     if (!item || !color) {
-        inc = std::make_unique<Incomplete>();
+        auto *inc = new Incomplete;
         if (!item) {
-            inc->m_item_id = itemid.toLatin1();
-            inc->m_itemtype_id = itemtypeid;
+            inc->m_item_id = itemId;
+            inc->m_itemtype_id = itemTypeId;
         }
         if (!color) {
-            inc->m_color_id = colorid;
-            inc->m_color_name = QString::number(colorid);
+            inc->m_color_id = colorId;
+            inc->m_color_name = u"BL #"_qs + QString::number(colorId);
         }
-
-        if (core()->applyChangeLog(item, color, inc.get()))
-            inc.reset();
+        lot->setIncomplete(inc);
     }
-    lot = std::make_unique<Lot>(item, color);
-    if (inc)
-        lot->setIncomplete(inc.release());
 
     // alternate, cpart and altid are left out on purpose!
 
     qint8 status = 0, cond = 0, scond = 0, retain = 0, stockroom = 0;
     ds >> status >> cond >> scond >> retain >> stockroom
-            >> lot->m_lot_id >> lot->m_reserved >> lot->m_comments >> lot->m_remarks
-            >> lot->m_quantity >> lot->m_bulk_quantity
-            >> lot->m_tier_quantity[0] >> lot->m_tier_quantity[1] >> lot->m_tier_quantity[2]
-            >> lot->m_sale >> lot->m_price >> lot->m_cost
-            >> lot->m_tier_price[0] >> lot->m_tier_price[1] >> lot->m_tier_price[2]
-            >> lot->m_weight;
-    if (version >= 3)
-        ds >> lot->m_markerText >> lot->m_markerColor;
-    if (version >= 4)
-        ds >> lot->m_dateAdded >> lot->m_dateLastSold;
+        >> lot->m_lot_id >> lot->m_reserved >> lot->m_comments >> lot->m_remarks
+        >> lot->m_quantity >> lot->m_bulk_quantity
+        >> lot->m_tier_quantity[0] >> lot->m_tier_quantity[1] >> lot->m_tier_quantity[2]
+        >> lot->m_sale >> lot->m_price >> lot->m_cost
+        >> lot->m_tier_price[0] >> lot->m_tier_price[1] >> lot->m_tier_price[2]
+        >> lot->m_weight >> lot->m_markerText >> lot->m_markerColor
+        >> lot->m_dateAdded >> lot->m_dateLastSold;
 
     if (ds.status() != QDataStream::Ok)
         return nullptr;

src/bricklink/lot.h

@@ -141,7 +141,7 @@ public:
     void setIncomplete(Incomplete *inc) { m_incomplete.reset(inc); }
 
     void save(QDataStream &ds) const;
-    static Lot *restore(QDataStream &ds);
+    static Lot *restore(QDataStream &ds, uint startChangelogAt);
 
 private:
     const Item * m_item;

src/bricklink/textimport.cpp

@@ -390,12 +390,14 @@ bool BrickLink::TextImport::readInventory(const Item *item, ImportInventoriesSte
             // if this itemid was involved in a changelog entry after the last time we downloaded
             // the inventory, we need to reload
             QByteArray itemTypeAndId = itemTypeId + itemId;
-            auto it = std::lower_bound(m_itemChangelog.cbegin(), m_itemChangelog.cend(), itemTypeAndId);
-            if ((it != m_itemChangelog.cend()) && (*it == itemTypeAndId) && (it->date() > fileDate)) {
-                throw Exception("Item id %1 changed on %2 (last download: %3)")
-                    .arg(QString::fromLatin1(itemTypeAndId))
-                    .arg(it->date().toString(u"yyyy/MM/dd"))
-                    .arg(fileDate.toString(u"yyyy/MM/dd"));
+            auto [lit, uit] = std::equal_range(m_itemChangelog.cbegin(), m_itemChangelog.cend(), itemTypeAndId);
+            for (auto it = lit; it != uit; ++it) {
+                if (it->date() > fileDate) {
+                    throw Exception("Item id %1 changed on %2 (last download: %3)")
+                        .arg(QString::fromLatin1(itemTypeAndId))
+                        .arg(it->date().toString(u"yyyy/MM/dd"))
+                        .arg(fileDate.toString(u"yyyy/MM/dd"));
+                }
             }
 
             inventory.append(co);
@@ -752,6 +754,8 @@ void BrickLink::TextImport::readChangeLog(const QString &path)
     if (!f.open(QFile::ReadOnly))
         throw ParseException(&f, "could not open file");
 
+    m_latestChangelogId = 0;
+
     QTextStream ts(&f);
     int lineNumber = 0;
     while (!ts.atEnd()) {
@@ -764,21 +768,23 @@ void BrickLink::TextImport::readChangeLog(const QString &path)
         if (strs.count() < 7)
             throw ParseException(&f, "expected at least 7 fields in line %1").arg(lineNumber);
 
+        uint id = strs.at(0).toUInt();
+        QDate date = QDate::fromString(strs.at(1), u"M/d/yyyy"_qs);
         char c = ItemType::idFromFirstCharInString(strs.at(2));
-        QDate date = QDate::fromString(strs.at(1), u"M/d/yyyy"_qs); // not stored in the database
 
         switch (c) {
         case 'I':   // ItemId
         case 'T':   // ItemType
         case 'M': { // ItemMerge
-            QString fromType = strs.at(3);
-            QString fromId = strs.at(4);
-            QString toType = strs.at(5);
-            QString toId = strs.at(6);
+            const QString &fromType = strs.at(3);
+            const QString &fromId = strs.at(4);
+            const QString &toType = strs.at(5);
+            const QString &toId = strs.at(6);
             if ((fromType.length() == 1) && (toType.length() == 1)
                     && !fromId.isEmpty() && !toId.isEmpty()) {
-                m_itemChangelog.emplace_back((fromType + fromId).toLatin1(),
-                                             (toType + toId).toLatin1(), date);
+                m_itemChangelog.emplace_back(id, date, (fromType + fromId).toLatin1(),
+                                             (toType + toId).toLatin1());
+                m_latestChangelogId = std::max(m_latestChangelogId, id);
             }
             break;
         }
@@ -786,8 +792,10 @@ void BrickLink::TextImport::readChangeLog(const QString &path)
             bool fromOk = false, toOk = false;
             uint fromId = strs.at(3).toUInt(&fromOk);
             uint toId = strs.at(5).toUInt(&toOk);
-            if (fromOk && toOk)
-                m_colorChangelog.emplace_back(fromId, toId, date);
+            if (fromOk && toOk) {
+                m_colorChangelog.emplace_back(id, date, fromId, toId);
+                m_latestChangelogId = std::max(m_latestChangelogId, id);
+            }
             break;
         }
         case 'E':   // CategoryName
@@ -813,6 +821,7 @@ void BrickLink::TextImport::exportTo(Database *db)
     std::swap(db->m_pccs, m_pccs);
     std::swap(db->m_itemChangelog, m_itemChangelog);
     std::swap(db->m_colorChangelog, m_colorChangelog);
+    db->m_latestChangelogId = m_latestChangelogId;
 
     for (auto it = m_consists_of_hash.cbegin(); it != m_consists_of_hash.cend(); ++it) {
         Item &item = db->m_items[it.key()];

src/bricklink/textimport.h

@@ -72,6 +72,8 @@ private:
     QHash<uint, QHash<uint, QVector<QPair<int, uint>>>> m_appears_in_hash;
     // item-idx -> { vector < consists-of > }
     QHash<uint, QVector<Item::ConsistsOf>>   m_consists_of_hash;
+
+    uint m_latestChangelogId = 0;
 };
 
 } // namespace BrickLink

src/common/document.cpp

@@ -2444,12 +2444,13 @@ void Document::autosave() const
     QByteArray ba;
     QDataStream ds(&ba, QIODevice::WriteOnly);
     ds << QByteArray(autosaveMagic)
-       << qint32(5) // version
+       << qint32(6) // version
        << title()
        << filePath()
        << m_model->currencyCode()
        << saveColumnsState()
        << model()->saveSortFilterState()
+       << BrickLink::core()->latestChangelogId()
        << qint32(lots.count());
 
     for (auto lot : lots) {
@@ -2488,24 +2489,25 @@ int Document::processAutosaves(AutosaveAction action)
             QByteArray columnState;
             QByteArray savedSortFilterState;
             qint32 count = 0;
+            uint startChangelogAt = 0;
 
             QDataStream ds(&f);
             ds >> magic >> version;
-            if ((magic != QByteArray(autosaveMagic)) || (version != 5))
+            if ((magic != QByteArray(autosaveMagic)) || (version != 6))
                 continue;
             ds >> savedTitle >> savedFileName >> savedCurrencyCode >> columnState
-                    >> savedSortFilterState >> count;
+                >> savedSortFilterState >> startChangelogAt >> count;
 
             BrickLink::IO::ParseResult pr;
             pr.setCurrencyCode(savedCurrencyCode);
 
             if (count > 0) {
                 for (int i = 0; i < count; ++i) {
-                    if (auto lot = Lot::restore(ds)) {
+                    if (auto lot = Lot::restore(ds, startChangelogAt)) {
                         bool hasBase = false;
                         ds >> hasBase;
                         if (hasBase) {
-                            if (auto base = Lot::restore(ds)) {
+                            if (auto base = Lot::restore(ds, startChangelogAt)) {
                                 pr.addToDifferenceModeBase(lot, *base);
                                 delete base;
                             } else {

src/common/documentio.cpp

@@ -141,7 +141,9 @@ QCoro::Task<Document *> DocumentIO::importBrickLinkXML(QString fileName)
     QFile f(fn);
     if (f.open(QIODevice::ReadOnly)) {
         try {
-            auto result = BrickLink::IO::fromBrickLinkXML(f.readAll(), BrickLink::IO::Hint::PlainOrWanted);
+            auto result = BrickLink::IO::fromBrickLinkXML(f.readAll(),
+                                                          BrickLink::IO::Hint::PlainOrWanted,
+                                                          f.fileTime(QFile::FileModificationTime));
             auto *document = new Document(new DocumentModel(std::move(result))); // Document owns the items now
             document->setTitle(tr("Import of %1").arg(QFileInfo(fn).fileName()));
             co_return document;
@@ -423,13 +425,15 @@ bool DocumentIO::parseLDrawModelInternal(QFile *f, bool isStudio, const QString
 
 
 
-Document *DocumentIO::parseBsxInventory(QIODevice *in)
+Document *DocumentIO::parseBsxInventory(QFile *in)
 {
     //stopwatch loadBsxWatch("Load BSX");
 
     Q_ASSERT(in);
     QXmlStreamReader xml(in);
     BsxContents bsx;
+    QDateTime creationTime = in->fileTime(QFile::FileModificationTime);
+    uint startAtChangelogId = 0;
 
     try {
         bsx.setCurrencyCode(u"$$$"_qs);  // flag as legacy currency
@@ -550,7 +554,7 @@ Document *DocumentIO::parseBsxInventory(QIODevice *in)
                     }
                 }
 
-                switch (BrickLink::core()->resolveIncomplete(lot)) {
+                switch (BrickLink::core()->resolveIncomplete(lot, startAtChangelogId, creationTime)) {
                 case BrickLink::Core::ResolveResult::Fail: bsx.incInvalidLotCount(); break;
                 case BrickLink::Core::ResolveResult::ChangeLog: bsx.incFixedLotCount(); break;
                 default: break;
@@ -575,7 +579,8 @@ Document *DocumentIO::parseBsxInventory(QIODevice *in)
                             (*it)(&base, attr.value().toString());
                         }
                     }
-                    if (BrickLink::core()->resolveIncomplete(&base) == BrickLink::Core::ResolveResult::Fail) {
+                    if (BrickLink::core()->resolveIncomplete(&base, startAtChangelogId, creationTime)
+                        == BrickLink::Core::ResolveResult::Fail) {
                         if (!base.item() && lot->item())
                             base.setItem(lot->item());
                         if (!base.color() && lot->color())
@@ -618,6 +623,7 @@ Document *DocumentIO::parseBsxInventory(QIODevice *in)
                     if (xml.name() == u"Inventory") {
                         foundInventory = true;
                         bsx.setCurrencyCode(xml.attributes().value(u"Currency"_qs).toString());
+                        startAtChangelogId = xml.attributes().value(u"BrickLinkChangelogId"_qs).toUInt();
                         parseInventory();
                     } else if ((xml.name() == u"GuiState")
                                 && (xml.attributes().value(u"Application"_qs) == u"BrickStore")
@@ -668,6 +674,7 @@ bool DocumentIO::createBsxInventory(QIODevice *out, const Document *doc)
     xml.writeStartElement(u"BrickStoreXML"_qs);
     xml.writeStartElement(u"Inventory"_qs);
     xml.writeAttribute(u"Currency"_qs, doc->model()->currencyCode());
+    xml.writeAttribute(u"BrickLinkChangelogId"_qs, QString::number(BrickLink::core()->latestChangelogId()));
 
     const Lot *lot;
     const Lot *base;

src/common/documentio.h

@@ -51,7 +51,7 @@ public:
     static QString exportBrickLinkUpdateClipboard(const DocumentModel *doc,
                                                   const LotList &lots);
 
-    static Document *parseBsxInventory(QIODevice *in);
+    static Document *parseBsxInventory(QFile *in);
     static bool createBsxInventory(QIODevice *out, const Document *doc);
 
 private:

src/common/documentmodel.cpp

@@ -2758,9 +2758,9 @@ void DocumentLotsMimeData::setLots(const LotList &lots, const QString &currencyC
     QString text;
 
     QDataStream ds(&data, QIODevice::WriteOnly);
-    ds << QByteArray("LOTS") << qint32(1);
+    ds << QByteArray("LOTS") << qint32(2);
 
-    ds << currencyCode << quint32(lots.count());
+    ds << currencyCode << BrickLink::core()->latestChangelogId() << quint32(lots.count());
     for (const Lot *lot : lots) {
         lot->save(ds);
         if (!text.isEmpty())
@@ -2779,22 +2779,23 @@ std::tuple<LotList, QString> DocumentLotsMimeData::lots(const QMimeData *md)
     if (md) {
         QByteArray data = md->data(s_mimetype);
         QDataStream ds(data);
+        uint startChangelogAt = 0;
 
         QByteArray tag;
         qint32 version;
         ds >> tag >> version;
-        if ((ds.status() != QDataStream::Ok) || (tag != "LOTS") || (version != 1))
+        if ((ds.status() != QDataStream::Ok) || (tag != "LOTS") || (version != 2))
             return { };
 
         quint32 count = 0;
-        ds >> currencyCode >> count;
+        ds >> currencyCode >> startChangelogAt >> count;
 
         if ((ds.status() != QDataStream::Ok) || (currencyCode.size() != 3) || (count > 1000000))
             return { };
 
         lots.reserve(count);
         for (; count && !ds.atEnd(); count--) {
-            if (auto lot = Lot::restore(ds))
+            if (auto lot = Lot::restore(ds, startChangelogAt))
                 lots << lot;
         }
     }