Allow matching similar csv files for batch import

src/ImportCsvDialog.cpp

@@ -229,13 +229,18 @@ void ImportCsvDialog::updatePreview()
 
 void ImportCsvDialog::checkInput()
 {
-    bool checkedItem = false;
-    for (int i = 0; i < ui->filePicker->count(); i++) {
-        if (ui->filePicker->item(i)->checkState() == Qt::Checked) checkedItem = true;
+    bool allowImporting = false;
+    if (ui->filePickerBlock->isVisible()) {
+        bool checkedItem = false;
+        for (int i = 0; i < ui->filePicker->count(); i++) {
+            if (ui->filePicker->item(i)->checkState() == Qt::Checked) checkedItem = true;
+        }
+        allowImporting = !ui->editName->text().isEmpty() && checkedItem;
+    } else {
+        allowImporting = !ui->editName->text().isEmpty();
     }
 
-    bool allowImporting = !ui->editName->text().isEmpty() && checkedItem;
-
+    ui->matchSimilar->setEnabled(ui->filePicker->currentItem() != nullptr);
     ui->editName->setEnabled(!ui->checkBoxSeparateTables->isChecked());
     ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(allowImporting);
 }
@@ -269,6 +274,38 @@ void ImportCsvDialog::updateSelection(bool selected)
     checkInput();
 }
 
+void ImportCsvDialog::matchSimilar()
+{
+    auto item = ui->filePicker->currentItem();
+    auto selectedHeader = fetchCsvHeader(item->data(Qt::DisplayRole).toString());
+
+    for (int i = 0; i < ui->filePicker->count(); i++)
+    {
+        auto item = ui->filePicker->item(i);
+        auto headers = fetchCsvHeader(item->data(Qt::DisplayRole).toString());
+        bool matchingHeader = false;
+
+        if (selectedHeader.count() == headers.count())
+        {
+            matchingHeader = std::equal(selectedHeader.begin(), selectedHeader.end(), headers.begin(),
+                                        [](const sqlb::FieldPtr& item1, const sqlb::FieldPtr& item2) -> bool {
+                                            return (item1->name() == item2->name());
+                                        });
+            if (matchingHeader) {
+                item->setCheckState(Qt::Checked);
+                item->setBackgroundColor(Qt::green);
+            }
+        }
+        else
+        {
+            item->setCheckState(Qt::Unchecked);
+            item->setBackground(Qt::white);
+        }
+    }
+
+    checkInput();
+}
+
 void ImportCsvDialog::importCsv(const QString& fileName)
 {
     QString tableName;
@@ -412,6 +449,55 @@ void ImportCsvDialog::importCsv(const QString& fileName)
     }
 }
 
+sqlb::FieldVector ImportCsvDialog::fetchCsvHeader(const QString& fileName)
+{
+    // Parse all csv data
+    QFile file(fileName);
+    file.open(QIODevice::ReadOnly);
+
+    CSVParser csv(ui->checkBoxTrimFields->isChecked(), currentSeparatorChar(), currentQuoteChar());
+    csv.setCSVProgress(new CSVImportProgress(file.size()));
+
+    QTextStream tstream(&file);
+    tstream.setCodec(currentEncoding().toUtf8());
+    csv.parse(tstream);
+    file.close();
+
+    if (csv.csv().size() == 0) return sqlb::FieldVector();
+
+    // Generate field names. These are either taken from the first CSV row or are generated in the format of "fieldXY" depending on the user input
+    sqlb::FieldVector fieldList;
+    CSVParser::TCSVResult::const_iterator itBegin = csv.csv().begin();
+    if(ui->checkboxHeader->isChecked())
+    {
+        ++itBegin;
+        for(QStringList::const_iterator it = csv.csv().at(0).begin();
+            it != csv.csv().at(0).end();
+            ++it)
+        {
+            // Remove invalid characters
+            QString thisfield = *it;
+            thisfield.replace("`", "");
+            thisfield.replace(" ", "");
+            thisfield.replace('"', "");
+            thisfield.replace("'","");
+            thisfield.replace(",","");
+            thisfield.replace(";","");
+
+            // Avoid empty field names
+            if(thisfield.isEmpty())
+                thisfield = QString("field%1").arg(std::distance(csv.csv().at(0).begin(), it) + 1);
+
+            fieldList.push_back(sqlb::FieldPtr(new sqlb::Field(thisfield, "")));
+        }
+    } else {
+        for(size_t i=0; i < csv.columns(); ++i)
+            fieldList.push_back(sqlb::FieldPtr(new sqlb::Field(QString("field%1").arg(i+1), "")));
+    }
+
+    return fieldList;
+}
+
 void ImportCsvDialog::setQuoteChar(const QChar& c)
 {
     QComboBox* combo = ui->comboQuote;

src/ImportCsvDialog.h

@@ -1,6 +1,7 @@
 #ifndef IMPORTCSVDIALOG_H
 #define IMPORTCSVDIALOG_H
 
+#include "sqlitetypes.h"
 #include <QDialog>
 
 class DBBrowserDB;
@@ -26,6 +27,7 @@ private slots:
     void selectFiles();
     void updateSelectedFilePreview(QListWidgetItem*);
     void updateSelection(bool);
+    void matchSimilar();
 
 private:
     Ui::ImportCsvDialog* ui;
@@ -35,6 +37,7 @@ private:
     QCompleter* encodingCompleter;
 
     void importCsv(const QString& f);
+    sqlb::FieldVector fetchCsvHeader(const QString& f);
 
     void setQuoteChar(const QChar& c);
     char currentQuoteChar() const;

src/ImportCsvDialog.ui

@@ -266,6 +266,18 @@
       <bool>true</bool>
      </property>
      <layout class="QHBoxLayout" name="filePickerLayout">
+      <property name="leftMargin">
+       <number>0</number>
+      </property>
+      <property name="topMargin">
+       <number>0</number>
+      </property>
+      <property name="rightMargin">
+       <number>0</number>
+      </property>
+      <property name="bottomMargin">
+       <number>0</number>
+      </property>
       <item>
        <widget class="QListWidget" name="filePicker"/>
       </item>
@@ -293,6 +305,16 @@
           </property>
          </widget>
         </item>
+        <item>
+         <widget class="QPushButton" name="matchSimilar">
+          <property name="enabled">
+           <bool>false</bool>
+          </property>
+          <property name="text">
+           <string>Match Similar</string>
+          </property>
+         </widget>
+        </item>
         <item>
          <spacer name="verticalSpacer">
           <property name="orientation">
@@ -548,10 +570,27 @@
     </hint>
    </hints>
   </connection>
+  <connection>
+   <sender>matchSimilar</sender>
+   <signal>pressed()</signal>
+   <receiver>ImportCsvDialog</receiver>
+   <slot>matchSimilar()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>682</x>
+     <y>279</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>368</x>
+     <y>244</y>
+    </hint>
+   </hints>
+  </connection>
  </connections>
  <slots>
   <slot>updatePreview()</slot>
   <slot>checkInput()</slot>
   <slot>updateSelection(bool)</slot>
+  <slot>matchSimilar()</slot>
  </slots>
 </ui>