mirror/sqlitebrowser
1
0
Fork 0
mirror of https://github.com/sqlitebrowser/sqlitebrowser.git synced 2026-01-20 02:50:46 -06:00
Code Issues Packages Projects Releases Wiki Activity

Implement better default behaviour for CSV import

Browse Source 
This changes the default behaviour for the CSV import to follow a set of
rules which hopefully makes most people happy.

It also add an "Advanced" section to the settings bits of the dialog to
modify this new default behaviour.

See issue #1395.
This commit is contained in:
Martin Kleusberg
2018-05-25 21:06:16 +02:00
parent b647026051
commit e851c161d6
3 changed files with 150 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
src/ImportCsvDialog.cpp
View File

@@ -33,6 +33,9 @@ ImportCsvDialog::ImportCsvDialog(const QStringList &filenames, DBBrowserDB* db,
{
ui->setupUi(this);
// Hide "Advanced" section of the settings
toggleAdvancedSection(false);
// Get the actual file name out of the provided path and use it as the default table name for import
// For importing several files at once, the fields have to be the same so we can safely use the first
QFileInfo file(filenames.first());
@@ -53,7 +56,6 @@ ImportCsvDialog::ImportCsvDialog(const QStringList &filenames, DBBrowserDB* db,
ui->comboSeparator->blockSignals(true);
ui->comboQuote->blockSignals(true);
ui->comboEncoding->blockSignals(true);
ui->comboMissingValues->blockSignals(true);
ui->checkboxHeader->setChecked(Settings::getValue("importcsv", "firstrowheader").toBool());
ui->checkBoxTrimFields->setChecked(Settings::getValue("importcsv", "trimfields").toBool());
@@ -61,7 +63,6 @@ ImportCsvDialog::ImportCsvDialog(const QStringList &filenames, DBBrowserDB* db,
setSeparatorChar(Settings::getValue("importcsv", "separator").toInt());
setQuoteChar(Settings::getValue("importcsv", "quotecharacter").toInt());
setEncoding(Settings::getValue("importcsv", "encoding").toString());
setMissingValues(Settings::getValue("importcsv", "missingvalues").toString());
ui->checkboxHeader->blockSignals(false);
ui->checkBoxTrimFields->blockSignals(false);
@@ -69,7 +70,6 @@ ImportCsvDialog::ImportCsvDialog(const QStringList &filenames, DBBrowserDB* db,
ui->comboSeparator->blockSignals(false);
ui->comboQuote->blockSignals(false);
ui->comboEncoding->blockSignals(false);
ui->comboMissingValues->blockSignals(false);
// Prepare and show interface depending on how many files are selected
if (csvFilenames.length() > 1)
@@ -169,7 +169,6 @@ void ImportCsvDialog::accept()
Settings::setValue("importcsv", "trimfields", ui->checkBoxTrimFields->isChecked());
Settings::setValue("importcsv", "separatetables", ui->checkBoxSeparateTables->isChecked());
Settings::setValue("importcsv", "encoding", currentEncoding());
Settings::setValue("importcsv", "missingvalues", missingValues());
// Get all the selected files and start the import
if (ui->filePickerBlock->isVisible())
@@ -540,7 +539,9 @@ bool ImportCsvDialog::importCsv(const QString& fileName, const QString& name)
// Create table
QVector<QByteArray> nullValues;
bool missingValuesAsNull = missingValues() == "null";
std::vector<bool> failOnMissingFieldList;
bool ignoreDefaults = ui->checkIgnoreDefaults->isChecked();
bool failOnMissing = ui->checkFailOnMissing->isChecked();
if(!importToExistingTable)
{
if(!pdb->createTable(sqlb::ObjectIdentifier("main", tableName), fieldList))
@@ -562,14 +563,39 @@ bool ImportCsvDialog::importCsv(const QString& fileName, const QString& name)
{
for(const sqlb::FieldPtr& f : tbl->fields())
{
if(f->isInteger() && f->notnull()) // If this is an integer column but NULL isn't allowed, insert 0
nullValues << "0";
else if(f->isInteger() && !f->notnull()) // If this is an integer column and NULL is allowed, insert NULL
nullValues << QByteArray();
else if(missingValuesAsNull) // If we requested NULL values, do that
nullValues << QByteArray();
else // Otherwise, insert an empty string, like .import does
nullValues << "";
// For determining the value for empty fields we follow a set of rules
// Normally we don't have to fail the import when importing an empty field. This last value of the vector
// is changed to true later if we actually do want to fail the import for this field.
failOnMissingFieldList.push_back(false);
// If a field has a default value, that gets priority over everything else.
// Exception: if the user wants to ignore default values we never use them.
if(!ignoreDefaults && !f->defaultValue().isNull())
{
nullValues << f->defaultValue().toUtf8();
} else {
// If it has no default value, check if the field is NOT NULL
if(f->notnull())
{
// The field is NOT NULL
// If this is an integer column insert 0. Otherwise insert an empty string.
if(f->isInteger())
nullValues << "0";
else
nullValues << "";
// If the user wants to fail the import, remember this field
if(failOnMissing)
failOnMissingFieldList.back() = true;
} else {
// The field is not NOT NULL (stupid double negation here! NULL values are allowed in this case)
// Just insert a NULL value
nullValues << QByteArray();
}
}
}
}
}
@@ -606,12 +632,16 @@ bool ImportCsvDialog::importCsv(const QString& fileName, const QString& name)
// When importing into an existing table where we could find out something about its table definition
if(importToExistingTable && data.fields[i].data_length == 0 && static_cast<size_t>(nullValues.size()) > i)
{
// Do we want to fail when trying to import an empty value into this field? Then exit with an error.
if(failOnMissingFieldList.at(i))
return false;
// This is an empty value. We'll need to look up how to handle it depending on the field to be inserted into.
const QByteArray& val = nullValues.at(i);
if(!val.isNull()) // No need to bind NULL values here as that is the default bound value in SQLite
sqlite3_bind_text(stmt, i+1, val, val.size(), SQLITE_STATIC);
// When importing into a new table, use the missing values setting directly
} else if(!importToExistingTable && data.fields[i].data_length == 0 && missingValuesAsNull) {
} else if(!importToExistingTable && data.fields[i].data_length == 0) {
// No need to bind NULL values here as that is the default bound value in SQLite
} else {
// This is a non-empty value, or we want to insert the empty string. Just add it to the statement
@@ -742,18 +772,10 @@ QString ImportCsvDialog::currentEncoding() const
return ui->comboEncoding->currentText();
}
void ImportCsvDialog::setMissingValues(const QString& sValue)
void ImportCsvDialog::toggleAdvancedSection(bool show)
{
if(sValue == "null")
ui->comboMissingValues->setCurrentIndex(1);
else
ui->comboMissingValues->setCurrentIndex(0);
}
QString ImportCsvDialog::missingValues() const
{
if(ui->comboMissingValues->currentIndex() == 0)
return "emptystring";
else
return "null";
ui->labelFailOnMissing->setVisible(show);
ui->checkFailOnMissing->setVisible(show);
ui->labelIgnoreDefaults->setVisible(show);
ui->checkIgnoreDefaults->setVisible(show);
}

4
src/ImportCsvDialog.h
View File

@@ -31,6 +31,7 @@ private slots:
void updateSelectedFilePreview();
void updateSelection(bool);
void matchSimilar();
void toggleAdvancedSection(bool show);
private:
Ui::ImportCsvDialog* ui;
@@ -53,9 +54,6 @@ private:
void setEncoding(const QString& sEnc);
QString currentEncoding() const;
void setMissingValues(const QString& sValue);
QString missingValues() const;
};
#endif

162
src/ImportCsvDialog.ui
View File

@@ -7,13 +7,13 @@
<x>0</x>
<y>0</y>
<width>788</width>
<height>637</height>
<height>717</height>
</rect>
</property>
<property name="windowTitle">
<string>Import CSV file</string>
</property>
<layout class="QVBoxLayout" name="verticalLayout_3">
<layout class="QVBoxLayout" name="verticalLayout_2">
<item>
<layout class="QFormLayout" name="formLayout">
<property name="fieldGrowthPolicy">
@@ -22,7 +22,7 @@
<item row="0" column="0">
<widget class="QLabel" name="labelName">
<property name="text">
<string>&amp;Table name</string>
<string>Table na&amp;me</string>
</property>
<property name="buddy">
<cstring>editName</cstring>
@@ -49,7 +49,7 @@
</property>
</widget>
</item>
<item row="3" column="0">
<item row="2" column="0">
<widget class="QLabel" name="labelSeparator">
<property name="text">
<string>Field &amp;separator</string>
@@ -59,7 +59,7 @@
</property>
</widget>
</item>
<item row="3" column="1">
<item row="2" column="1">
<layout class="QHBoxLayout" name="horizontalLayout">
<item>
<widget class="QComboBox" name="comboSeparator">
@@ -112,7 +112,7 @@
</item>
</layout>
</item>
<item row="4" column="0">
<item row="3" column="0">
<widget class="QLabel" name="labelQuote">
<property name="text">
<string>&amp;Quote character</string>
@@ -122,7 +122,7 @@
</property>
</widget>
</item>
<item row="4" column="1">
<item row="3" column="1">
<layout class="QHBoxLayout" name="horizontalLayout_2">
<item>
<widget class="QComboBox" name="comboQuote">
@@ -170,7 +170,7 @@
</item>
</layout>
</item>
<item row="5" column="0">
<item row="4" column="0">
<widget class="QLabel" name="labelEncoding">
<property name="text">
<string>&amp;Encoding</string>
@@ -180,7 +180,7 @@
</property>
</widget>
</item>
<item row="5" column="1">
<item row="4" column="1">
<layout class="QHBoxLayout" name="horizontalLayout_3">
<item>
<widget class="QComboBox" name="comboEncoding">
@@ -224,52 +224,17 @@
</item>
</layout>
</item>
<item row="6" column="0">
<widget class="QLabel" name="labelMissingValues">
<property name="text">
<string>Missing values</string>
</property>
</widget>
</item>
<item row="6" column="1">
<layout class="QHBoxLayout" name="horizontalLayout_4">
<item>
<widget class="QComboBox" name="comboMissingValues">
<item>
<property name="text">
<string>Empty string</string>
</property>
</item>
<item>
<property name="text">
<string>NULL</string>
</property>
</item>
</widget>
</item>
<item>
<spacer name="horizontalSpacer_4">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>40</width>
<height>20</height>
</size>
</property>
</spacer>
</item>
</layout>
</item>
<item row="7" column="0">
<item row="5" column="0">
<widget class="QLabel" name="labelTrim">
<property name="text">
<string>Trim fields?</string>
</property>
<property name="buddy">
<cstring>checkBoxTrimFields</cstring>
</property>
</widget>
</item>
<item row="7" column="1">
<item row="5" column="1">
<widget class="QCheckBox" name="checkBoxTrimFields">
<property name="text">
<string/>
@@ -279,20 +244,71 @@
</property>
</widget>
</item>
<item row="8" column="0">
<item row="6" column="0">
<widget class="QLabel" name="separateTables">
<property name="text">
<string>Separate tables</string>
</property>
<property name="buddy">
<cstring>checkBoxSeparateTables</cstring>
</property>
</widget>
</item>
<item row="8" column="1">
<item row="6" column="1">
<widget class="QCheckBox" name="checkBoxSeparateTables">
<property name="text">
<string/>
</property>
</widget>
</item>
<item row="7" column="1">
<widget class="QPushButton" name="buttonAdvanced">
<property name="text">
<string>Advanced</string>
</property>
<property name="icon">
<iconset resource="icons/icons.qrc">
<normaloff>:/icons/down</normaloff>:/icons/down</iconset>
</property>
<property name="checkable">
<bool>true</bool>
</property>
</widget>
</item>
<item row="8" column="1">
<widget class="QCheckBox" name="checkIgnoreDefaults">
<property name="toolTip">
<string>When importing an empty value from the CSV file into an existing table with a default value for this column, that default value is inserted. Activate this option to insert an empty value instead.</string>
</property>
</widget>
</item>
<item row="8" column="0">
<widget class="QLabel" name="labelIgnoreDefaults">
<property name="text">
<string>Ignore default &amp;values</string>
</property>
<property name="buddy">
<cstring>checkIgnoreDefaults</cstring>
</property>
</widget>
</item>
<item row="9" column="1">
<widget class="QCheckBox" name="checkFailOnMissing">
<property name="toolTip">
<string>Activate this option to stop the import when trying to import an empty value into a NOT NULL column without a default value.</string>
</property>
</widget>
</item>
<item row="9" column="0">
<widget class="QLabel" name="labelFailOnMissing">
<property name="text">
<string>Fail on missing values </string>
</property>
<property name="buddy">
<cstring>checkFailOnMissing</cstring>
</property>
</widget>
</item>
</layout>
</item>
<item>
@@ -400,12 +416,17 @@
<tabstop>editCustomEncoding</tabstop>
<tabstop>checkBoxTrimFields</tabstop>
<tabstop>checkBoxSeparateTables</tabstop>
<tabstop>buttonAdvanced</tabstop>
<tabstop>checkIgnoreDefaults</tabstop>
<tabstop>checkFailOnMissing</tabstop>
<tabstop>filePicker</tabstop>
<tabstop>toggleSelected</tabstop>
<tabstop>matchSimilar</tabstop>
<tabstop>tablePreview</tabstop>
</tabstops>
<resources/>
<resources>
<include location="icons/icons.qrc"/>
</resources>
<connections>
<connection>
<sender>comboSeparator</sender>
@@ -510,8 +531,8 @@
<slot>updatePreview()</slot>
<hints>
<hint type="sourcelabel">
<x>266</x>
<y>165</y>
<x>263</x>
<y>183</y>
</hint>
<hint type="destinationlabel">
<x>572</x>
@@ -542,8 +563,8 @@
<slot>updateSelection(bool)</slot>
<hints>
<hint type="sourcelabel">
<x>674</x>
<y>258</y>
<x>780</x>
<y>337</y>
</hint>
<hint type="destinationlabel">
<x>368</x>
@@ -575,7 +596,7 @@
<hints>
<hint type="sourcelabel">
<x>272</x>
<y>478</y>
<y>677</y>
</hint>
<hint type="destinationlabel">
<x>157</x>
@@ -591,7 +612,7 @@
<hints>
<hint type="sourcelabel">
<x>340</x>
<y>478</y>
<y>677</y>
</hint>
<hint type="destinationlabel">
<x>286</x>
@@ -606,8 +627,8 @@
<slot>checkInput()</slot>
<hints>
<hint type="sourcelabel">
<x>176</x>
<y>216</y>
<x>194</x>
<y>236</y>
</hint>
<hint type="destinationlabel">
<x>368</x>
@@ -622,8 +643,8 @@
<slot>matchSimilar()</slot>
<hints>
<hint type="sourcelabel">
<x>682</x>
<y>279</y>
<x>780</x>
<y>378</y>
</hint>
<hint type="destinationlabel">
<x>368</x>
@@ -631,11 +652,28 @@
</hint>
</hints>
</connection>
<connection>
<sender>buttonAdvanced</sender>
<signal>toggled(bool)</signal>
<receiver>ImportCsvDialog</receiver>
<slot>toggleAdvancedSection(bool)</slot>
<hints>
<hint type="sourcelabel">
<x>214</x>
<y>259</y>
</hint>
<hint type="destinationlabel">
<x>393</x>
<y>358</y>
</hint>
</hints>
</connection>
</connections>
<slots>
<slot>updatePreview()</slot>
<slot>checkInput()</slot>
<slot>updateSelection(bool)</slot>
<slot>matchSimilar()</slot>
<slot>toggleAdvancedSection(bool)</slot>
</slots>
</ui>