initially select schema with most tables

- The wording of the dialog for analyzing a scheme has been made more
understandable.
  - Sorting columns in the result of a query in SQL Console did not work
    if a column did not have an explicit name. This bug was fixed.
  - The error message of the primary key checker tool is now more
readable.

.classpath

@@ -1,25 +1,25 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
-	<classpathentry kind="src" path="src/test"/>
-	<classpathentry kind="src" path="src/main/engine"/>
-	<classpathentry kind="src" path="src/main/gui"/>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER">
-		<attributes>
-			<attribute name="module" value="true"/>
-		</attributes>
-	</classpathentry>
-	<classpathentry kind="lib" path="lib/dbunit-2.4.4.jar"/>
-	<classpathentry kind="lib" path="lib/junit-4.4.jar"/>
-	<classpathentry kind="lib" path="lib/log4j.jar"/>
-	<classpathentry kind="lib" path="lib/prefuse.jar"/>
-	<classpathentry kind="lib" path="lib/sdoc-0.5.0-beta.jar"/>
-	<classpathentry kind="lib" path="lib/args4j.jar"/>
-	<classpathentry kind="lib" path="config"/>
-	<classpathentry kind="lib" path="lib/jsqlparser-1.3.jar"/>
-	<classpathentry kind="lib" path="lib/tablefilter-swing-5.3.1.jar"/>
-	<classpathentry kind="lib" path="lib/jaxb-api-2.3.0-b170201.1204.jar"/>
-	<classpathentry kind="lib" path="lib/jaxb-core-2.3.0-b170127.1453.jar"/>
-	<classpathentry kind="lib" path="lib/jaxb-impl-2.3.0-b170127.1453.jar"/>
-	<classpathentry kind="lib" path="lib/activation-1.0.2.jar"/>
-	<classpathentry kind="output" path="bin"/>
-</classpath>
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path="src/test"/>
+	<classpathentry kind="src" path="src/main/engine"/>
+	<classpathentry kind="src" path="src/main/gui"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER">
+		<attributes>
+			<attribute name="module" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="lib" path="lib/dbunit-2.4.4.jar"/>
+	<classpathentry kind="lib" path="lib/junit-4.4.jar"/>
+	<classpathentry kind="lib" path="lib/log4j.jar"/>
+	<classpathentry kind="lib" path="lib/prefuse.jar"/>
+	<classpathentry kind="lib" path="lib/sdoc-0.5.0-beta.jar"/>
+	<classpathentry kind="lib" path="lib/args4j.jar"/>
+	<classpathentry kind="lib" path="config"/>
+	<classpathentry kind="lib" path="lib/tablefilter-swing-5.3.1.jar"/>
+	<classpathentry kind="lib" path="lib/jaxb-api-2.3.0-b170201.1204.jar"/>
+	<classpathentry kind="lib" path="lib/jaxb-core-2.3.0-b170127.1453.jar"/>
+	<classpathentry kind="lib" path="lib/jaxb-impl-2.3.0-b170127.1453.jar"/>
+	<classpathentry kind="lib" path="lib/activation-1.0.2.jar"/>
+	<classpathentry kind="lib" path="lib/jsqlparser-3.2.jar"/>
+	<classpathentry kind="output" path="bin"/>
+</classpath>

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: Jailer
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.gitignore

@@ -0,0 +1,3 @@
+**/*.iml
+.idea/
+out/

.project

@@ -1,17 +1,17 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
-	<name>Jailer</name>
-	<comment></comment>
-	<projects>
-	</projects>
-	<buildSpec>
-		<buildCommand>
-			<name>org.eclipse.jdt.core.javabuilder</name>
-			<arguments>
-			</arguments>
-		</buildCommand>
-	</buildSpec>
-	<natures>
-		<nature>org.eclipse.jdt.core.javanature</nature>
-	</natures>
-</projectDescription>
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>Jailer</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>

Jailer.html

@@ -1,2 +0,0 @@
-<html><head><meta http-equiv="Refresh" content="0; URL=http://jailer.sourceforge.net/doc?src=doc">
-</head><body></body></html>

README.md

@@ -1,15 +1,16 @@
 # Jailer Database Tool
 
-Jailer is a tool for database subsetting and relational data browsing. 
+Jailer is a tool for database subsetting and relational data browsing.
 
  - The Subsetter exports consistent, referentially intact row-sets from relational databases,
    generates topologically sorted SQL-DML, DbUnit datasets and hierarchically structured XML.
- - The Data Browser allows bidirectional navigation through the database 
+ - The Data Browser allows bidirectional navigation through the database
    by following foreign-key-based or user-defined relationships.
 
 
 <img src="/docs/screenshot.png" width="850" />
 
+
 ## Features
 
  - Exports consistent and referentially intact row-sets from your productive database
@@ -20,38 +21,40 @@ Jailer is a tool for database subsetting and relational data browsing.
  - SQL Console with code completion, syntax highlighting and database metadata visualization.
  - A demo database is included with which you can get a first impression without any configuration effort.
 
+
 ## Prerequisites
 
- - Java JRE 7 (or above)
-   Important: due to HiDPI graphics support, Java JRE 11 (or above) is strongly recommended.
+ - Java JRE 8 (or above)
+    - Windows and Linux installation programs (\*.msi and \*.deb) already contain a JRE. If you use them, no further installation is needed.
+    - Important: due to HiDPI graphics support, Java JRE 11 (or above) is strongly recommended.
  - JDBC-driver for your RDBMS
+     - For most  popular database systems a driver is already included.
 
 ## News
 
- - 2020-01-01   The Jailer engine is published in Maven repository. https://mvnrepository.com/artifact/io.github.wisser/jailer-engine
- - 2019-02-01   The new "Model Migration Tool" allows you to easily find and edit the newly added associations if the data model has been extended after the last change to this extraction model.
- - 2018-04-26   The new feature "Analyze SQL" analyzes SQL statements and proposes association definitions. This allows to reverse-engineer the data model based on existing SQL queries.
- - 2018-03-06   SQL Console with code completion, syntax highlighting and database metadata visualization.
- - 2017-05-10   New API provides programmatic access to the data export and import functionality. http://jailer.sourceforge.net/api.html
- - 2017-03-30 	Improved filter management. Support for import-filters and literal-filters.
- - 2017-01-27 	Referential cycles can now be exported by deferring the insert of nullable foreign keys.
- - 2016-21-10 	Filter Templates allows you to define rules for assigning filters to columns.Filters on primary key columns will automatically be propagated to the corresponding foreign key columns.
- - 2015-12-04 	Support for oracle's ROWID pseudo-column.
- - 2016-09-08 	New "Export To" mode allows to export rows directly into a different schema in the same database.
- - 2015-12-04 	Support for oracle's ROWID pseudo-column.
- - 2015-10-23 	Release 5.0 introduces the ability to collect rows in a separate embedded database. This allows it to export data from read-only databases.
- - 2011-07-20 	Implemented the "Subset by Example" feature: Use the Data Browser to collect all the rows to be extracted and let Jailer create a model for that subset.
- - 2010-04-15 	A Data Browser has been introduced. Navigate bidirectionally through the database by following foreign-key-based or user-defined relationships.
- - 2008-12-23 	Jailer now supports the DbUnit flat XML dataset file format, thus allowing the users of the famous JUnit extension DbUnit to use the extracted data for unit testing.
- - 2007-12-05 	Version 2.0 comes with new graphical user interface.
- - 2007-06-05 	Tutorial for Jailer now available.
+ - 2021-02-04    Cycles in parent-child relationships will be detected and broken. Thus, such data can be exported by deferring the insertion of nullable foreign keys.
+ - 2020-01-01    The Jailer engine is published in Maven repository. https://mvnrepository.com/artifact/io.github.wisser/jailer-engine
+ - 2019-02-01    The new "Model Migration Tool" allows you to easily find and edit the newly added associations if the data model has been extended after the last change to this extraction model.
+ - 2018-04-26    The new feature "Analyze SQL" analyzes SQL statements and proposes association definitions. This allows to reverse-engineer the data model based on existing SQL queries.
+ - 2018-03-06    SQL Console with code completion, syntax highlighting and database metadata visualization.
+ - 2017-05-10    New API provides programmatic access to the data export and import functionality. http://jailer.sourceforge.net/api.html
+ - 2017-03-30    Improved filter management. Templates allows you to define rules for assigning filters to columns. Filters on primary key columns will automatically be propagated to the corresponding foreign key columns. http://jailer.sourceforge.net/filters.html
+ - 2015-12-04    Data can now also be exported directly to a schema of the same database. This ensures optimal performance.
+ - 2015-10-23    Rows can alternatively be collected in a separate embedded database. This allows exporting data from read-only databases.
+ - 2014-07-20    Implemented the "Subset by Example" feature: Use the Data Browser to collect all the rows to be extracted and let Jailer create a model for that subset. http://jailer.sourceforge.net/subset-by-example.html
+ - 2014-04-15    A Data Browser has been introduced. Navigate bidirectionally through the database by following foreign-key-based or user-defined relationships.
+
 
 
 ## Installation
 
-Use the installer "Jailer-Install-n.n.n.exe" or unzip the file "jailer_n.n.n.zip".
+If you do not want to install Java yourself, use the installation file "Jailer-n.n.n-with-java-JRE.msi" (for Windows) or "jailer-database-tools_10.2.2-x64-with-java-JRE.deb" (for Linux).
+
+Otherwise use the installer "Jailer-Install-n.n.n.exe" or unzip the file "jailer_n.n.n.zip".
 See also <a href="http://jailer.sourceforge.net/faq.html#multiuser">http://jailer.sourceforge.net/faq.html#multiuser</a>
 
+To start the tool from the unpacked zip:
+
  - Database Subsetter
     - On windows platform execute "Jailer.exe". You can also start "jailerGUI.bat".
     - On Unix/Linux platform execute the script "jailerGUI.sh" or use "java -jar jailer.jar"
@@ -60,6 +63,7 @@ See also <a href="http://jailer.sourceforge.net/faq.html#multiuser">http://jaile
     - On windows platform execute "jailerDataBrowser.exe", or "jailerDataBrowser.bat"
     - On Unix/Linux platform execute the script "jailerDataBrowser.sh"
 
+
 ## Building
 
 Clone the git repository:
@@ -71,7 +75,39 @@ To build the tool you can just use ant: ( https://ant.apache.org )
 * `cd Jailer`
 * `ant`
 
+
 ## Contact
  - Home:    http://jailer.sourceforge.net/ or https://github.com/Wisser/Jailer
  - Forum:   https://sourceforge.net/p/jailer/discussion/
  - Support: rwisser@users.sourceforge.net
+
+
+## Contributors
+
+### Code Contributors
+
+This project exists thanks to all the people who contribute.
+<a href="https://github.com/Wisser/Jailer/graphs/contributors"><img src="https://opencollective.com/Jailer/contributors.svg?width=890&button=false" /></a>
+
+### Financial Contributors
+
+Become a financial contributor and help us sustain our community. [[Contribute](https://opencollective.com/Jailer/contribute)]
+
+#### Individuals
+
+<a href="https://opencollective.com/Jailer"><img src="https://opencollective.com/Jailer/individuals.svg?width=890"></a>
+
+#### Organizations
+
+Support this project with your organization. Your logo will show up here with a link to your website. [[Contribute](https://opencollective.com/Jailer/contribute)]
+
+<a href="https://opencollective.com/Jailer/organization/0/website"><img src="https://opencollective.com/Jailer/organization/0/avatar.svg"></a>
+<a href="https://opencollective.com/Jailer/organization/1/website"><img src="https://opencollective.com/Jailer/organization/1/avatar.svg"></a>
+<a href="https://opencollective.com/Jailer/organization/2/website"><img src="https://opencollective.com/Jailer/organization/2/avatar.svg"></a>
+<a href="https://opencollective.com/Jailer/organization/3/website"><img src="https://opencollective.com/Jailer/organization/3/avatar.svg"></a>
+<a href="https://opencollective.com/Jailer/organization/4/website"><img src="https://opencollective.com/Jailer/organization/4/avatar.svg"></a>
+<a href="https://opencollective.com/Jailer/organization/5/website"><img src="https://opencollective.com/Jailer/organization/5/avatar.svg"></a>
+<a href="https://opencollective.com/Jailer/organization/6/website"><img src="https://opencollective.com/Jailer/organization/6/avatar.svg"></a>
+<a href="https://opencollective.com/Jailer/organization/7/website"><img src="https://opencollective.com/Jailer/organization/7/avatar.svg"></a>
+<a href="https://opencollective.com/Jailer/organization/8/website"><img src="https://opencollective.com/Jailer/organization/8/avatar.svg"></a>
+<a href="https://opencollective.com/Jailer/organization/9/website"><img src="https://opencollective.com/Jailer/organization/9/avatar.svg"></a>

admin/buildjailer.sh

@@ -3,4 +3,4 @@ export JAVA_HOME
 
 cd git/Jailer/admin
 dos2unix *.sh
-sh jbuild.sh $1
+sh jbuild.sh $1 > buildprotokoll.txt

admin/databrowserlauncher-linux.properties

@@ -0,0 +1,4 @@
+icon=jailer.png
+main-jar=jailer.jar
+main-class=net.sf.jailer.ui.databrowser.DataBrowser
+arguments=JailerDataBrowser -jpack

admin/databrowserlauncher.properties

@@ -1,2 +1,4 @@
 icon=jailer.ico
+main-jar=jailer.jar
 main-class=net.sf.jailer.ui.databrowser.DataBrowser
+arguments=JailerDataBrowser

admin/dbeauty.nsi

@@ -1,102 +0,0 @@
-!define Version "%VERSION%"
-
-; The name of the installer
-Name "DBeauty ${Version}"
-
-; The file to write
-OutFile "DBeauty-Install-${Version}.exe"
-
-; The default installation directory
-InstallDir $PROGRAMFILES\DBeauty
-
-; Registry key to check for directory (so if you install again, it will 
-; overwrite the old one automatically)
-InstallDirRegKey HKLM "Software\DBeauty" "Install_Dir"
-
-; Request application privileges for Windows Vista
-RequestExecutionLevel admin
-
-;--------------------------------
-
-; Pages
-
-Page components
-Page directory
-Page instfiles
-
-UninstPage uninstConfirm
-UninstPage instfiles
-
-Icon "..\src\main\gui\net\sf\jailer\ui\resource\Jailer.ico"
-
-;--------------------------------
-
-; The stuff to install
-Section "DBeauty"
-
-  SectionIn RO
-  
-  ; Set output path to the installation directory.
-  SetOutPath $INSTDIR
-  
-  ; Put file there
-
-  File /r "C:\tmp\dbeauty\*.*"
-  
-  FileOpen $4 ".singleuser" w
-  FileWrite $4 "1"
-  FileClose $4
-
-  ; Write the installation path into the registry
-  WriteRegStr HKLM SOFTWARE\DBeauty "Install_Dir" "$INSTDIR"
-  
-  ; Write the uninstall keys for Windows
-  WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\DBeauty" "DisplayName" "DBeauty"
-  WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\DBeauty" "UninstallString" '"$INSTDIR\uninstall.exe"'
-  WriteRegDWORD HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\DBeauty" "NoModify" 1
-  WriteRegDWORD HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\DBeauty" "NoRepair" 1
-  WriteUninstaller "uninstall.exe"
-  
-SectionEnd
-
-; Optional section (can be disabled by the user)
-Section "Start Menu Shortcuts"
-
-  CreateDirectory "$SMPROGRAMS\DBeauty"
-  Delete "$SMPROGRAMS\DBeauty\*.*"
-  CreateShortcut "$SMPROGRAMS\DBeauty\DBeauty ${Version}  .lnk" "$INSTDIR\dbeauty.exe" 
-;  CreateShortcut "$SMPROGRAMS\DBeauty\Uninstall.lnk" "$INSTDIR\uninstall.exe" 
-  
-SectionEnd
-
-Section "Desktop Shortcuts"
-
-  CreateShortcut "$DESKTOP\DBeauty.lnk" "$INSTDIR\dbeauty.exe"  
-  
-SectionEnd
-
-;--------------------------------
-
-; Uninstaller
-
-Section "Uninstall"
-  
-  ; Remove registry keys
-  DeleteRegKey HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\DBeauty"
-  DeleteRegKey HKLM SOFTWARE\DBeauty
-
-  ; Remove files and uninstaller
-;  Delete $INSTDIR\example2.nsi
-;  Delete $INSTDIR\uninstall.exe
-
-  ; Remove shortcuts, if any
-  Delete "$SMPROGRAMS\DBeauty\*.*"
-  Delete "$INSTDIR\*.*"
-
-  Delete "$DESKTOP\DBeauty ${Version}.lnk"
-
-  ; Remove directories used
-  RMDir "$SMPROGRAMS\DBeauty"
-  RMDir /r "$INSTDIR"
-
-SectionEnd

admin/j14pack.bat

@@ -1,66 +1,65 @@
-set PATH=d:\jdk-14\bin;%PATH%
+set version=%1
 
-rm -r c:\tmp\_
-rm -r c:\tmp\myjre
+set PATH=C:\Program Files\Java\jdk-15\bin;%PATH%
+
+del /S /Q c:\tmp\_
+del /S /Q c:\tmp\jre%version%
 mkdir c:\tmp\_
 cd ..
 
-cp -r bookmark c:\tmp\_
-cp -r build.xml c:\tmp\_
-cp -r config c:\tmp\_
-cp -r datamodel c:\tmp\_
-cp -r dbeauty.bat c:\tmp\_
-cp -r dbeauty.exe c:\tmp\_
-cp -r dbeauty.sh c:\tmp\_
-cp -r demo-sakila-1.4.mv.db c:\tmp\_
-cp -r demo-scott-1.4.mv.db c:\tmp\_
-cp -r demo-scott-subset-1.4.mv.db c:\tmp\_
-cp -r driverlist.csv c:\tmp\_
-cp -r extractionmodel c:\tmp\_
-cp -r jailer.bat c:\tmp\_
-cp -r Jailer.exe c:\tmp\_
-cp -r jailer.jar c:\tmp\_
-cp -r jailer.sh c:\tmp\_
-cp -r jailer.xml c:\tmp\_
-cp -r jailerDataBrowser.bat c:\tmp\_
-cp -r JailerDataBrowser.exe c:\tmp\_
-cp -r jailerDataBrowser.sh c:\tmp\_
-cp -r jailerGUI.bat c:\tmp\_
-cp -r jailerGUI.sh c:\tmp\_
-cp -r layout c:\tmp\_
-cp -r lib c:\tmp\_
-cp -r license-prefuse.txt c:\tmp\_
-cp -r license.txt c:\tmp\_
-cp -r manifest.mf c:\tmp\_
-cp -r README.md c:\tmp\_
-cp -r releasenotes.txt c:\tmp\_
-cp -r render c:\tmp\_
-cp admin\jailer.ico c:\tmp\_
-cp admin\databrowserlauncher.properties c:\tmp\_
+xcopy /S /E maven-artifacts c:\tmp\_\maven-artifacts\
+xcopy /S /E bookmark c:\tmp\_\bookmark\
+xcopy /S /E config c:\tmp\_\config\
+xcopy /S /E datamodel c:\tmp\_\datamodel\
+xcopy /S /E template c:\tmp\_\template\
+xcopy demo-sakila-1.4.mv.db c:\tmp\_
+xcopy demo-scott-1.4.mv.db c:\tmp\_
+xcopy demo-scott-subset-1.4.mv.db c:\tmp\_
+xcopy driverlist.csv c:\tmp\_
+xcopy /S /E extractionmodel c:\tmp\_\extractionmodel\
+xcopy jailer.bat c:\tmp\_
+xcopy Jailer.exe c:\tmp\_
+xcopy jailer.jar c:\tmp\_
+xcopy jailer.sh c:\tmp\_
+xcopy jailer.xml c:\tmp\_
+xcopy jailerDataBrowser.bat c:\tmp\_
+xcopy JailerDataBrowser.exe c:\tmp\_
+xcopy jailerDataBrowser.sh c:\tmp\_
+xcopy jailerGUI.bat c:\tmp\_
+xcopy jailerGUI.sh c:\tmp\_
+xcopy /S /E layout c:\tmp\_\layout\
+xcopy /S /E lib c:\tmp\_\lib\
+xcopy license-prefuse.txt c:\tmp\_
+xcopy license.txt c:\tmp\_
+xcopy manifest.mf c:\tmp\_
+xcopy README.md c:\tmp\_
+xcopy releasenotes.txt c:\tmp\_
+xcopy /S /E render c:\tmp\_\render\
+xcopy admin\jailer.ico c:\tmp\_
+xcopy admin\databrowserlauncher.properties c:\tmp\_
 
-cp -r c:\tmp\_\lib c:\tmp\_\jdbc_lib
-rm c:\tmp\_\lib\*
-rm c:\tmp\_\*.bat
-rm c:\tmp\_\*.exe
-rm c:\tmp\_\*.sh
+xcopy /S /E c:\tmp\_\lib c:\tmp\_\jdbc_lib\
+del /Q c:\tmp\_\lib\*
+del /Q c:\tmp\_\*.sh
 
-mv c:\tmp\_\jdbc_lib\activation-1.0.2.jar c:\tmp\_\lib\activation-1.0.2.jar
-mv c:\tmp\_\jdbc_lib\args4j.jar c:\tmp\_\lib\args4j.jar
-mv c:\tmp\_\jdbc_lib\jaxb-api-2.3.0-b170201.1204.jar c:\tmp\_\lib\jaxb-api-2.3.0-b170201.1204.jar
-mv c:\tmp\_\jdbc_lib\jaxb-core-2.3.0-b170127.1453.jar c:\tmp\_\lib\jaxb-core-2.3.0-b170127.1453.jar
-mv c:\tmp\_\jdbc_lib\jaxb-impl-2.3.0-b170127.1453.jar c:\tmp\_\lib\jaxb-impl-2.3.0-b170127.1453.jar
-mv c:\tmp\_\jdbc_lib\jsqlparser-1.3.jar c:\tmp\_\lib\jsqlparser-1.3.jar
-mv c:\tmp\_\jdbc_lib\log4j.jar c:\tmp\_\lib\log4j.jar
-mv c:\tmp\_\jdbc_lib\prefuse.jar c:\tmp\_\lib\prefuse.jar
-mv c:\tmp\_\jdbc_lib\sdoc-0.5.0-beta.jar c:\tmp\_\lib\sdoc-0.5.0-beta.jar
-mv c:\tmp\_\jdbc_lib\tablefilter-swing-5.3.1.jar c:\tmp\_\lib\tablefilter-swing-5.3.1.jar
+move c:\tmp\_\jdbc_lib\activation-1.0.2.jar c:\tmp\_\lib\activation-1.0.2.jar
+move c:\tmp\_\jdbc_lib\args4j.jar c:\tmp\_\lib\args4j.jar
+move c:\tmp\_\jdbc_lib\jaxb-api-2.3.0-b170201.1204.jar c:\tmp\_\lib\jaxb-api-2.3.0-b170201.1204.jar
+move c:\tmp\_\jdbc_lib\jaxb-core-2.3.0-b170127.1453.jar c:\tmp\_\lib\jaxb-core-2.3.0-b170127.1453.jar
+move c:\tmp\_\jdbc_lib\jaxb-impl-2.3.0-b170127.1453.jar c:\tmp\_\lib\jaxb-impl-2.3.0-b170127.1453.jar
+move c:\tmp\_\jdbc_lib\jsqlparser-3.2.jar c:\tmp\_\lib\jsqlparser-3.2.jar
+move c:\tmp\_\jdbc_lib\log4j.jar c:\tmp\_\lib\log4j.jar
+move c:\tmp\_\jdbc_lib\prefuse.jar c:\tmp\_\lib\prefuse.jar
+move c:\tmp\_\jdbc_lib\sdoc-0.5.0-beta.jar c:\tmp\_\lib\sdoc-0.5.0-beta.jar
+move c:\tmp\_\jdbc_lib\tablefilter-swing-5.3.1.jar c:\tmp\_\lib\tablefilter-swing-5.3.1.jar
+copy c:\tmp\_\jdbc_lib\h2-1.4.199.jar c:\tmp\_\lib\h2-1.4.199.jar
 
-rm c:\tmp\_\jdbc_lib\dbunit-2.4.4.jar
-rm c:\tmp\_\jdbc_lib\h2-1.3.160.jar
-rm c:\tmp\_\jdbc_lib\h2-1.3.175.jar
-rm c:\tmp\_\jdbc_lib\jsqlparser-1.1.jar
-rm c:\tmp\_\jdbc_lib\junit-4.4.jar
-rm c:\tmp\_\jdbc_lib\postgresql-42.2.0.jre7.jar
+del c:\tmp\_\jdbc_lib\dbunit-2.4.4.jar
+del c:\tmp\_\jdbc_lib\h2-1.3.160.jar
+del c:\tmp\_\jdbc_lib\h2-1.3.175.jar
+del c:\tmp\_\jdbc_lib\jsqlparser-1.1.jar
+del c:\tmp\_\jdbc_lib\junit-4.4.jar
+del c:\tmp\_\jdbc_lib\postgresql-42.2.0.jre7.jar
 
 cd 
 SETLOCAL ENABLEDELAYEDEXPANSION
@@ -71,8 +70,8 @@ for /f "tokens=*" %%f in ('dir /b c:\tmp\_\jdbc_lib\*') do (
 cd c:\tmp\_
 echo "" > .singleuser
 
-jlink --add-modules java.base,java.datatransfer,java.desktop,java.logging,java.management,java.scripting,java.sql,java.xml,java.rmi,java.scripting,java.xml.crypto --output myjre 
-jpackage --name myapp --input . --main-jar jailer.jar --type msi --icon jailer.ico --win-menu --win-menu-group JailerJ14 --vendor Wisser --app-version 2.26 --win-upgrade-uuid d636b4ee-6f10-451e-bf57-c89656780e22 --add-launcher "Jailer Data Browser"=databrowserlauncher.properties --runtime-image myjre
+jlink --add-modules java.se --output jre%version% 
+jpackage --name "Jailer %version%" --input . --main-jar jailer.jar --type msi --icon jailer.ico --win-menu --win-menu-group Jailer --vendor Wisser --app-version %version% --win-upgrade-uuid d636b4ee-6f10-451e-bf57-c89656780e22 --add-launcher "Jailer %version% Data Browser"=databrowserlauncher.properties --runtime-image jre%version%
 
-
-pause
+move *.msi C:\Users\ralfw\tmp\"Jailer-%version%-with-java-JRE.msi"
+move *.msi c:\tmp\"Jailer-%version%-with-java-JRE.msi"

admin/j14pack.sh

@@ -1,75 +1,78 @@
+ #!/bin/bash
 
-rm -r ~/tmp/_
-mkdir ~/tmp/_
-rm -r ~/tmp/myjre
-cd ..
+rm -rf ~/tmp/_ 
+mkdir ~/tmp/_ 
+rm -rf ~/tmp/jre$1  
+cd .. 
+ 
+cp -r maven-artifacts ~/tmp/_ 
+cp -r bookmark ~/tmp/_ 
+cp -r template ~/tmp/_ 
+cp -r build.xml ~/tmp/_ 
+cp -r config ~/tmp/_ 
+cp -r datamodel ~/tmp/_ 
+cp -r demo-sakila-1.4.mv.db ~/tmp/_ 
+cp -r demo-scott-1.4.mv.db ~/tmp/_ 
+cp -r demo-scott-subset-1.4.mv.db ~/tmp/_ 
+cp -r driverlist.csv ~/tmp/_ 
+cp -r extractionmodel ~/tmp/_ 
+cp -r jailer.bat ~/tmp/_ 
+cp -r Jailer.exe ~/tmp/_ 
+cp -r jailer.jar ~/tmp/_ 
+cp -r jailer.sh ~/tmp/_ 
+cp -r jailer.xml ~/tmp/_ 
+cp -r jailerDataBrowser.bat ~/tmp/_ 
+cp -r JailerDataBrowser.exe ~/tmp/_ 
+cp -r jailerDataBrowser.sh ~/tmp/_ 
+cp -r jailerGUI.bat ~/tmp/_ 
+cp -r jailerGUI.sh ~/tmp/_ 
+cp -r layout ~/tmp/_ 
+cp -r lib ~/tmp/_ 
+cp -r license-prefuse.txt ~/tmp/_ 
+cp -r license.txt ~/tmp/_ 
+cp -r manifest.mf ~/tmp/_ 
+cp -r README.md ~/tmp/_ 
+cp -r releasenotes.txt ~/tmp/_ 
+cp -r render ~/tmp/_ 
+cp admin/jailer.png ~/tmp/_ 
+cp admin/databrowserlauncher.properties ~/tmp/_ 
+cp admin/databrowserlauncher-linux.properties ~/tmp/_ 
+ 
+cp -r ~/tmp/_/lib ~/tmp/_/jdbc_lib 
+rm ~/tmp/_/lib/* 
+rm ~/tmp/_/*.bat 
+rm ~/tmp/_/*.exe 
 
-cp -r bookmark ~/tmp/_
-cp -r build.xml ~/tmp/_
-cp -r config ~/tmp/_
-cp -r datamodel ~/tmp/_
-cp -r dbeauty.bat ~/tmp/_
-cp -r dbeauty.exe ~/tmp/_
-cp -r dbeauty.sh ~/tmp/_
-cp -r demo-sakila-1.4.mv.db ~/tmp/_
-cp -r demo-scott-1.4.mv.db ~/tmp/_
-cp -r demo-scott-subset-1.4.mv.db ~/tmp/_
-cp -r driverlist.csv ~/tmp/_
-cp -r extractionmodel ~/tmp/_
-cp -r jailer.bat ~/tmp/_
-cp -r Jailer.exe ~/tmp/_
-cp -r jailer.jar ~/tmp/_
-cp -r jailer.sh ~/tmp/_
-cp -r jailer.xml ~/tmp/_
-cp -r jailerDataBrowser.bat ~/tmp/_
-cp -r JailerDataBrowser.exe ~/tmp/_
-cp -r jailerDataBrowser.sh ~/tmp/_
-cp -r jailerGUI.bat ~/tmp/_
-cp -r jailerGUI.sh ~/tmp/_
-cp -r layout ~/tmp/_
-cp -r lib ~/tmp/_
-cp -r license-prefuse.txt ~/tmp/_
-cp -r license.txt ~/tmp/_
-cp -r manifest.mf ~/tmp/_
-cp -r README.md ~/tmp/_
-cp -r releasenotes.txt ~/tmp/_
-cp -r render ~/tmp/_
-cp admin/jailer.png ~/tmp/_
-cp admin/databrowserlauncher.properties ~/tmp/_
+chmod a+x ~/tmp/_/*.sh 
+ 
+mv ~/tmp/_/jdbc_lib/activation-1.0.2.jar ~/tmp/_/lib/activation-1.0.2.jar 
+mv ~/tmp/_/jdbc_lib/args4j.jar ~/tmp/_/lib/args4j.jar 
+mv ~/tmp/_/jdbc_lib/jaxb-api-2.3.0-b170201.1204.jar ~/tmp/_/lib/jaxb-api-2.3.0-b170201.1204.jar 
+mv ~/tmp/_/jdbc_lib/jaxb-core-2.3.0-b170127.1453.jar ~/tmp/_/lib/jaxb-core-2.3.0-b170127.1453.jar 
+mv ~/tmp/_/jdbc_lib/jaxb-impl-2.3.0-b170127.1453.jar ~/tmp/_/lib/jaxb-impl-2.3.0-b170127.1453.jar 
+mv ~/tmp/_/jdbc_lib/jsqlparser-3.2.jar ~/tmp/_/lib/jsqlparser-3.2.jar 
+mv ~/tmp/_/jdbc_lib/log4j.jar ~/tmp/_/lib/log4j.jar 
+mv ~/tmp/_/jdbc_lib/prefuse.jar ~/tmp/_/lib/prefuse.jar 
+mv ~/tmp/_/jdbc_lib/sdoc-0.5.0-beta.jar ~/tmp/_/lib/sdoc-0.5.0-beta.jar 
+mv ~/tmp/_/jdbc_lib/tablefilter-swing-5.3.1.jar ~/tmp/_/lib/tablefilter-swing-5.3.1.jar 
+cp ~/tmp/_/jdbc_lib/h2-1.4.199.jar ~/tmp/_/lib/h2-1.4.199.jar 
+ 
+rm ~/tmp/_/jdbc_lib/dbunit-2.4.4.jar 
+rm ~/tmp/_/jdbc_lib/h2-1.3.160.jar 
+rm ~/tmp/_/jdbc_lib/h2-1.3.175.jar 
+rm ~/tmp/_/jdbc_lib/jsqlparser-1.1.jar 
+rm ~/tmp/_/jdbc_lib/junit-4.4.jar 
+rm ~/tmp/_/jdbc_lib/postgresql-42.2.0.jre7.jar 
 
-cp -r ~/tmp/_/lib ~/tmp/_/jdbc_lib
-rm ~/tmp/_/lib/*
-rm ~/tmp/_/*.bat
-rm ~/tmp/_/*.exe
-rm ~/tmp/_/*.sh
+cd  
+for file in ~/tmp/_/jdbc_lib/*; do
+  mv $file $file.x
+done
 
-mv ~/tmp/_/jdbc_lib/activation-1.0.2.jar ~/tmp/_/lib/activation-1.0.2.jar
-mv ~/tmp/_/jdbc_lib/args4j.jar ~/tmp/_/lib/args4j.jar
-mv ~/tmp/_/jdbc_lib/jaxb-api-2.3.0-b170201.1204.jar ~/tmp/_/lib/jaxb-api-2.3.0-b170201.1204.jar
-mv ~/tmp/_/jdbc_lib/jaxb-core-2.3.0-b170127.1453.jar ~/tmp/_/lib/jaxb-core-2.3.0-b170127.1453.jar
-mv ~/tmp/_/jdbc_lib/jaxb-impl-2.3.0-b170127.1453.jar ~/tmp/_/lib/jaxb-impl-2.3.0-b170127.1453.jar
-mv ~/tmp/_/jdbc_lib/jsqlparser-1.3.jar ~/tmp/_/lib/jsqlparser-1.3.jar
-mv ~/tmp/_/jdbc_lib/log4j.jar ~/tmp/_/lib/log4j.jar
-mv ~/tmp/_/jdbc_lib/prefuse.jar ~/tmp/_/lib/prefuse.jar
-mv ~/tmp/_/jdbc_lib/sdoc-0.5.0-beta.jar ~/tmp/_/lib/sdoc-0.5.0-beta.jar
-mv ~/tmp/_/jdbc_lib/tablefilter-swing-5.3.1.jar ~/tmp/_/lib/tablefilter-swing-5.3.1.jar
+cd ~/tmp/_ 
+echo "" > .singleuser 
 
-rm ~/tmp/_/jdbc_lib/dbunit-2.4.4.jar
-rm ~/tmp/_/jdbc_lib/h2-1.3.160.jar
-rm ~/tmp/_/jdbc_lib/h2-1.3.175.jar
-rm ~/tmp/_/jdbc_lib/jsqlparser-1.1.jar
-rm ~/tmp/_/jdbc_lib/junit-4.4.jar
-rm ~/tmp/_/jdbc_lib/postgresql-42.2.0.jre7.jar
+/home/ralf/jdk-15.0.1/bin/jlink --add-modules java.se --output ../jre$1  
+/home/ralf/jdk-15.0.1/bin/jpackage --name "Jailer" --linux-package-name jailer-database-tools --arguments "-jpack" --input . --main-jar jailer.jar --type deb --icon jailer.png --vendor Wisser --app-version "$1" --add-launcher "Jailer Data Browser"=databrowserlauncher-linux.properties --runtime-image ../jre$1 
 
-cd 
-SETLOCAL ENABLEDELAYEDEXPANSION
-for /f "tokens=*" %%f in ('dir /b ~/tmp/_/jdbc_lib/*') do (
-  move ~/tmp/_/jdbc_lib/"%%f" ~/tmp/_/jdbc_lib/"%%f.x"
-)
-exit
-
-cd ~/tmp/_
-echo "" > .singleuser
-
-~/jdk-14/bin/jlink --add-modules java.base,java.datatransfer,java.desktop,java.logging,java.management,java.scripting,java.sql,java.xml,java.rmi,java.scripting,java.xml.crypto --output myjre 
-~/jdk-14/bin/jpackage --name Jailer --input . --main-jar jailer.jar --type deb --icon jailer.png --vendor Wisser --app-version 2.8 --add-launcher "Jailer Data Browser"=databrowserlauncher.properties --runtime-image myjre
+cp *.deb /mnt/c/Users/ralfw/tmp/jailer-database-tools_$1-x64.deb

admin/jbuild.sh

@@ -1,3 +1,5 @@
+echo on
+
 echo $JAVA_HOME
 
 echo "$JAVA_HOME/bin/java" -version 
@@ -18,6 +20,21 @@ cd git/Jailer
 git pull
 sh admin/release.sh $1
 
+# make *.sh executable
+cd
+cd tmp
+rm -r _2$1
+mkdir _2$1
+echo wsl rm -rf /home/ralf/jailer > _.bat
+./_.bat
+echo wsl unzip /mnt/c/Users/ralfw/tmp/jailer_$1.zip -d /home/ralf/ > _.bat
+./_.bat
+echo wsl chmod a+x /home/ralf/jailer/*.sh > _.bat
+./_.bat
+rm jailer_$1.zip
+echo "wsl cd; zip -r /mnt/c/Users/ralfw/tmp/jailer_$1.zip jailer" > _.bat
+./_.bat
+
 cd
 cd tmp
 rm -r _$1
@@ -26,6 +43,33 @@ cd _$1
 unzip ../jailer_$1.zip 
 cd jailer/
 
+unzip docs/admin.zip
+echo rm -rf ../../oss
+rm -rf ../../oss
+mkdir ../../oss
+sed s/VERSION/$1/g admin/oss/jailer-engine.pom > ../../oss/jailer-engine-$1.pom
+sed s/VERSION/$1/g admin/oss/oss.bat > ../../oss/oss.bat
+cp maven-artifacts/* ../../oss/
+
+cd admin
+./j14pack.bat $1
+cd
+cp /mnt/c/tmp/*.msi .
+
+cd
+cd tmp
+rm -r _$1
+mkdir _$1
+cd _$1
+unzip ../jailer_$1.zip 
+cd jailer/
+
+unzip docs/admin.zip
+cd admin
+dos2unix *.sh
+echo "wsl sh j14pack.sh $1" > _.bat
+./_.bat
+
 cd
 cd tmp
 cd _$1
@@ -44,7 +88,7 @@ CP="$CP;$LIB/activation-1.0.2.jar"
 CP="$CP;$LIB/jaxb-core-2.3.0-b170127.1453.jar"
 CP="$CP;$LIB/jaxb-impl-2.3.0-b170127.1453.jar"
 CP="$CP;$LIB/jaxb-api-2.3.0-b170201.1204.jar"
-CP="$CP;$LIB/jsqlparser-1.3.jar"
+CP="$CP;$LIB/jsqlparser-3.2.jar"
 CP="$CP;$LIB/tablefilter-swing-5.3.1.jar"
 CP="$CP;jailer.jar"
 

admin/launch4j.xml

@@ -13,7 +13,7 @@
   <stayAlive>true</stayAlive>
   <restartOnCrash>false</restartOnCrash>
   <manifest></manifest>
-  <icon>D:\git\Jailer\src\main\gui\net\sf\jailer\ui\resource\jailer.ico</icon>
+  <icon>C:\Users\RalfW\git\Jailer2\src\main\gui\net\sf\jailer\ui\resource\jailer.ico</icon>
   <classPath>
     <mainClass>net.sf.jailer.ui.ExtractionModelFrame</mainClass>
     <cp>lib/activation-1.0.2.jar</cp>
@@ -25,17 +25,18 @@
     <cp>lib/log4j.jar</cp>
     <cp>lib/args4j.jar</cp>
     <cp>lib/sdoc-0.5.0-beta.jar</cp>
-    <cp>lib/jsqlparser-1.3.jar</cp>
     <cp>lib/tablefilter-swing-5.3.1.jar</cp>
+    <cp>lib/jsqlparser-3.2.jar</cp>
   </classPath>
   <jre>
     <path></path>
     <bundledJre64Bit>false</bundledJre64Bit>
     <bundledJreAsFallback>false</bundledJreAsFallback>
-    <minVersion>1.7.0</minVersion>
+    <minVersion>1.8.0</minVersion>
     <maxVersion></maxVersion>
     <jdkPreference>preferJre</jdkPreference>
     <runtimeBits>64/32</runtimeBits>
     <maxHeapSize>1200</maxHeapSize>
+    <opt>-Djava.util.Arrays.useLegacyMergeSort=true</opt>
   </jre>
 </launch4jConfig>

admin/launch4jdb.xml

@@ -13,7 +13,7 @@
   <stayAlive>true</stayAlive>
   <restartOnCrash>false</restartOnCrash>
   <manifest></manifest>
-  <icon>D:\git\Jailer\src\main\gui\net\sf\jailer\ui\resource\jailer.ico</icon>
+  <icon>C:\Users\RalfW\git\Jailer2\src\main\gui\net\sf\jailer\ui\resource\jailer.ico</icon>
   <classPath>
     <mainClass>net.sf.jailer.ui.databrowser.DataBrowser</mainClass>
     <cp>lib/activation-1.0.2.jar</cp>
@@ -25,17 +25,18 @@
     <cp>lib/log4j.jar</cp>
     <cp>lib/args4j.jar</cp>
     <cp>lib/sdoc-0.5.0-beta.jar</cp>
-    <cp>lib/jsqlparser-1.3.jar</cp>
+    <cp>lib/jsqlparser-3.2jar</cp>
     <cp>lib/tablefilter-swing-5.3.1.jar</cp>
   </classPath>
   <jre>
     <path></path>
     <bundledJre64Bit>false</bundledJre64Bit>
     <bundledJreAsFallback>false</bundledJreAsFallback>
-    <minVersion>1.7.0</minVersion>
+    <minVersion>1.8.0</minVersion>
     <maxVersion></maxVersion>
     <jdkPreference>preferJre</jdkPreference>
     <runtimeBits>64/32</runtimeBits>
     <maxHeapSize>1200</maxHeapSize>
+    <opt>-Djava.util.Arrays.useLegacyMergeSort=true</opt>
   </jre>
 </launch4jConfig>

admin/oss/jailer-engine.pom

@@ -5,7 +5,7 @@
 
 	<groupId>io.github.wisser</groupId>
 	<artifactId>jailer-engine</artifactId>
-	<!version>X</version>
+	<version>VERSION</version>
 	<packaging>jar</packaging>
 
 	<name>jailer-engine</name>

admin/oss/oss.bat

@@ -1,5 +1,5 @@
-gpg -ab jailer-engine-X.jar
-gpg -ab jailer-engine-X-sources.jar
-gpg -ab jailer-engine-X-javadoc.jar
-gpg -ab jailer-engine-X.pom
-jar -cvf bundle.jar *.jar *.pom *.asc
+gpg -ab jailer-engine-VERSION.jar
+gpg -ab jailer-engine-VERSION-sources.jar
+gpg -ab jailer-engine-VERSION-javadoc.jar
+gpg -ab jailer-engine-VERSION.pom
+zip oss.zip *.jar *.pom *.asc

admin/postProcess.sh

@@ -1,20 +0,0 @@
-dir=`pwd`
-echo $dir
-rm -rf result
-mkdir result
-for filename in *.zip; do 
-	echo $filename
-	rm -rf ~/_
-	mkdir ~/_
-	unzip $filename -d ~/_ > /dev/null
-	chmod a+x ~/_/jailer/*.sh
-	chmod a+x ~/_/dbeauty/*.sh
-	chmod a+x ~/_/jailer/*.bat
-	chmod a+x ~/_/dbeauty/*.bat
-	chmod a+x ~/_/jailer/*.exe
-	chmod a+x ~/_/dbeauty/*.exe
-	cd ~/_
-	zip -r $dir/result/$filename *
-	cd $dir
-	pwd
-done

admin/release.sh

@@ -1,9 +1,7 @@
 rm -rf ~/tmp/jailer*
-rm -rf ~/tmp/dbeauty*
 rm -rf ~/tmp/$1
 rm -rf ~/tmp/$1.co
 mkdir C:/tmp/jailer
-mkdir C:/tmp/dbeauty
 mkdir ~/tmp/jailer
 mkdir ~/tmp/$1
 mkdir ~/tmp/$1.co
@@ -18,6 +16,9 @@ sed "s/stateOffset = 100/stateOffset = 0/g" src/main/gui/net/sf/jailer/ui/Enviro
 ant all
 sed "s/stateOffset = 0/stateOffset = 100/g" src/main/gui/net/sf/jailer/ui/Environment.java --in-place
 
+rm maven-artifacts/dummy
+mv jailer-engine* maven-artifacts
+
 rm -rf docs/api
 rm -rf out
 
@@ -31,21 +32,6 @@ makensis tmp.nsi
 cd ..
 rm admin/tmp.nsi
 
-rm -rf C:/tmp/dbeauty
-cp -r . C:/tmp/dbeauty/
-rm -rf C:/tmp/dbeauty/admin
-
-sed s/%VERSION%/$1/g admin/dbeauty.nsi > admin/tmp.nsi
-cd admin
-makensis tmp.nsi
-
-cd ..
-rm admin/tmp.nsi
-
-sed s/%VERSION%/$1/g admin/Jailer.nsi > admin/tmp.nsi
-makensis admin/tmp.nsi
-rm admin/tmp.nsi
-
 mv admin/*nstall* ..
 
 dos2unix *.sh
@@ -57,8 +43,6 @@ rm -rf admin
 cd ..
 rm $1.zip
 zip -r jailer_$1.zip jailer
-cp -r jailer dbeauty
-zip -r dbeauty_$1.zip dbeauty 
 
 # Web upload
 # cd docs

build.xml

@@ -15,8 +15,8 @@ available targets:
 	<property name="SRC_DIR" location="src" />
 	<property name="JAR" location="jailer.jar" />
 	<property name="JAR-ENGINE" location="jailer-engine.jar" />
-	<property name="JAR-ENGINE-SRC" location="jailer-engine-src.zip" />
-	<property name="JAR-ENGINE-DOC" location="jailer-engine-javadoc.zip" />
+	<property name="JAR-ENGINE-SRC" location="jailer-engine-sources.jar" />
+	<property name="JAR-ENGINE-DOC" location="jailer-engine-javadoc.jar" />
 	<property name="TEST_BASE_DIR" location="src/test" />
 	<property file="src/test/test.properties" />
 
@@ -39,7 +39,7 @@ available targets:
 	<target name="compile-engine" depends="clean">
 		<delete file="${JAR-ENGINE}" />
 		<mkdir dir="${COMPILE_DIR}" />
-		<javac source="1.7" target="1.7" destdir="${COMPILE_DIR}" srcdir="${SRC_DIR}/main/engine" debug="on" deprecation="off" encoding="ISO-8859-1">
+		<javac source="1.8" target="1.8" destdir="${COMPILE_DIR}" srcdir="${SRC_DIR}/main/engine" debug="on" deprecation="off" encoding="ISO-8859-1">
 			<classpath>
 				<path refid="default.classpath" />
 			</classpath>
@@ -65,7 +65,7 @@ available targets:
 	<target name="compile-gui" depends="compile-engine">
 		<delete file="${JAR}" />
 		<mkdir dir="${COMPILE_DIR}" />
-		<javac source="1.7" target="1.7" destdir="${COMPILE_DIR}" srcdir="${SRC_DIR}/main/gui" debug="on" deprecation="off" encoding="ISO-8859-1">
+		<javac source="1.8" target="1.8" destdir="${COMPILE_DIR}" srcdir="${SRC_DIR}/main/gui" debug="on" deprecation="off" encoding="ISO-8859-1">
 			<classpath>
 				<path refid="default.classpath" />
 			</classpath>
@@ -91,7 +91,7 @@ available targets:
 		<jar destfile="${JAR}" basedir="${COMPILE_DIR}">
 			<manifest>
 				<attribute name="Main-Class" value="net.sf.jailer.ui.ExtractionModelFrame" />
-				<attribute name="Class-Path" value="lib/activation-1.0.2.jar lib/jaxb-core-2.3.0-b170127.1453.jar lib/jaxb-impl-2.3.0-b170127.1453.jar lib/jaxb-api-2.3.0-b170201.1204.jar lib/prefuse.jar config/ lib/log4j.jar lib/args4j.jar lib/sdoc-0.5.0-beta.jar lib/jsqlparser-1.3.jar lib/tablefilter-swing-5.3.1.jar" />
+				<attribute name="Class-Path" value="lib/activation-1.0.2.jar lib/jaxb-core-2.3.0-b170127.1453.jar lib/jaxb-impl-2.3.0-b170127.1453.jar lib/jaxb-api-2.3.0-b170201.1204.jar lib/prefuse.jar config/ lib/log4j.jar lib/args4j.jar lib/sdoc-0.5.0-beta.jar lib/jsqlparser-3.2.jar lib/tablefilter-swing-5.3.1.jar" />
 			</manifest>
 		</jar>
 		<copy todir=".">
@@ -107,6 +107,7 @@ available targets:
 		<java
 		    fork="true"
 		    failonerror="true"
+			logError="true"
 			classname="net.sf.jailer.JailerVersion"
 			outputproperty="VERSION">
 			<classpath>
@@ -118,7 +119,7 @@ available targets:
 
 	<target name="compile-test" depends="compile-engine">
 		<mkdir dir="${TEST_DIR}" />
-		<javac source="1.7" target="1.7" destdir="${TEST_DIR}" srcdir="${SRC_DIR}/test" debug="on" deprecation="off" encoding="ISO-8859-1">
+		<javac source="1.8" target="1.8" destdir="${TEST_DIR}" srcdir="${SRC_DIR}/test" debug="on" deprecation="off" encoding="ISO-8859-1">
 			<classpath>
 				<path refid="test.classpath" />
 			</classpath>
@@ -193,7 +194,7 @@ available targets:
 		</javadoc>
 		<zip destfile="${JAR-ENGINE-DOC}" basedir="docs/api">
 		</zip>
-		<rename src="${JAR-ENGINE-SRC}" dest="jailer-engine-${VERSION}-src.zip"/>
-		<rename src="${JAR-ENGINE-DOC}" dest="jailer-engine-${VERSION}-javadoc.zip"/>
+		<rename src="${JAR-ENGINE-SRC}" dest="jailer-engine-${VERSION}-sources.jar"/>
+		<rename src="${JAR-ENGINE-DOC}" dest="jailer-engine-${VERSION}-javadoc.jar"/>
 	</target>
 </project>

config/log4j.properties

@@ -29,4 +29,3 @@ log4j.appender.A4.MaxBackupIndex=10
 log4j.appender.A4.File=jailer.log
 log4j.appender.A4.layout=org.apache.log4j.PatternLayout
 log4j.appender.A4.layout.ConversionPattern=%d [%t] %-5p %x - %m%n
-

datamodel/Demo-Sakila/column.csv

@@ -1,5 +1,5 @@
 # Table; Columns
-ACTOR; ACTOR_ID DECIMAL(65535, 32767); FIRST_NAME VARCHAR(45); LAST_NAME VARCHAR(45); LAST_UPDATE TIMESTAMP; ; 
+ACTOR; ACTOR_ID DECIMAL; FIRST_NAME VARCHAR(45); LAST_NAME VARCHAR(45); LAST_UPDATE TIMESTAMP; ; 
 ADDRESS; ADDRESS_ID INTEGER; ADDRESS VARCHAR(50); ADDRESS2 VARCHAR(50) null; DISTRICT VARCHAR(20); CITY_ID INTEGER; POSTAL_CODE VARCHAR(10) null; PHONE VARCHAR(20); LAST_UPDATE TIMESTAMP; ; 
 CATEGORY; CATEGORY_ID SMALLINT; NAME VARCHAR(25); LAST_UPDATE TIMESTAMP; ; 
 CITY; CITY_ID INTEGER; CITY VARCHAR(50); COUNTRY_ID SMALLINT; LAST_UPDATE TIMESTAMP; ; 

datamodel/Demo-Sakila/table.csv

@@ -1,5 +1,5 @@
 # Name; Upsert; Primary Key; ; Author
-ACTOR; N; ACTOR_ID DECIMAL(65535, 32767); ; H2 JDBC Driver; ; 
+ACTOR; N; ACTOR_ID DECIMAL; ; H2 JDBC Driver; ; 
 ADDRESS; N; ADDRESS_ID INTEGER; ; H2 JDBC Driver; ; 
 CATEGORY; N; CATEGORY_ID SMALLINT; ; H2 JDBC Driver; ; 
 CITY; N; CITY_ID INTEGER; ; H2 JDBC Driver; ; 

datamodel/Demo-Sakila/version.csv

@@ -1 +1 @@
-8.7.5
+10.2.1

dbeauty.bat

@@ -1,26 +0,0 @@
-echo off
-set LIB=lib
-
-rem JDBC-driver
-rem set CP=%CP%;<jdbc-driver>.jar
-
-rem configuration files in the config directory
-set CP=%CP%;config
-
-rem the libraries
-set CP=%CP%;%LIB%\junit.jar
-set CP=%CP%;%LIB%\commons-logging.jar
-set CP=%CP%;%LIB%\log4j.jar
-set CP=%CP%;%LIB%\args4j.jar
-set CP=%CP%;%LIB%\spring.jar
-set CP=%CP%;%LIB%\prefuse.jar
-set CP=%CP%;%LIB%\sdoc-0.5.0-beta.jar
-set CP=%CP%;%LIB%\activation-1.0.2.jar
-set CP=%CP%;%LIB%\jaxb-core-2.3.0-b170127.1453.jar
-set CP=%CP%;%LIB%\jaxb-impl-2.3.0-b170127.1453.jar
-set CP=%CP%;%LIB%\jaxb-api-2.3.0-b170201.1204.jar
-set CP=%CP%;%LIB%\jsqlparser-1.3.jar
-set CP=%CP%;%LIB%\tablefilter-swing-5.3.1.jar
-set CP=%CP%;jailer.jar
-
-start javaw -Xmx1024M -cp %CP% net.sf.jailer.ui.databrowser.DataBrowser %*

dbeauty.sh

@@ -1,33 +0,0 @@
-#!/bin/sh
-
-instdir=`dirname $0`
-cd $instdir
-
-LIB=lib
-
-# JDBC-driver
-# CP=$CP:<driver-jar>
-
-# configuration files in the config directory
-CP=$CP:config
-
-# the libraries
-CP=$CP:$LIB/junit.jar
-CP=$CP:$LIB/commons-logging.jar
-CP=$CP:$LIB/log4j.jar
-CP=$CP:$LIB/args4j.jar
-CP=$CP:$LIB/spring.jar
-CP=$CP:$LIB/prefuse.jar
-CP=$CP:$LIB/sdoc-0.5.0-beta.jar
-CP=$CP:$LIB/activation-1.0.2.jar
-CP=$CP:$LIB/jaxb-core-2.3.0-b170127.1453.jar
-CP=$CP:$LIB/jaxb-impl-2.3.0-b170127.1453.jar
-CP=$CP:$LIB/jaxb-api-2.3.0-b170201.1204.jar
-CP=$CP:$LIB/jsqlparser-1.3.jar
-CP=$CP:$LIB/tablefilter-swing-5.3.1.jar
-CP=$CP:jailer.jar
-
-# echo $CP
-
-java -Xmx1024M -cp $CP net.sf.jailer.ui.databrowser.DataBrowser $@
-

docs/Jailer.html

@@ -1,2 +1,2 @@
-<html><head><meta http-equiv="Refresh" content="0; URL=web/index.html">
+<html><head><meta http-equiv="Refresh" content="0; URL=index.html">
 </head><body></body></html>

docs/api.html

@@ -145,7 +145,7 @@
                                 <img src="apipackages.gif" /><br />
                                 <br />
                                 The data model "Demo-Scott" and the extraction model
-                                "Demo-Scott.csv" are both embedded into the package and
+                                "Demo-Scott.jm" are both embedded into the package and
                                 thus accessible as class resources.<br />
                                 <br />
                                 <br /></li>

docs/exporting-data-Dateien/styles.css

@@ -104,18 +104,18 @@ A.llink:hover
 	font-family:Arial, Helvetica, sans-serif;
 	font-size:12px;
 }
-
-/*** Highlight Markup Styles ***/
-#vc_markup .num  { color: #000000; }
-#vc_markup .esc  { color: #bd8d8b; }
-#vc_markup .str  { color: #bd8d8b; }
-#vc_markup .dstr { color: #bd8d8b; }
-#vc_markup .slc  { color: #ac2020; font-style: italic; }
-#vc_markup .com  { color: #ac2020; font-style: italic; }
-#vc_markup .dir  { color: #000000; }
-#vc_markup .sym  { color: #000000; }
-#vc_markup .line { color: #555555; }
-#vc_markup .kwa  { color: #9c20ee; font-weight: bold; }
-#vc_markup .kwb  { color: #208920; }
-#vc_markup .kwc  { color: #0000ff; }
-#vc_markup .kwd  { color: #404040; }
+
+/*** Highlight Markup Styles ***/
+#vc_markup .num  { color: #000000; }
+#vc_markup .esc  { color: #bd8d8b; }
+#vc_markup .str  { color: #bd8d8b; }
+#vc_markup .dstr { color: #bd8d8b; }
+#vc_markup .slc  { color: #ac2020; font-style: italic; }
+#vc_markup .com  { color: #ac2020; font-style: italic; }
+#vc_markup .dir  { color: #000000; }
+#vc_markup .sym  { color: #000000; }
+#vc_markup .line { color: #555555; }
+#vc_markup .kwa  { color: #9c20ee; font-weight: bold; }
+#vc_markup .kwb  { color: #208920; }
+#vc_markup .kwc  { color: #0000ff; }
+#vc_markup .kwd  { color: #404040; }

docs/faq.html

@@ -1,6 +1,6 @@
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
-  
+
   <title>Jailer - FAQ</title>
   <meta http-equiv="Content-Type" content="text/html; charset=us-ascii" />
   <meta name="keywords" content="Data Export Tool" />
@@ -142,7 +142,7 @@
                             until you get all the rows you are looking for.<br />
                             &nbsp;<br />
                             It is easier to get a working extraction model in this way, but it is more difficult to ensure that the model is not too restrictive, i.e. that you get <b>all</b> the rows you want.<br><br />
-                            
+
                             In my experience the best way to define an extraction model
                             is to use the "<span style="font-weight: bold;">Closure</span>" view.<br />
                             &nbsp;<br />
@@ -197,7 +197,7 @@
                                   </tr>
 
                                   <tr>
-                                    <td style="vertical-align: top;">$DISTANCE</td>
+                                    <td style="vertical-align: top;"><b>A.$DISTANCE</b> or <b>B.$DISTANCE</b></td>
 
                                     <td style="vertical-align: top;">INTEGER &nbsp;</td>
 
@@ -207,19 +207,27 @@
                                   </tr>
 
                                   <tr>
-                                    <td>$BIRTHDAY</td>
+                                    <td><b>A.$BIRTHDAY</b></td>
 
                                     <td>INTEGER</td>
 
-                                    <td>Synonym for $DISTANCE.</td>
+                                    <td>Synonym for A.$DISTANCE</td>
                                   </tr>
 
                                   <tr>
-                                    <td>$IS_SUBJECT</td>
+                                    <td><b>A.$IS_SUBJECT</b></td>
 
                                     <td>BOOLEAN</td>
 
-                                    <td>Synonym for $DISTANCE=0.</td>
+                                    <td>Synonym for A.$DISTANCE=0</td>
+                                  </tr>
+
+                                  <tr>
+                                    <td><b>$IN_DELETE_MODE</b></td>
+
+                                    <td>BOOLEAN</td>
+
+                                    <td>Is <i>true</i> during "Delete Reduction" stage, otherwise <i>false</i></td>
                                   </tr>
                                 </tbody>
                               </table><br />
@@ -318,8 +326,10 @@
                             <td>Jailer cannot handle tables without a primary key.
                             Exporting data is not possible if the subject table is associated with a table without a primary key. In this case, you should manually define a primary key in the data model of the tool (not in the database!) using the <i>Data Model Editor</i>. Note that a key must be unique.<br />
                             <br />
-                            (On Oracle, however, <i>rowid</i>-pseudo columns can be used
-                            instead of primary keys)<br />
+
+                            However, when working with Oracle or PostgreSQL, you can use the <i>rowid</i> respectively the <i>ctid</i> pseudo columns instead of primary keys. <br>
+                            See section <i>"Row identification"</i> in the <i>"Data Export"</i> dialog.
+                            <br />
                             &nbsp;</td>
                           </tr>
 
@@ -328,6 +338,24 @@
                             "background-color: rgb(242, 242, 242); font-weight: bold;">
                             Q</td>
 
+                          <td style=
+                            "background-color: rgb(242, 242, 242); font-weight: bold;">
+                            How can I safely export rows without their parents?<br /></td>
+                          </tr>
+
+                          <tr>
+                            <td style="vertical-align: top;">A</td>
+
+                            <td>If a foreign key of a table is nullable, a "<i>null</i>" filter on it (resp. on all columns of a composite foreign key) causes only the foreign key columns of those rows to be filled with "<i>null</i>", whose parent row is not exported as well.<br>
+                            In the same way, only the foreign key columns of those rows are set to "<i>null</i>" when deleting, which are not deleted themselves but their parent.<br>
+                            See <a href="filters.html#nullfilter">filter documentation</a><br /><br /></td>
+                          </tr>
+
+<tr>
+                            <td style=
+                            "background-color: rgb(242, 242, 242); font-weight: bold;">
+                            Q</td>
+
                             <td style=
                             "background-color: rgb(242, 242, 242); font-weight: bold;">
                             The GUI is slow, what can I do?<br /></td>

docs/filters.html

@@ -1,6 +1,6 @@
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
-  
+
   <title>Jailer - Filters</title>
   <meta http-equiv="Content-Type" content="text/html; charset=us-ascii" />
   <meta name="description" content="Data Export Tool" />
@@ -128,8 +128,8 @@
                     <td class="content" height="520" valign="top" width="100%">
                       <small><small>&nbsp;</small></small><br />
                       <br />
-                      A filter assigns a SQL expression to a table column.
-
+                      A filter assigns an SQL expression to a table column.
+					  <br /><br />
                       <h3>Export filter</h3>An export filter is evaluated during the
                       export phase. The column values will be replaced by the result of
                       the expression when the export file is written. ${old-value} is a
@@ -137,7 +137,7 @@
                       <br />
                       With the filter EMPLOYEE.NAME := lower(${old-value})<br />
                       <br />
-                      <img src="filterimg/image001.png" /><br />
+                      <img width="472" height="300" src="filterimg/image001.png" /><br />
                       <br />
                       The employee names will be exported with all letters
                       lowercase:<br />
@@ -150,7 +150,7 @@
                       expression. To define a literal filter, add the prefix "literal:"
                       to the filter expression. The filter:<br />
                       <br />
-                      <img src="filterimg/image003.png" /><br />
+                      <img width="564" height="242" src="filterimg/image003.png" /><br />
                       <br />
                       replaces employee names with numbers from a sequence in the target
                       database.<br />
@@ -170,14 +170,24 @@
                       This filter uses a sequence to generate new EMPNOs during the
                       import phase:<br />
                       <br />
-                      <img src="filterimg/image005.png" /><br />
+                      <img width="1023" height="402" src="filterimg/image005.png" /><br />
                       <br />
                       <br />
 
                       <h3>Filter templates</h3>Templates allow defining filters based on
                       a condition on a column.<br />
                       <br />
-                      <img src="filterimg/image007.png" />
+                      <img width="1066" height="550" src="filterimg/image007.png" />
+                      <br />
+                      <br /><br />
+
+                      <h3 id="nullfilter">"<i>null</i>" filter on foreign key columns</h3>
+                      If a foreign key of a table is nullable, a "<i>null</i>" filter on it (resp. on all columns of a composite foreign key) causes only the foreign key columns of those rows to be filled with "<i>null</i>", whose parent row is not exported as well.<br>
+                      In the same way, only the foreign key columns of those rows are set to "<i>null</i>" when deleting, which are not deleted themselves but their parent.
+                      <br /><br />
+                      The extraction model editor offers the corresponding checkbox "Set foreign key to null ...".
+                      <br /><br/>
+                      <img width="1227" height="359" src="nullfilter.png" />
                     </td>
                   </tr>
                 </tbody>

docs/home.htm

@@ -142,16 +142,22 @@
                       </ul><br />
                       <span style="font-weight: bold;">News</span><br />
                       <br />
-
-                      <table class="bodyTable">
+ 
+                       <table class="bodyTable">
                         <tbody>
+                          <tr class="a">
+                            <td valign="top" nowrap="" style="background-color: rgb(238, 238, 238);">
+                            <small class="date">2021-02-04</small></td>
+
+                            <td valign="top" style="background-color: rgb(238, 238, 238);">Cycles in parent-child relationships will be detected and broken. Thus, such data can be exported by deferring the insertion of nullable foreign keys.</td>
+                          </tr>
                           <tr class="b">
                             <td valign="top" nowrap style="background-color: rgb(221, 221, 238);">
                             <small class="date">2020-01-01</small></td>
 
                             <td valign="top" style="background-color: rgb(221, 221, 238);">The Jailer engine is published in <a href="https://mvnrepository.com/artifact/io.github.wisser/jailer-engine">Maven repository</a>.</td>
                           </tr>
-                          <tr class="a">
+                           <tr class="a">
                             <td valign="top" nowrap style="background-color: rgb(238, 238, 238);">
                             <small class="date">2019-02-01</small></td>
 
@@ -192,68 +198,38 @@
                             <small class="date">2017-03-30</small></td>
 
                             <td valign="top" style="background-color: rgb(238, 238, 238);">Improved
-                            <a href="filters.html">filter management</a>. Support for
-                            import-filters and literal-filters.</td>
-                          </tr>
-
-                          <tr class="b">
-                            <td valign="top" nowrap style="background-color: rgb(221, 221, 238);">
-                            <small class="date">2017-01-27</small></td>
-
-                            <td valign="top" style="background-color: rgb(221, 221, 238);">Referential
-                            cycles can now be exported by deferring the insert of
-                            nullable foreign keys.</td>
-                          </tr>
-
-                          <tr class="a">
-                            <td valign="top" nowrap style="background-color: rgb(238, 238, 238);">
-                            <small class="date">2016-21-10</small></td>
-
-                            <td valign="top" style="background-color: rgb(238, 238, 238);">Filter
+                            <a href="filters.html">filter management</a>. Filter
                             Templates allows you to define rules for assigning filters to
-                            columns.<br />
+                            columns.<br>
                             Filters on primary key columns will automatically be
                             propagated to the corresponding foreign key columns.</td>
                           </tr>
 
-                          <tr class="b">
-                            <td valign="top" nowrap style="background-color: rgb(221, 221, 238);">
-                            <small class="date">2015-12-04</small></td>
+                          
 
-                            <td valign="top" style="background-color: rgb(221, 221, 238);">Support for
-                            oracle's ROWID pseudo-column.</td>
-                          </tr>
+                          
 
-                          <tr class="a">
-                            <td valign="top" nowrap style="background-color: rgb(238, 238, 238);">
-                            <small class="date">2016-09-08</small></td>
+                          
 
-                            <td valign="top" style="background-color: rgb(238, 238, 238);">New "Export
-                            To" mode allows to export rows directly into a different
-                            schema in the same database.</td>
-                          </tr>
+                          
 
                           <tr class="b">
                             <td valign="top" nowrap style="background-color: rgb(221, 221, 238);">
                             <small class="date">2015-12-04</small></td>
 
-                            <td valign="top" style="background-color: rgb(221, 221, 238);">Support for
-                            oracle's ROWID pseudo-column.</td>
+                            <td valign="top" style="background-color: rgb(221, 221, 238);">Data can now also be exported directly to a schema of the same database. This ensures optimal performance.</td>
                           </tr>
 
                           <tr class="a">
                             <td valign="top" nowrap style="background-color: rgb(238, 238, 238);">
                             <small class="date">2015-10-23</small></td>
 
-                            <td valign="top" style="background-color: rgb(238, 238, 238);">Release 5.0
-                            introduces the ability to collect rows in a separate embedded
-                            database. This allows it to export data from read-only
-                            databases.</td>
+                            <td valign="top" style="background-color: rgb(238, 238, 238);">Rows can alternatively be collected in a separate embedded database. This allows exporting data from read-only databases.</td>
                           </tr>
 
                           <tr class="b">
                             <td valign="top" nowrap style="background-color: rgb(221, 221, 238);">
-                            <small class="date">2011-07-20</small></td>
+                            <small class="date">2014-07-20</small></td>
 
                             <td valign="top" style="background-color: rgb(221, 221, 238);">Implemented
                             the "<a href="subset-by-example.html">Subset by Example"</a>
@@ -263,7 +239,7 @@
 
                           <tr class="a">
                             <td valign="top" nowrap style="background-color: rgb(238, 238, 238);">
-                            <small class="date">2010-04-15</small></td>
+                            <small class="date">2014-04-15</small></td>
 
                             <td valign="top" style="background-color: rgb(238, 238, 238);">A <a href=
                             "data-browsing.html">Data Browser</a> has been introduced.
@@ -271,36 +247,11 @@
                             foreign-key-based or user-defined relationships.</td>
                           </tr>
 
-                          <tr class="b">
-                            <td style=
-                            "vertical-align: top; background-color: rgb(221, 221, 238);">
-                            <small class="date">2008-12-23</small></td>
+                          
 
-                            <td style=
-                            "vertical-align: top; background-color: rgb(221, 221, 238);">
-                            Jailer now supports the <a href=
-                            "http://www.dbunit.org">DbUnit</a> flat XML dataset file
-                            format, thus allowing the users of the famous JUnit extension
-                            <a href="http://www.dbunit.org">DbUnit</a> to use the
-                            extracted data for unit testing.</td>
-                          </tr>
+                          
 
-                          <tr class="a">
-                            <td valign="top" nowrap style="background-color: rgb(238, 238, 238);">
-                            <small class="date">2007-12-05</small></td>
-
-                            <td valign="top" style="background-color: rgb(238, 238, 238);">Version 2.0
-                            comes with new graphical user interface.</td>
-                          </tr>
-
-                          <tr class="b">
-                            <td valign="top" nowrap style="background-color: rgb(221, 221, 238);">
-                            <small class="date">2007-06-05</small></td>
-
-                            <td valign="top" style="background-color: rgb(221, 221, 238);"><a href=
-                            "exporting-data.htm">Tutorial</a> for Jailer now
-                            available.<br /></td>
-                          </tr>
+                          
                         </tbody>
                       </table><br />
                       <br />

docs/index.html

@@ -145,6 +145,12 @@
  
                        <table class="bodyTable">
                         <tbody>
+                          <tr class="a">
+                            <td valign="top" nowrap="" style="background-color: rgb(238, 238, 238);">
+                            <small class="date">2021-02-04</small></td>
+
+                            <td valign="top" style="background-color: rgb(238, 238, 238);">Cycles in parent-child relationships will be detected and broken. Thus, such data can be exported by deferring the insertion of nullable foreign keys.</td>
+                          </tr>
                           <tr class="b">
                             <td valign="top" nowrap style="background-color: rgb(221, 221, 238);">
                             <small class="date">2020-01-01</small></td>
@@ -192,68 +198,38 @@
                             <small class="date">2017-03-30</small></td>
 
                             <td valign="top" style="background-color: rgb(238, 238, 238);">Improved
-                            <a href="filters.html">filter management</a>. Support for
-                            import-filters and literal-filters.</td>
-                          </tr>
-
-                          <tr class="b">
-                            <td valign="top" nowrap style="background-color: rgb(221, 221, 238);">
-                            <small class="date">2017-01-27</small></td>
-
-                            <td valign="top" style="background-color: rgb(221, 221, 238);">Referential
-                            cycles can now be exported by deferring the insert of
-                            nullable foreign keys.</td>
-                          </tr>
-
-                          <tr class="a">
-                            <td valign="top" nowrap style="background-color: rgb(238, 238, 238);">
-                            <small class="date">2016-21-10</small></td>
-
-                            <td valign="top" style="background-color: rgb(238, 238, 238);">Filter
+                            <a href="filters.html">filter management</a>. Filter
                             Templates allows you to define rules for assigning filters to
-                            columns.<br />
+                            columns.<br>
                             Filters on primary key columns will automatically be
                             propagated to the corresponding foreign key columns.</td>
                           </tr>
 
-                          <tr class="b">
-                            <td valign="top" nowrap style="background-color: rgb(221, 221, 238);">
-                            <small class="date">2015-12-04</small></td>
+                          
 
-                            <td valign="top" style="background-color: rgb(221, 221, 238);">Support for
-                            oracle's ROWID pseudo-column.</td>
-                          </tr>
+                          
 
-                          <tr class="a">
-                            <td valign="top" nowrap style="background-color: rgb(238, 238, 238);">
-                            <small class="date">2016-09-08</small></td>
+                          
 
-                            <td valign="top" style="background-color: rgb(238, 238, 238);">New "Export
-                            To" mode allows to export rows directly into a different
-                            schema in the same database.</td>
-                          </tr>
+                          
 
                           <tr class="b">
                             <td valign="top" nowrap style="background-color: rgb(221, 221, 238);">
                             <small class="date">2015-12-04</small></td>
 
-                            <td valign="top" style="background-color: rgb(221, 221, 238);">Support for
-                            oracle's ROWID pseudo-column.</td>
+                            <td valign="top" style="background-color: rgb(221, 221, 238);">Data can now also be exported directly to a schema of the same database. This ensures optimal performance.</td>
                           </tr>
 
                           <tr class="a">
                             <td valign="top" nowrap style="background-color: rgb(238, 238, 238);">
                             <small class="date">2015-10-23</small></td>
 
-                            <td valign="top" style="background-color: rgb(238, 238, 238);">Release 5.0
-                            introduces the ability to collect rows in a separate embedded
-                            database. This allows it to export data from read-only
-                            databases.</td>
+                            <td valign="top" style="background-color: rgb(238, 238, 238);">Rows can alternatively be collected in a separate embedded database. This allows exporting data from read-only databases.</td>
                           </tr>
 
                           <tr class="b">
                             <td valign="top" nowrap style="background-color: rgb(221, 221, 238);">
-                            <small class="date">2011-07-20</small></td>
+                            <small class="date">2014-07-20</small></td>
 
                             <td valign="top" style="background-color: rgb(221, 221, 238);">Implemented
                             the "<a href="subset-by-example.html">Subset by Example"</a>
@@ -263,7 +239,7 @@
 
                           <tr class="a">
                             <td valign="top" nowrap style="background-color: rgb(238, 238, 238);">
-                            <small class="date">2010-04-15</small></td>
+                            <small class="date">2014-04-15</small></td>
 
                             <td valign="top" style="background-color: rgb(238, 238, 238);">A <a href=
                             "data-browsing.html">Data Browser</a> has been introduced.
@@ -271,36 +247,11 @@
                             foreign-key-based or user-defined relationships.</td>
                           </tr>
 
-                          <tr class="b">
-                            <td style=
-                            "vertical-align: top; background-color: rgb(221, 221, 238);">
-                            <small class="date">2008-12-23</small></td>
+                          
 
-                            <td style=
-                            "vertical-align: top; background-color: rgb(221, 221, 238);">
-                            Jailer now supports the <a href=
-                            "http://www.dbunit.org">DbUnit</a> flat XML dataset file
-                            format, thus allowing the users of the famous JUnit extension
-                            <a href="http://www.dbunit.org">DbUnit</a> to use the
-                            extracted data for unit testing.</td>
-                          </tr>
+                          
 
-                          <tr class="a">
-                            <td valign="top" nowrap style="background-color: rgb(238, 238, 238);">
-                            <small class="date">2007-12-05</small></td>
-
-                            <td valign="top" style="background-color: rgb(238, 238, 238);">Version 2.0
-                            comes with new graphical user interface.</td>
-                          </tr>
-
-                          <tr class="b">
-                            <td valign="top" nowrap style="background-color: rgb(221, 221, 238);">
-                            <small class="date">2007-06-05</small></td>
-
-                            <td valign="top" style="background-color: rgb(221, 221, 238);"><a href=
-                            "exporting-data.htm">Tutorial</a> for Jailer now
-                            available.<br /></td>
-                          </tr>
+                          
                         </tbody>
                       </table><br />
                       <br />

docs/installation-2.htm

@@ -1,6 +1,6 @@
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
-  
+
   <title>Jailer - Installation</title>
   <meta http-equiv="Content-Type" content="text/html; charset=us-ascii" />
   <meta name="keywords" content="Data Export Tool" />
@@ -111,102 +111,29 @@
 
                   <tr>
                     <td class="content" height="520" valign="top" width="100%">
-                      <p><small><small>&nbsp;</small></small><br />
-                      Unpack the file <span style=
-                      "font-style: italic;">jailer_n.n.n.zip.</span> You will get the
-                      following files:<br /></p>
+                      <p>If you do not want to install Java yourself, use the installation file <i>&quot;Jailer-n.n.n-with-java-JRE.msi&quot;</i> (for Windows) or <i>&quot;jailer-database-tools_10.2.2-x64-with-java-JRE.deb&quot;</i> (for Linux).
+</p><p>Otherwise use the installer <i>&quot;Jailer-Install-n.n.n.exe&quot;</i> or unzip the file <i>&quot;jailer_n.n.n.zip&quot;</i>.<br> See also <a href="faq.html#multiuser">FAQ "How do I setup multi-user mode?"</a><br>
+</p>To start the tool from the unpacked zip:
+<ul><li>Database Subsetter
+<ul><li>On windows platform execute <i>&quot;Jailer.exe&quot;</i>. You can also start <i>&quot;jailerGUI.bat&quot;</i></li><li>On Unix/Linux platform execute the script <i>&quot;jailerGUI.sh&quot;</i> or use <i>&quot;java -jar jailer.jar&quot;</i>
+</li></ul></li></ul><ul>
+
+<li>Data Browser
+<ul>
+<li>On windows platform execute <i>&quot;jailerDataBrowser.exe&quot;</i>, or <i>&quot;jailerDataBrowser.bat&quot;</i>
+</li><li>On Unix/Linux platform execute the script <i>&quot;jailerDataBrowser.sh&quot;</i>
+</li></ul></li></ul>
+<br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /><br /></p>
 
                       <table style="width: 100%; text-align: left;" border="0"
                       cellpadding="0" cellspacing="0">
                         <tbody>
                           <tr>
-                            <td style=
-                            "vertical-align: top; background-color: rgb(242, 242, 242);">
-                            <span style="font-family: monospace;">$unzip
-                            jailer_3.6.4.zip<br />
-                            $cd jailer<br style=
-                            "font-family: monospace;" /></span><span style=
-                            "font-family: monospace;">$ll</span><br style=
-                            "font-family: monospace;" />
-                            <span style="font-family: monospace;">-rw-r--r-- 1 wisser
-                            users &nbsp;&nbsp;5090 2011-08-04 08:46 build.xml<br />
-                            drwxr-xr-x 2 wisser users &nbsp;&nbsp;4096 2011-08-04 08:46
-                            datamodel<br />
-                            -rw-r--r-- 1 wisser users &nbsp;&nbsp;3527 2011-08-04 08:46
-                            Demo.csv<br />
-                            drwxr-xr-x 2 wisser users &nbsp;&nbsp;4096 2011-08-04 08:46
-                            domainmodel<br />
-                            -rwxr-xr-x 1 wisser users &nbsp;&nbsp;2784 2011-08-04 08:46
-                            driverlist.csv<br />
-                            drwxr-xr-x 4 wisser users &nbsp;&nbsp;4096 2011-08-04 08:46
-                            epilog<br />
-                            drwxr-xr-x 2 wisser users &nbsp;&nbsp;4096 2011-08-04 08:46
-                            example<br />
-                            drwxr-xr-x 2 wisser users &nbsp;&nbsp;4096 2011-08-04 08:46
-                            extractionmodel<br />
-                            -rw-r--r-- 1 wisser users &nbsp;&nbsp;&nbsp;429 2011-08-04
-                            08:46 jailer.bat<br />
-                            -rw-r--r-- 1 wisser users &nbsp;&nbsp;&nbsp;500 2011-08-04
-                            08:46 jailerDataBrowser.bat<br />
-                            -rwxr-xr-x 1 wisser users &nbsp;43008 2011-08-04 08:46
-                            jailerDataBrowser.exe<br />
-                            -rwxr-xr-x 1 wisser users &nbsp;&nbsp;&nbsp;444 2011-08-04
-                            08:46 jailerDataBrowser.sh<br />
-                            -rwxr-xr-x 1 wisser users &nbsp;36352 2011-08-04 08:46
-                            Jailer.exe<br />
-                            -rw-r--r-- 1 wisser users &nbsp;&nbsp;&nbsp;497 2011-08-04
-                            08:46 jailerGUI.bat<br />
-                            -rw-r--r-- 1 wisser users &nbsp;&nbsp;&nbsp;441 2011-08-04
-                            08:46 jailerGUI.sh<br />
-                            -rw-r--r-- 1 wisser users &nbsp;&nbsp;&nbsp;129 2011-08-04
-                            08:46 Jailer.html<br />
-                            -rw-r--r-- 1 wisser users 934386 2011-08-04 08:46
-                            jailer.jar<br />
-                            -rw-r--r-- 1 wisser users &nbsp;&nbsp;&nbsp;378 2011-08-04
-                            08:46 jailer.sh<br />
-                            -rw-r--r-- 1 wisser users &nbsp;12448 2011-08-04 08:46
-                            jailer.xml<br />
-                            drwxr-xr-x 2 wisser users &nbsp;&nbsp;4096 2011-08-04 08:46
-                            lib<br />
-                            -rw-r--r-- 1 wisser users &nbsp;&nbsp;1573 2011-08-04 08:46
-                            license-prefuse.txt<br />
-                            -rw-r--r-- 1 wisser users &nbsp;10172 2011-08-04 08:46
-                            license.txt<br />
-                            drwxr-xr-x 4 wisser users &nbsp;&nbsp;4096 2011-08-04 08:46
-                            prolog<br />
-                            -rw-r--r-- 1 wisser users &nbsp;&nbsp;2340 2011-08-04 08:46
-                            README<br />
-                            -rw-r--r-- 1 wisser users &nbsp;10694 2011-08-04 08:46
-                            releasenotes.txt<br />
-                            drwxr-xr-x 2 wisser users &nbsp;&nbsp;4096 2011-08-04 08:46
-                            render<br />
-                            drwxr-xr-x 9 wisser users &nbsp;&nbsp;4096 2011-08-04 08:46
-                            script<br />
-                            drwxr-xr-x 4 wisser users &nbsp;&nbsp;4096 2011-08-04 08:46
-                            src<br />
-                            drwxr-xr-x 2 wisser users &nbsp;&nbsp;4096 2011-08-04 08:46
-                            template<br /></span></td>
+
                           </tr>
                         </tbody>
-                      </table><br />
-                      <b>Database Subsetter</b><br />
+                      </table>
 
-                      <p>On windows platform execute <span style=
-                      "font-style: italic;">Jailer.exe</span>. You can also start
-                      <span style="font-style: italic;">jailerGUI.bat</span> or start
-                      Jailer by double-clicking the <span style=
-                      "font-style: italic;">jailer.jar</span> file.<br />
-                      On Unix/Linux platform execute the script <span style=
-                      "font-style: italic;">jailerGUI.sh</span> or use <span style=
-                      "font-style: italic;">java -jar jailer.jar</span></p><br />
-                      <b>Data Browser</b><br />
-
-                      <p>On windows platform execute <span style=
-                      "font-style: italic;">jailerDataBrowser.exe</span>, or use
-                      <span style=
-                      "font-style: italic;">jailerDataBrowser.bat</span><br />
-                      On Unix/Linux platform execute the script <span style=
-                      "font-style: italic;">jailerDataBrowser.sh</span></p>
 
                       <p><br /></p>
                     </td>
@@ -215,9 +142,7 @@
               </table>
             </div>
 
-            <p><br /></p>
-
-            <p><br /></p>
+            <br>
 
             <p><br />
             &nbsp;</p>

docs/prerequisites.htm

@@ -113,18 +113,19 @@
                     <td class="content" height="520" valign="top" width="100%">
                       <p><small><small>&nbsp;<br /></small></small>
                       <ul>
-                      <li>Java JRE 7 (or above)
+                      <li>Java JRE 8 (or above)
                       <a href=
                       "http://www.oracle.com/technetwork/java/javase/downloads/index.html"><img alt=""
                       src="http://jailer.sourceforge.net/arrow.gif" style=
                       "border: 0px solid ; width: 13px; height: 13px;" hspace=
                       "4" />download</a>
-                      <br>
-                      <font color="ff0000">Important: </font>
+                      <ul><li>Windows and Linux installation programs (*.msi and *.deb) already contain a JRE. <b>If you use them, no further installation is needed.</b></li>
+                      <li><font color="ff0000">Important: </font>
                       due to HiDPI graphics support, <b>Java JRE 11</b> (or above) is strongly recommended.
-                      <br><br>
-                      </li>
-                      <li>JDBC-driver for your RDBMS</li></p>
+                      </li></ul>
+                      <li>JDBC-driver for your RDBMS</li>
+                      <ul><li>For many popular database systems a driver is already included.</li></ul>
+                      </p>
 
                       <p><br /></p>
 

driverlist.csv

@@ -1,10 +1,11 @@
-PostgreSQL;jdbc:postgresql://<HOST(localhost)>[:<PORT(5432)>]/[<DATABASE>];org.postgresql.Driver;lib/postgresql-42.2.6.jre7.jar
-Oracle Thin;jdbc:oracle:thin:@<HOST(localhost)>:<PORT(1521)>:<SID>;oracle.jdbc.driver.OracleDriver
-Oracle OCI;jdbc:oracle:oci:@<TNS_NAME>;oracle.jdbc.driver.OracleDriver
-MySQL;jdbc:mysql://<HOST(localhost)>:<PORT(3306)>[/<DATABASE>];org.gjt.mm.mysql.Driver;lib/mysql-connector-java-5.1.5-bin.jar
+PostgreSQL;jdbc:postgresql://<HOST(localhost)>[:<PORT(5432)>]/[<DATABASE>];org.postgresql.Driver;lib/postgresql-42.2.16.jar
+Oracle Thin;jdbc:oracle:thin:@<HOST(localhost)>:<PORT(1521)>:<SID>;oracle.jdbc.driver.OracleDriver;;https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/19.7.0.0/ojdbc8-19.7.0.0.jar https://repo1.maven.org/maven2/com/oracle/database/xml/xdb/19.7.0.0/xdb-19.7.0.0.jar https://repo1.maven.org/maven2/com/oracle/database/xml/xmlparserv2/19.7.0.0/xmlparserv2-19.7.0.0.jar
+Oracle OCI;jdbc:oracle:oci:@<TNS_NAME>;oracle.jdbc.driver.OracleDriver;;https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/19.7.0.0/ojdbc8-19.7.0.0.jar https://repo1.maven.org/maven2/com/oracle/database/xml/xdb/19.7.0.0/xdb-19.7.0.0.jar https://repo1.maven.org/maven2/com/oracle/database/xml/xmlparserv2/19.7.0.0/xmlparserv2-19.7.0.0.jar
+MySQL 8;jdbc:mysql://<HOST(localhost)>:<PORT(3306)>[/<DATABASE>];com.mysql.jdbc.Driver;lib/mysql-connector-java-8.0.21.jar
+MySQL 5;jdbc:mysql://<HOST(localhost)>:<PORT(3306)>[/<DATABASE>];org.gjt.mm.mysql.Driver;lib/mysql-connector-java-5.1.5-bin.jar
 MariaDB;jdbc:mariadb://<HOST(localhost)>:<PORT(3306)>[/<DATABASE>];org.mariadb.jdbc.Driver;lib/mariadb-java-client-2.4.4.jar
 Microsoft SQL Server;jdbc:sqlserver://<HOST(localhost)>[\\<INSTANCE>][:<PORT>];com.microsoft.sqlserver.jdbc.SQLServerDriver;lib/mssql-jdbc-7.2.1.jre8.jar
-IBM Db2;jdbc:db2://<HOST(localhost)>:<PORT(50000)>/<DATABASE>;COM.ibm.db2.jdbc.app.DB2Driver
+IBM Db2;jdbc:db2://<HOST(localhost)>:<PORT(50000)>/<DATABASE>;com.ibm.db2.jcc.DB2Driver;;https://repo1.maven.org/maven2/com/ibm/db2/jcc/11.5.4.0/jcc-11.5.4.0.jar
 SQLite;jdbc:sqlite:<DATABASEFILE>;org.sqlite.JDBC;lib/sqlite-jdbc-3.28.0.jar
 Sybase;jdbc:sybase:Tds:<HOST(localhost)>[:<PORT>];com.sybase.jdbc3.jdbc.SybDriver
 Firebird;jdbc:firebirdsql://<HOST(localhost)>[:<PORT(3050)>]/<DATABASE>;org.firebirdsql.jdbc.FBDriver

jailer.bat

@@ -1,4 +1,4 @@
-echo off
+@echo off
 set LIB=lib
 
 rem JDBC-driver

jailerDataBrowser.bat

@@ -1,4 +1,4 @@
-echo off
+@echo off
 set LIB=lib
 
 rem JDBC-driver
@@ -19,8 +19,8 @@ set CP=%CP%;%LIB%\activation-1.0.2.jar
 set CP=%CP%;%LIB%\jaxb-core-2.3.0-b170127.1453.jar
 set CP=%CP%;%LIB%\jaxb-impl-2.3.0-b170127.1453.jar
 set CP=%CP%;%LIB%\jaxb-api-2.3.0-b170201.1204.jar
-set CP=%CP%;%LIB%\jsqlparser-1.3.jar
+set CP=%CP%;%LIB%\jsqlparser-3.2.jar
 set CP=%CP%;%LIB%\tablefilter-swing-5.3.1.jar
 set CP=%CP%;jailer.jar
 
-start javaw -Xmx1024M -cp %CP% net.sf.jailer.ui.databrowser.DataBrowser %*
+start javaw -Xmx1024M -Djava.util.Arrays.useLegacyMergeSort=true -cp %CP% net.sf.jailer.ui.databrowser.DataBrowser %*

jailerDataBrowser.sh

@@ -23,11 +23,11 @@ CP=$CP:$LIB/activation-1.0.2.jar
 CP=$CP:$LIB/jaxb-core-2.3.0-b170127.1453.jar
 CP=$CP:$LIB/jaxb-impl-2.3.0-b170127.1453.jar
 CP=$CP:$LIB/jaxb-api-2.3.0-b170201.1204.jar
-CP=$CP:$LIB/jsqlparser-1.3.jar
+CP=$CP:$LIB/jsqlparser-3.2.jar
 CP=$CP:$LIB/tablefilter-swing-5.3.1.jar
 CP=$CP:jailer.jar
 
 # echo $CP
 
-java -Xmx1024M -cp $CP net.sf.jailer.ui.databrowser.DataBrowser "$@"
+java -Xmx1024M -Djava.util.Arrays.useLegacyMergeSort=true -cp $CP net.sf.jailer.ui.databrowser.DataBrowser "$@"
 

jailerGUI.bat

@@ -1,4 +1,4 @@
-echo off
+@echo off
 set LIB=lib
 
 rem configuration files in the config directory
@@ -14,8 +14,8 @@ set CP=%CP%;%LIB%\activation-1.0.2.jar
 set CP=%CP%;%LIB%\jaxb-core-2.3.0-b170127.1453.jar
 set CP=%CP%;%LIB%\jaxb-impl-2.3.0-b170127.1453.jar
 set CP=%CP%;%LIB%\jaxb-api-2.3.0-b170201.1204.jar
-set CP=%CP%;%LIB%\jsqlparser-1.3.jar
+set CP=%CP%;%LIB%\jsqlparser-3.2.jar
 set CP=%CP%;%LIB%\tablefilter-swing-5.3.1.jar
 set CP=%CP%;jailer.jar
 
-start javaw -Xmx1200M -cp %CP% net.sf.jailer.ui.ExtractionModelFrame %*
+start javaw -Xmx1200M -Djava.util.Arrays.useLegacyMergeSort=true -cp %CP% net.sf.jailer.ui.ExtractionModelFrame %*

jailerGUI.sh

@@ -21,11 +21,11 @@ CP=$CP:$LIB/activation-1.0.2.jar
 CP=$CP:$LIB/jaxb-core-2.3.0-b170127.1453.jar
 CP=$CP:$LIB/jaxb-impl-2.3.0-b170127.1453.jar
 CP=$CP:$LIB/jaxb-api-2.3.0-b170201.1204.jar
-CP=$CP:$LIB/jsqlparser-1.3.jar
+CP=$CP:$LIB/jsqlparser-3.2.jar
 CP=$CP:$LIB/tablefilter-swing-5.3.1.jar
 CP=$CP:jailer.jar
 
 # echo $CP
 
-java -Xmx1200M -cp $CP net.sf.jailer.ui.ExtractionModelFrame "$@"
+java -Xmx1200M -Djava.util.Arrays.useLegacyMergeSort=true -cp $CP net.sf.jailer.ui.ExtractionModelFrame "$@"
 

releasenotes.txt

@@ -1,4 +1,108 @@
-9.4
+10.3.4
+  - Detection and renewal of invalid database connections.
+  - With nullable (virtual) primary key columns, errors could occur in the generated SQL statements. This has been fixed.
+
+10.3.3
+  - The wording of the dialog for analyzing a scheme has been made more understandable.
+  - Sorting columns in the result of a query in SQL Console did not work
+    if a column did not have an explicit name. This bug was fixed.
+  - The error message of the primary key checker tool is now more readable.
+  - Feature request 67 "Export as imagemap", https://sourceforge.net/p/jailer/feature-requests/67/
+
+10.3.2
+  - The SQL formatting algorithm has been improved.
+  - Increased responsiveness of the SQL console.
+
+10.3.1
+  - The data browser has been revised and improved.
+    In particular, an overview panel for the desktop was developed,
+    which simplifies orientation in complex data views.
+
+10.2.7
+  - Workaround for "Parser hangs in some queries #1013"
+  - Internal refactoring measures
+
+10.2.6
+  - If a foreign key is nullable, rows can now be exported with no parents without violating the integrity
+    by setting the foreign key to null instead. In the same way parents can be deleted without their children.
+  - Support of CLOB/BLOB in MS SQL
+
+10.2.4
+  - When working with PostgreSQL, you can use the "ctid" pseudo columns instead of primary keys
+    for row identification. See https://github.com/Wisser/Jailer/issues/36
+  - Under rare circumstances SQL statements were generated that could not be processed by Sybase database systems.
+    This has been corrected.
+  - The performance of the working table scope "local database" was improved. Especially with embedded usage (API).
+  - If the subsetter is used via API, access to the working tables is synchronized
+    so that different extraction models can be used concurrently.
+
+10.2.2
+  - Error "Streaming result set is still active" when a limit is defined on the subject was fixed (MySQL)
+
+10.2.1
+  - The response time in case of errors has been improved by immediately terminating all running SQL statements.
+
+10.2
+  - Support for the MS SQL data type "hierarchyid".
+  - Severe error in log file creation was fixed.
+
+10.1.2
+  - Automatic download of JDBC drivers.
+  - Workaround for bug "JDK-8215200 : IllegalArgumentException in sun.lwawt.macosx.CPlatformWindow"
+      https://github.com/AdoptOpenJDK/openjdk-jdk11/issues/10
+      https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8215200
+      https://bugs.openjdk.java.net/browse/JDK-8215200
+
+10.0
+  - Java 7 support was dropped in favor of reduced development overheads.
+  - Third party libraries and JDBC drivers have been upgraded accordingly.
+
+9.5.6
+  - New CLI tool for printing the closure of subject tables.
+    https://github.com/Wisser/Jailer/issues/34
+  - The command line interface has been cleaned up and simplified.
+  - An error in the HTML rendering of a data model was corrected.
+
+9.5.5
+  - Fix for "Need help understanding delete script"
+    https://sourceforge.net/p/jailer/discussion/700499/thread/b3685a49/
+    (A further release was necessary, as this was not completely corrected at first)
+  - improved CSV-handling
+
+9.5.4
+  - Deactivated dependencies had no effect on topological sorting. This was corrected.
+    https://sourceforge.net/p/jailer/discussion/700499/thread/b3685a49
+  - Fix for "NoClassDefFoundError was not logged, Jailer failed without showing reason #33"
+    https://github.com/Wisser/Jailer/issues/33
+
+9.5.3
+  - Improved Connection Settings Wizard.
+  - Some diagnostic means have been added.
+  - Minor fixes.
+
+9.5.2
+  - Fix for "[Bug] firebird database - No suitable Inline-View Style known #32"
+    https://github.com/Wisser/Jailer/issues/32
+  - Fix for "Delete Reduction is not deleting rows"
+    https://sourceforge.net/p/jailer/discussion/700499/thread/3d16e17980/
+
+9.5.1
+  - Fix for "[jailer:bugs] #43 PostgreSQL: Export of inet and interval"
+    https://sourceforge.net/p/jailer/bugs/43/
+  - Fix for "[jailer:bugs] #44 not a proxy instance if WorkingTableScope.LOCAL_DATABASEand Tomcat jdbc pool"
+    https://sourceforge.net/p/jailer/bugs/44/
+
+9.5
+  - Increased initial responsiveness of the Data Browser.
+  - Extensive internal redesigns since the last major release.
+
+9.4.3
+  - Fixed issue "Error when running sql script" https://sourceforge.net/p/jailer/discussion/700499/thread/90600d12b6
+
+9.4.2
+  - Improved database metadata management.
+
+9.4.1
   - The new "Degree view" allows quick and easy editing of the associations of tables with the highest connectivity.
 
 9.3.6

src/main/engine/net/sf/jailer/CommandLine.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,55 +25,34 @@ import net.sf.jailer.subsetting.ScriptFormat;
 
 /**
  * Holds command-line arguments.
- * 
+ *
  * @author Ralf Wisser
  */
 public class CommandLine {
 
 	@Option(name="-UTF8",usage="use UTF-8 encoding")
 	public boolean uTF8 = false;
-	
+
 	@Option(name="-format",usage="export file format: SQL, XML, DBUNIT_FLAT_XML, INTRA_DATABASE or LIQUIBASE_XML")
 	public String format = "SQL";
-	
+
 	@Option(name="-target-dbms", usage="target-DBMS: ORACLE, MSSQL, DB2, MySQL, POSTGRESQL, SYBASE, SQLITE, HSQL or H2", metaVar="<DBMS>")
 	public String targetDBMS = null;
-	
+
 	@Option(name="-xml",usage="export entities into XML file (deprecated, use -format XML instead)")
 	public boolean _asXml = false;
-	
+
 	@Option(name="-xml-root",usage="root tag of XML export file",metaVar="tag-name")
 	public String xmlRootTag = "entities";
 
 	@Option(name="-xml-date",usage="pattern for dates in XML and LIQUIBASE_XML export file",metaVar="pattern")
 	public String xmlDatePattern = "yyyy-MM-dd";
-	
+
 	@Option(name="-xml-time",usage="pattern for times in XML and LIQUIBASE_XML export file",metaVar="pattern")
 	public String xmlTimePattern = "HH.mm.ss";
 
 	@Option(name="-xml-timestamp",usage="pattern for time-stamps in XML and LIQUIBASE_XML export file",metaVar="pattern")
 	public String xmlTimeStampPattern = "yyyy-MM-dd-HH.mm.ss";
-	
-	@Option(name="-c",usage="print restricted data-model with closures")
-	public boolean withClosures = false;
-	
-	@Option(name="-explain",usage="(no longer used)")
-	public boolean dummyExplain;
-
-	// explain is always false
-	public boolean explain = false;
-
-	@Option(name="-u",usage="consider associations as un-directed")
-	public boolean undirected = false;
-	
-	@Option(name="-m", usage="(no longer used)")
-	public int maxNumberOfEntities = 0;
-	
-	@Option(name="-script-enhancer", usage="(no longer used)")
-	public String scriptEnhancer = "";
-
-	@Option(name="-t", usage="(no longer used)")
-	public String tabu = "";
 
 	@Option(name="-e",usage="name of the export-script file (compressed if it ends with '.zip' or '.gz')", metaVar="export-script")
 	public String exportScriptFileName = null;
@@ -83,43 +62,43 @@ public class CommandLine {
 
 	@Option(name="-qualifyNames",usage="add schema prefix to table names after analysing the DB", metaVar="export-script")
 	public boolean qualifyNames = false;
-	
+
 	@Option(name="-analyse-alias",usage="look for aliases while analysing the DB")
 	public boolean analyseAlias = false;
-	
+
 	@Option(name="-analyse-synonym",usage="look for synonyms while analysing the DB")
 	public boolean analyseSynonym = false;
-	
+
 	@Option(name="-analyse-view",usage="look for views while analysing the DB")
 	public boolean analyseView = false;
-	
+
 	@Option(name="-d",usage="name of the delete-script file (compressed if it ends with '.zip' or '.gz')", metaVar="delete-script")
 	public String deleteScriptFileName = null;
 
 	@Option(name="-where",usage="subject condition", metaVar="SQL-expression")
 	public String where = null;
-	
+
 	@Option(name="-schemamapping",usage="schema mapping (Default schema is empty string)", metaVar="schema-in-model=schema-in-db[','x=y]*")
 	public String rawschemamapping = null;
-	
+
 	@Option(name="-source-schemamapping",usage="source schema mapping (Default schema is empty string)", metaVar="<from>=<to>[','<from>=<to>]*")
 	public String rawsourceschemamapping = null;
-	
+
 	@Option(name="-deletion-schemamapping",usage="deletion schema mapping (Default schema is empty string)", metaVar="<from>=<to>[','<from>=<to>]*")
 	public String rawdeletionschemamapping = null;
-	
+
 	@Option(name="-parameters",usage="parameters", metaVar="<parameter>=<value>[';'<parameter>=<value>]*")
 	public String parameters = null;
-	
+
 	@Option(name="-threads",usage="number of threads (default is 1)", metaVar="#threads")
 	public int numberOfThreads = 1;
-	
+
 	@Option(name="-entities",usage="maximum number of entities per insert-statement (in export-file, default is 10)", metaVar="#entities")
 	public int numberOfEntities = 10;
-	
+
 	@Option(name="-upsert-only",usage="generate 'upsert'-statements for all entities (in export-file)")
 	public boolean upsertOnly = false;
-	
+
 	@Option(name="-scope",usage="scope of working tables, GLOBAL, SESSION_LOCAL or LOCAL_DATABASE")
 	public String scope = null;
 
@@ -149,19 +128,25 @@ public class CommandLine {
 
 	@Option(name="-independent-working-tables", usage="create working tables that are independent of the extraction model. (Potentially less efficient)")
 	public boolean independentWorkingTables = false;
-	
+
 	@Option(name="-transactional", usage="import rows in a single transaction")
 	public boolean transactional = false;
-	
+
 	@Option(name="-isolation-level", usage="isolation level (optional), 1=READ_UNCOMMITTED, 2=READ_COMMITTED, 4=REPEATABLE_READ, 8=SERIALIZABLE")
 	public Integer isolationLevel = null;
-	
-	@Option(name="-no-rowid", usage="use primary keys to determine row identity (instead of rowid-column)")
-	public boolean noRowid = false;
-	
+
+	@Option(name="-no-rowid", usage="(no longer used, see \"-use-rowid\")")
+	public boolean noRowidNoLongerUsed = false;
+
+	@Option(name="-use-rowid", usage="use rowid/ctid-column to identify rows (instead of primary keys, see \"-use-rowid-if-needed\")")
+	public boolean useRowid = false;
+
+	@Option(name="-use-rowid-if-needed", usage="use rowid/ctid-column only for tables without primary key (see \"-use-rowid\")")
+	public boolean useRowIdsOnlyForTablesWithoutPK = false;
+
 	@Option(name="-import-filter-mapping-table-schema", usage="schema in which the import-filter mapping tables will be created")
 	public String importFilterMappingTableSchema = "";
-	
+
 	@Option(name="-check-primary-keys", usage="(no longer used)")
 	boolean checkPrimaryKeysNoLongerUsed = false;
 
@@ -174,9 +159,9 @@ public class CommandLine {
 	@Option(name="-row-limit", usage="maximum allowed number of exported rows. If this limit is exceeded, the export aborts with an error.")
 	public String limit = null;
 
-	@Option(name="-", usage="do not interpret the next word as an option, even if it begins with '-'. For example, if the username is \"-abc\", use \"- -abc\".")
+	@Option(name="-", usage="do not interpret the next word as an option, even if it begins with a '-'. E.g. if the username is: \"-abc\", use: \"- -abc\".")
 	public List<String> escapedWords = new ArrayList<String>();
-	
+
 	@Option(name="-file-lookup", usage="read the next parameter from the (1st line of the) file named VAL. \n(This is especially useful for not making passwords visible by querying the command line parameters)")
 	public List<String> parameterFile = new ArrayList<String>();
 
@@ -185,7 +170,7 @@ public class CommandLine {
 
 	/**
 	 * Gets the script format.
-	 * 
+	 *
 	 * @return the script format
 	 */
 	public ScriptFormat getScriptFormat() {
@@ -196,5 +181,5 @@ public class CommandLine {
 		}
 		return null;
 	}
-	
+
 }

src/main/engine/net/sf/jailer/CommandLineParser.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,35 +17,35 @@ package net.sf.jailer;
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 
 import org.kohsuke.args4j.CmdLineException;
 import org.kohsuke.args4j.CmdLineParser;
 
 /**
  * Parser for {@link CommandLine}.
- * 
+ *
  * @author Ralf Wisser
  */
 public class CommandLineParser {
-	
+
 	/**
 	 * Parses arguments and initializes the parser.
-	 * 
+	 *
 	 * @param args the arguments
 	 * @param silent if <code>true</code>, no error messages will be written
 	 */
-	public static CommandLine parse(String[] args, boolean silent) throws Exception {
+	public static CommandLine parse(String[] cliArgs, boolean silent) throws Exception {
 		CommandLine commandLine = new CommandLine();
 		try {
-			args = preprocessFileLookup(args);
+			String[] args = preprocessFileLookup(cliArgs);
 			List<String> theArgs = new ArrayList<String>();
-			
+
 			final String ESC_PREFIX = "((!JAILER_MINUS_ESC!!)";
 
 			int i = 0;
@@ -79,7 +79,7 @@ public class CommandLineParser {
 			return commandLine;
 		} catch (CmdLineException e) {
 			System.out.println(e.getMessage());
-			printUsage(args);
+			printUsage(cliArgs);
 			throw e;
 		}
 	}
@@ -94,6 +94,9 @@ public class CommandLineParser {
 				if (line == null) {
 					throw new RuntimeException("File \"" + cArgs[i + 1] + "\" is empty");
 				}
+				if (i > 0 && !"-".equals(cArgs[i - 1])) {
+					result.add("-");
+				}
 				result.add(line);
 				++i;
 			} else {
@@ -107,8 +110,12 @@ public class CommandLineParser {
 	 * Prints out usage.
 	 */
 	public static void printUsage(String[] args) {
+		String cmd = "sh jailer.sh";
+		if (System.getProperty("os.name", "").toLowerCase(Locale.ENGLISH).startsWith("windows")) {
+			cmd = "jailer.bat";
+		}
 		System.out.println("usage:");
-		System.out.println("  jailer export [options] <extraction-model> <jdbc-driver-class> <db-URL> <db-user> <db-password>");
+		System.out.println("  " + cmd + " export [options] <extraction-model> <jdbc-driver-class> <db-URL> <db-user> <db-password>");
 		System.out.println("    extracts data (see option '-e') and optionally creates a delete-script (see option '-d')");
 		System.out.println("    -where subject condition. Optional, overrides condition in extraction-model");
 		System.out.println("    -format [SQL, XML, DBUNIT_FLAT_XML or LIQUIBASE_XML]");
@@ -116,40 +123,35 @@ public class CommandLineParser {
 		System.out.println("    -xml-date pattern for dates in XML and LIQUIBASE_XML export file");
 		System.out.println("    -xml-time pattern for times in XML and LIQUIBASE_XML export file");
 		System.out.println("    -xml-timestamp pattern for time-stamps in XML and LIQUIBASE_XML export file");
-		System.out.println("    -t prevents deletion of entities from 'tabu'-tables");
 		System.out.println();
-		System.out.println("  jailer import <sql-script> <jdbc-driver-class> <db-URL> <db-user> <db-password>");
-		System.out.println("    imports data (with C|BLOB support)");
+		System.out.println("  " + cmd + " import <sql-script> <jdbc-driver-class> <db-URL> <db-user> <db-password>");
+		System.out.println("    imports data (with CLOB/BLOB/XML support)");
 		System.out.println();
-		System.out.println("  jailer delete [options] <extraction-model> <jdbc-driver-class> <db-URL> <db-user> <db-password>");
+		System.out.println("  " + cmd + " delete [options] <extraction-model> <jdbc-driver-class> <db-URL> <db-user> <db-password>");
 		System.out.println("    Like export, but skips the export and creates a delete-script (see option '-d')");
-		System.out.println("    -where subject condition. Optional, overrides condition in extraction-model");
-		System.out.println("    -t prevents deletion of entities from 'tabu'-tables");
+		System.out.println("    -where <subject condition>: optional, overrides condition in extraction-model");
 		System.out.println();
-		System.out.println("  jailer create-ddl [-datamodel VAL] [-target-dbms <DBMS>] [-working-table-schema VAL] [<extraction-model> -independent-working-tables] [-no-rowid]");
+		System.out.println("  " + cmd + " create-ddl [-datamodel VAL] [-target-dbms <DBMS>] [-working-table-schema VAL] [<extraction-model> -independent-working-tables] [-use-rowid] [-use-rowid-if-needed]");
 		System.out.println("    creates the DDL for the working-tables and prints it to stdout");
 		System.out.println();
-		System.out.println("  jailer create-ddl <jdbc-driver-class> <db-URL> <db-user> <db-password> [<extraction-model> -independent-working-tables] [-no-rowid]");
-		System.out.println("    creates the DDL for the working-tables and executes it");
+		System.out.println("  " + cmd + " create-ddl <jdbc-driver-class> <db-URL> <db-user> <db-password> [<extraction-model> -independent-working-tables] [-use-rowid] [-use-rowid-if-needed]");
+		System.out.println("    creates the working-tables");
 		System.out.println();
-		System.out.println("  jailer build-model [-schema <schema>] <jdbc-driver-class> <db-URL> <db-user> <db-password>");
-		System.out.println("    automatically retrieves datamodel elements using the 'model-finder' beans");
-		System.out.println("    reduces JDBC-Introspection to schema <schema>");
+		System.out.println("  " + cmd + " build-model [-schema <schema>] <jdbc-driver-class> <db-URL> <db-user> <db-password>");
+		System.out.println("    determines table and relationship information through database analysis");
+		System.out.println("    -schema <schema>: limits analysis to the schema <schema>");
 		System.out.println();
-		System.out.println("  jailer print-datamodel [options] {<restriction-model>}*");
-		System.out.println("    prints restricted data-model");
-		System.out.println("    -c with closures ");
+		System.out.println("  " + cmd + " render-datamodel [-datamodel VAL] [<extraction-model>] ");
+		System.out.println("    generates a HTML render of the (restricted) data model into directory 'render'");
 		System.out.println();
-		System.out.println("  jailer render-datamodel [options] {<restriction-model>}* ");
-		System.out.println("    generates a HTML render of the restricted data-model into directory 'render'");
-		System.out.println();
-		System.out.println("  jailer find-association [options] <source-table> <destination-table> {<restriction-model>}*");
-		System.out.println("    finds the shortest path of associations between two tables");
-		System.out.println("    -u considers associations as un-directed");
+		System.out.println("  " + cmd + " print-closure <extraction-model> [<separator>] [-datamodel VAL]");
+		System.out.println("    prints a list of all tables that are directly or transitively associated with a subject table,");
+		System.out.println("    taking into account the restrictions on the associations (the so-called \"Closure\")");
+		System.out.println("    <separator>: optional separator between table names in the output");
 		System.out.println();
 		System.out.println("options:");
 		CmdLineParser cmdLineParser = new CmdLineParser(new CommandLine());
-		cmdLineParser.setUsageWidth(120);
+		cmdLineParser.setUsageWidth(160);
 		cmdLineParser.printUsage(System.out);
 		System.out.println();
 		printAruments(System.out, args, null);
@@ -163,7 +165,7 @@ public class CommandLineParser {
 			while (i < args.length) {
 				String arg = args[i];
 				if (arg.equals(password)) {
-					arg = "?";
+					arg = "<password> (not shown)";
 				}
 				if (i > 0) {
 					out.print(", ");

src/main/engine/net/sf/jailer/ExecutionContext.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -35,23 +35,24 @@ import net.sf.jailer.util.LayoutStorage;
 
 /**
  * Execution context of import-/export commands.
- * 
+ *
  * @author Ralf Wisser
  */
 public class ExecutionContext {
-	
+
 	/**
 	 * Default constructor.
 	 */
 	public ExecutionContext() {
 	}
-	
+
 	/**
 	 * Copy constructor.
 	 */
 	public ExecutionContext(ExecutionContext other) {
 		this.schemaMapping = copy(other.schemaMapping);
 		this.sourceSchemaMapping = copy(other.sourceSchemaMapping);
+		this.deletionSchemaMapping = copy(other.deletionSchemaMapping);
 		this.scriptFormat = other.scriptFormat;
 		this.currentModelSubfolder = other.currentModelSubfolder;
 		this.datamodelURL = other.datamodelURL;
@@ -80,7 +81,8 @@ public class ExecutionContext {
 		this.orderByPK = other.orderByPK;
 		this.transactional = other.transactional;
 		this.isolationLevel = other.isolationLevel;
-		this.noRowid = other.noRowid;
+		this.useRowid = other.useRowid;
+		this.useRowIdsOnlyForTablesWithoutPK = other.useRowIdsOnlyForTablesWithoutPK;
 		this.importFilterMappingTableSchema = other.importFilterMappingTableSchema;
 		this.scope = other.scope;
 		this.rawparameters = other.rawparameters;
@@ -96,7 +98,7 @@ public class ExecutionContext {
 
 	/**
 	 * Creates new context with attributes taken from {@link ExecutionContext}.
-	 * 
+	 *
 	 * @param executionContext the command line
 	 */
 	public ExecutionContext(CommandLine commandLine) {
@@ -492,7 +494,7 @@ public class ExecutionContext {
 
 	/**
 	 * Gets IsolationLevel.
-	 * 
+	 *
 	 * @see Connection#setTransactionIsolation(int)
 	 */
 	public Integer getIsolationLevel() {
@@ -501,7 +503,7 @@ public class ExecutionContext {
 
 	/**
 	 * Sets IsolationLevel.
-	 * 
+	 *
 	 * @see Connection#setTransactionIsolation(int)
 	 */
 	public void setIsolationLevel(Integer isolationLevel) {
@@ -509,26 +511,45 @@ public class ExecutionContext {
 	}
 
 	/**
-	 * If <code>true</code>, Use primary keys to determine row identity (instead
-	 * of rowid-column)
+	 * If <code>true</code>, use rowid/ctid-column to determine row identity (instead
+	 * of primary keys)
 	 *
-	 * @return <code>true</code> if Use primary keys to determine row identity
-	 *         (instead of rowid-column)
+	 * @return if <code>true</code> use rowid/ctid-column to determine row identity
+	 *         (instead of primary keys)
 	 */
-	public boolean getNoRowid() {
-		return noRowid;
+	public boolean getUseRowid() {
+		return useRowid;
 	}
 
 	/**
-	 * If <code>true</code>, Use primary keys to determine row identity (instead
-	 * of rowid-column)
+	 * If <code>true</code>, use rowid/ctid-column to determine row identity (instead
+	 * of primary keys)
 	 *
-	 * @param noRowid
-	 *            <code>true</code> if Use primary keys to determine row
-	 *            identity (instead of rowid-column)
+	 * @param useRowid
+	 *            if <code>true</code> use rowid/ctid-column to determine row
+	 *            identity (instead of primary keys)
 	 */
-	public void setNoRowid(boolean noRowid) {
-		this.noRowid = noRowid;
+	public void setUseRowid(boolean useRowid) {
+		this.useRowid = useRowid;
+	}
+
+	/**
+	 * If <code>true</code>, use rowid/ctid-column only for tables without primary key.
+	 *
+	 * @return if <code>true</code> use rowid/ctid-column only for tables without primary key
+	 */
+	public boolean getUseRowIdsOnlyForTablesWithoutPK() {
+		return useRowIdsOnlyForTablesWithoutPK;
+	}
+
+	/**
+	 * If <code>true</code>, use rowid/ctid-column only for tables without primary key.
+	 *
+	 * @param useRowIdsOnlyForTablesWithoutPK
+	 *            if <code>true</code> use rowid/ctid-column only for tables without primary key
+	 */
+	public void setUseRowIdsOnlyForTablesWithoutPK(boolean useRowIdsOnlyForTablesWithoutPK) {
+		this.useRowIdsOnlyForTablesWithoutPK = useRowIdsOnlyForTablesWithoutPK;
 	}
 
 	/**
@@ -607,7 +628,7 @@ public class ExecutionContext {
 	public Map<String, String> getParameters() {
 		if (parameters == null) {
 			Map<String, String> map = new TreeMap<String, String>();
-			
+
 			if (rawparameters != null) {
 				for (String pv: CsvFile.decodeLine(rawparameters)) {
 					int i = pv.indexOf('=');
@@ -617,13 +638,13 @@ public class ExecutionContext {
 				}
 			}
 			parameters = map;
-		}	    	
+		}
 		return parameters;
 	}
-	
+
 	/**
 	 * Sets a parameter.
-	 * 
+	 *
 	 * @param name parameter name
 	 * @param value value
 	 */
@@ -645,7 +666,7 @@ public class ExecutionContext {
 		}
 		return mapping;
 	}
-	
+
 	public Map<String, String> getSchemaMapping() {
 		if (schemaMapping == null) {
 			schemaMapping = getSchemaMapping(rawschemamapping);
@@ -686,7 +707,7 @@ public class ExecutionContext {
 		}
 		return sourceSchemaMapping;
 	}
-	
+
 	private Map<String, String> deletionSchemaMapping;
 
 	/**
@@ -716,10 +737,10 @@ public class ExecutionContext {
 	}
 
 	private ScriptFormat scriptFormat;
-	
+
 	/**
 	 * Gets the script format.
-	 * 
+	 *
 	 * @return the script format
 	 */
 	public ScriptFormat getScriptFormat() {
@@ -732,24 +753,24 @@ public class ExecutionContext {
 		}
 		return scriptFormat;
 	}
-	
+
 	/**
 	 * Sets the script format.
-	 * 
+	 *
 	 * @return the script format
 	 */
 	public void setScriptFormat(ScriptFormat scriptFormat) {
 		this.scriptFormat = scriptFormat;
 	}
-	
+
 	/**
 	 * Folder of current data model.
 	 */
 	private String currentModelSubfolder = null;
-	
+
 	/**
 	 * Sets folder of current data model.
-	 * 
+	 *
 	 * @param modelFolder the folder, <code>null</code> for default model
 	 */
 	public void setCurrentModelSubfolder(String modelFolder) {
@@ -758,7 +779,7 @@ public class ExecutionContext {
 
 	/**
 	 * Gets folder of current data model.
-	 * 
+	 *
 	 * @return modelFolder the folder, <code>null</code> for default model
 	 */
 	public String getCurrentModelSubfolder() {
@@ -801,7 +822,7 @@ public class ExecutionContext {
 			this.datamodelURL = datamodelURL;
 		}
 	}
-	
+
 	// use UTF-8 encoding
 	private boolean uTF8 = false;
 
@@ -878,50 +899,62 @@ public class ExecutionContext {
 
 	// import rows in a single transaction
 	private boolean transactional = false;
-	
+
 	// isolation level
 	private Integer isolationLevel = null;
-	
-	// use primary keys to determine row identity (instead of rowid-column)
-	private boolean noRowid = false;
+
+	// use rowid-column to determine row identity (instead of primary keys)
+	private boolean useRowid = false;
+
+	// use rowid-column only for tables without primary key
+	private boolean useRowIdsOnlyForTablesWithoutPK = false;
 
 	// collects the rows using multiple insert operations with a limited number of rows per operation
 	private boolean insertIncrementally = false;
 
 	// abort the process if the result is inconsistent due to insufficient transaction isolation
 	private boolean abortInCaseOfInconsistency = false;
-	
+
 	// schema in which the import-filter mapping tables will be created
 	private String importFilterMappingTableSchema = "";
 
 	// create working tables that are independent of the extraction model. (Potentially less efficient)
 	private boolean independentWorkingTables = false;
-	
+
 	// maximum allowed number of exported rows. If this limit is exceeded, the export aborts with an error.
 	private Long limit;
 
 	private WorkingTableScope scope = WorkingTableScope.GLOBAL;
 
 	private String rawparameters;
-	
+
 	private boolean embedded = false;
 	private Set<String> upkDomain;
 
 	private String currentConnectionAlias;
-	
+
 	private ProgressListenerRegistry progressListenerRegistry = new ProgressListenerRegistry();
 
 	/**
 	 * Gets the {@link ProgressListenerRegistry}.
-	 * 
+	 *
 	 * @return the {@link ProgressListenerRegistry}
 	 */
 	public ProgressListenerRegistry getProgressListenerRegistry() {
 		return progressListenerRegistry;
 	}
 
+	/**
+	 * Sets the {@link ProgressListenerRegistry}.
+	 *
+	 * @param progressListenerRegistry the {@link ProgressListenerRegistry}
+	 */
+	public void setProgressListenerRegistry(ProgressListenerRegistry progressListenerRegistry) {
+		this.progressListenerRegistry = progressListenerRegistry;
+	}
+
 	private LayoutStorage layoutStorage = new LayoutStorage();
-	
+
 	public LayoutStorage getLayoutStorage() {
 		return layoutStorage;
 	}
@@ -1010,7 +1043,8 @@ public class ExecutionContext {
 		independentWorkingTables = commandLine.independentWorkingTables;
 		transactional = commandLine.transactional;
 		isolationLevel = commandLine.isolationLevel;
-		noRowid = commandLine.noRowid;
+		useRowid = commandLine.useRowid;
+		useRowIdsOnlyForTablesWithoutPK = commandLine.useRowIdsOnlyForTablesWithoutPK;
 		importFilterMappingTableSchema = commandLine.importFilterMappingTableSchema;
 		insertIncrementally = commandLine.insertIncrementally;
 		abortInCaseOfInconsistency = commandLine.abortInCaseOfInconsistency;

src/main/engine/net/sf/jailer/Jailer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,18 +15,19 @@
  */
 package net.sf.jailer;
 
-import java.io.BufferedReader;
 import java.io.File;
-import java.io.StringReader;
+import java.io.IOException;
+import java.net.MalformedURLException;
 import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.LogManager;
 
 import javax.sql.DataSource;
 
@@ -36,42 +37,42 @@ import net.sf.jailer.configuration.Configuration;
 import net.sf.jailer.configuration.DBMS;
 import net.sf.jailer.database.BasicDataSource;
 import net.sf.jailer.database.Session;
-import net.sf.jailer.datamodel.Association;
 import net.sf.jailer.datamodel.DataModel;
 import net.sf.jailer.datamodel.PrimaryKeyFactory;
 import net.sf.jailer.datamodel.Table;
 import net.sf.jailer.ddl.DDLCreator;
-import net.sf.jailer.entitygraph.EntityGraph;
+import net.sf.jailer.extractionmodel.ExtractionModel;
+import net.sf.jailer.extractionmodel.ExtractionModel.AdditionalSubject;
 import net.sf.jailer.modelbuilder.ModelBuilder;
 import net.sf.jailer.progress.ProgressListener;
 import net.sf.jailer.render.DataModelRenderer;
+import net.sf.jailer.render.HtmlDataModelRenderer;
 import net.sf.jailer.restrictionmodel.RestrictionModel;
 import net.sf.jailer.subsetting.SubsettingEngine;
 import net.sf.jailer.util.CancellationException;
 import net.sf.jailer.util.CancellationHandler;
 import net.sf.jailer.util.ClasspathUtil;
 import net.sf.jailer.util.LogUtil;
-import net.sf.jailer.util.PrintUtil;
+import net.sf.jailer.util.Quoting;
 import net.sf.jailer.util.SqlScriptExecutor;
-import net.sf.jailer.util.SqlUtil;
 
 /**
  * Jailer is a tool for database subsetting and relational data browsing. <br>
  * <ul>
  * <li>
- * The Subsetter exports consistent, referentially intact row-sets from relational databases, 
- * generates topologically sorted SQL-DML, DbUnit datasets 
+ * The Subsetter exports consistent, referentially intact row-sets from relational databases,
+ * generates topologically sorted SQL-DML, DbUnit datasets
  * and hierarchically structured XML.
  * </li>
  * <li>
- * The Data Browser allows bidirectional navigation through the database 
+ * The Data Browser allows bidirectional navigation through the database
  * by following foreign-key-based or user-defined relationships.
  * </li>
  * </ul>
- * 
+ *
  * <a href="http://jailer.sourceforge.net/">http://jailer.sourceforge.net</a> <br>
  * <a href="https://github.com/Wisser/Jailer">https://github.com/Wisser/Jailer</a> <br><br>
- * 
+ *
  * @author Ralf Wisser
  */
 public class Jailer {
@@ -79,11 +80,21 @@ public class Jailer {
 	/**
 	 * The logger.
 	 */
-	private static final Logger _log = Logger.getLogger(Jailer.class);
+	private static Logger logger;
+
+	/**
+	 * Gets the logger.
+	 */
+	private static synchronized Logger getLogger() {
+		if (logger == null) {
+			logger = Logger.getLogger(Jailer.class);
+		}
+		return logger;
+	}
 
 	/**
 	 * Main-method for CLI.
-	 * 
+	 *
 	 * @param args arguments
 	 */
 	public static void main(String[] args) {
@@ -103,14 +114,32 @@ public class Jailer {
 			}
 		});
 
-		if (new File(".singleuser").exists() // legacy 
+		try {
+			java.util.logging.Logger rootLogger = LogManager.getLogManager().getLogger("");
+			rootLogger.setLevel(Level.OFF);
+			for (Handler h : rootLogger.getHandlers()) {
+			    h.setLevel(Level.OFF);
+			}
+			System.setProperty("com.sun.xml.bind.v2.bytecode.ClassTailor.noOptimize", "true");
+		} catch (Exception e) {
+		}
+
+		if (new File(".singleuser").exists() // legacy
 				|| new File(".multiuser").exists()) {
 			File home = new File(System.getProperty("user.home"), ".jailer");
 			home.mkdirs();
 			LogUtil.reloadLog4jConfig(home);
 			Configuration configuration = Configuration.getInstance();
 			configuration.setTempFileFolder(new File(home, "tmp").getPath());
+			try {
+				HtmlDataModelRenderer renderer = configuration.getRenderer();
+				if (renderer != null) {
+					renderer.setOutputFolder(new File(home, renderer.getOutputFolder()).getAbsolutePath());
+				}
+			} catch (Exception e) {
+			}
 		}
+		getLogger();
 		try {
 			System.setProperty("db2.jcc.charsetDecoderEncoder", "3");
 		} catch (Exception e) {
@@ -136,7 +165,7 @@ public class Jailer {
 
 	/**
 	 * Main-method for GUI.
-	 * 
+	 *
 	 * @param args
 	 *            arguments
 	 * @param warnings
@@ -158,21 +187,24 @@ public class Jailer {
 			if (progressListener != null) {
 				executionContext.getProgressListenerRegistry().addProgressListener(progressListener);
 			}
-			
+
 			String command = commandLine.arguments.get(0);
 			if (!"create-ddl".equalsIgnoreCase(command)) {
-				if (!"find-association".equalsIgnoreCase(command)) {
-					_log.info("Jailer " + JailerVersion.VERSION);
+				if (!"print-closure".equalsIgnoreCase(command)) {
+					getLogger().info("Jailer " + JailerVersion.VERSION);
 				}
 			}
-			
+
 			URL[] jdbcJarURLs = ClasspathUtil.toURLArray(commandLine.jdbcjar, commandLine.jdbcjar2, commandLine.jdbcjar3, commandLine.jdbcjar4);
 
 			if ("render-datamodel".equalsIgnoreCase(command)) {
-				if (commandLine.arguments.size() <= 1) {
+				if (commandLine.arguments.size() > 2) {
 					CommandLineParser.printUsage(args);
 				} else {
-					renderDataModel(commandLine.arguments, commandLine.withClosures, commandLine.schema, executionContext);
+					if (commandLine.arguments.size() > 1) {
+						updateDataModelFolder(commandLine, commandLine.arguments.get(1), executionContext);
+					}
+					renderDataModel(commandLine.arguments, commandLine.schema, executionContext);
 				}
 			} else if ("import".equalsIgnoreCase(command)) {
 				if (commandLine.arguments.size() != 6) {
@@ -191,22 +223,17 @@ public class Jailer {
 						}
 					}
 				}
-			} else if ("print-datamodel".equalsIgnoreCase(command)) {
-				printDataModel(commandLine.arguments, commandLine.withClosures, executionContext);
 			} else if ("export".equalsIgnoreCase(command)) {
 				if (commandLine.arguments.size() != 6) {
 					CommandLineParser.printUsage(args);
 				} else {
 					pw = commandLine.arguments.get(5);
-					if (commandLine.maxNumberOfEntities > 0) {
-						EntityGraph.maxTotalRowcount = commandLine.maxNumberOfEntities;
-						_log.info("max-rowcount=" + EntityGraph.maxTotalRowcount);
-					}
-					
+
 					if (commandLine.exportScriptFileName == null) {
 						System.out.println("missing '-e' option");
 						CommandLineParser.printUsage(args);
 					} else {
+						updateDataModelFolder(commandLine, commandLine.arguments.get(1), executionContext);
 						if (!commandLine.independentWorkingTables) {
 							PrimaryKeyFactory.createUPKScope(commandLine.arguments.get(1), executionContext);
 						}
@@ -215,7 +242,7 @@ public class Jailer {
 								commandLine.arguments.get(4), commandLine.arguments.get(5), 0, jdbcJarURLs);
 						URL modelURL = new File(commandLine.arguments.get(1)).toURI().toURL();
 						new SubsettingEngine(executionContext).export(commandLine.where, modelURL, commandLine.exportScriptFileName, commandLine.deleteScriptFileName,
-								dataSource, dataSource.dbms, commandLine.explain, executionContext.getScriptFormat(), 0);
+								dataSource, dataSource.dbms, executionContext.getScriptFormat(), 0);
 					}
 				}
 			} else if ("delete".equalsIgnoreCase(command)) {
@@ -227,6 +254,7 @@ public class Jailer {
 						System.out.println("can't delete: missing '-d' option");
 						CommandLineParser.printUsage(args);
 					} else {
+						updateDataModelFolder(commandLine, commandLine.arguments.get(1), executionContext);
 						BasicDataSource dataSource = new BasicDataSource(commandLine.arguments.get(2), commandLine.arguments.get(3),
 								commandLine.arguments.get(4), commandLine.arguments.get(5), 0, jdbcJarURLs);
 						// note we are passing null for script format and the export script name, as we are using the export tool
@@ -236,14 +264,15 @@ public class Jailer {
 						}
 						URL modelURL = new File(commandLine.arguments.get(1)).toURI().toURL();
 						new SubsettingEngine(executionContext).export(commandLine.where, modelURL, /* clp.exportScriptFileName*/ null, commandLine.deleteScriptFileName,
-								dataSource, dataSource.dbms, commandLine.explain, /*scriptFormat*/ null, 0);
+								dataSource, dataSource.dbms, /*scriptFormat*/ null, 0);
 					}
 				}
-			} else if ("find-association".equalsIgnoreCase(command)) {
-				if (commandLine.arguments.size() < 3) {
+			} else if ("print-closure".equalsIgnoreCase(command)) {
+				if (commandLine.arguments.size() < 2) {
 					CommandLineParser.printUsage(args);
 				} else {
-					findAssociation(commandLine.arguments.get(1), commandLine.arguments.get(2), commandLine.arguments.subList(3, commandLine.arguments.size()), commandLine.undirected, executionContext);
+					updateDataModelFolder(commandLine, commandLine.arguments.get(1), executionContext);
+					printClosure(commandLine.arguments.get(1), commandLine.arguments.size() > 2? commandLine.arguments.get(2) : null, executionContext);
 				}
 			} else if ("create-ddl".equalsIgnoreCase(command)) {
 				String extractionModelFileName = null;
@@ -252,13 +281,7 @@ public class Jailer {
 				} else if (!commandLine.independentWorkingTables && commandLine.arguments.size() > 1) {
 					extractionModelFileName = commandLine.arguments.get(1);
 				}
-				if ("datamodel".equals(commandLine.datamodelFolder) && extractionModelFileName == null
-						||
-					!"datamodel".equals(commandLine.datamodelFolder) && extractionModelFileName != null) {
-					if (fromCli) {
-						throw new RuntimeException("Please specify either a data model (e.g., \"-datamodel datamodel/Demo-Scott\") or an extraction model (But not both)");
-					}
-				}
+				updateDataModelFolder(commandLine, extractionModelFileName, executionContext);
 				if (commandLine.arguments.size() >= 5) {
 					pw = commandLine.arguments.get(4);
 					if (!commandLine.independentWorkingTables && commandLine.arguments.size() > 5) {
@@ -287,7 +310,7 @@ public class Jailer {
 					CommandLineParser.printUsage(args);
 				} else {
 					pw = commandLine.arguments.get(4);
-					_log.info("Building data model.");
+					getLogger().info("Building data model.");
 					BasicDataSource dataSource = new BasicDataSource(commandLine.arguments.get(1), commandLine.arguments.get(2), commandLine.arguments.get(3), commandLine.arguments.get(4), 0, jdbcJarURLs);
 					ModelBuilder.build(dataSource, dataSource.dbms, commandLine.schema, warnings, executionContext);
 				}
@@ -296,7 +319,7 @@ public class Jailer {
 					CommandLineParser.printUsage(args);
 				} else {
 					pw = commandLine.arguments.get(4);
-					_log.info("Building data model.");
+					getLogger().info("Building data model.");
 					BasicDataSource dataSource = new BasicDataSource(commandLine.arguments.get(1), commandLine.arguments.get(2), commandLine.arguments.get(3), commandLine.arguments.get(4), 0, jdbcJarURLs);
 					ModelBuilder.buildAndMerge(dataSource, dataSource.dbms, commandLine.schema, warnings, executionContext);
 				}
@@ -305,26 +328,71 @@ public class Jailer {
 				return false;
 			}
 			return true;
-		} catch (Exception e) {
-			if (e instanceof CancellationException) {
-				_log.warn("cancelled");
-				throw e;
+		} catch (Throwable t) {
+			if (t instanceof CancellationException) {
+				getLogger().warn("cancelled");
+				throw t;
 			}
-			_log.error(e.getMessage(), e);
-			System.err.println("Error: " + e.getClass().getName() + ": " + e.getMessage());
+			getLogger().error(t.getMessage(), t);
+			System.err.println("Error: " + t.getClass().getName() + ": " + t.getMessage());
 			CommandLineParser.printAruments(System.err, args, pw);
 			String workingDirectory = System.getProperty("user.dir");
-			_log.error("working directory is " + workingDirectory);
-			throw e;
+			getLogger().error("working directory is " + workingDirectory);
+			throw t;
+		}
+	}
+
+	private static void printClosure(String extractionModelFileName, String separator, ExecutionContext executionContext) throws MalformedURLException, IOException {
+		ExtractionModel extractionModel = new ExtractionModel(new File(extractionModelFileName).toURI().toURL(), executionContext.getSourceSchemaMapping(), executionContext.getParameters(), executionContext, true);
+		Set<Table> subjects = new HashSet<Table>();
+		if (extractionModel.additionalSubjects != null) {
+			for (AdditionalSubject as: extractionModel.additionalSubjects) {
+				subjects.add(as.getSubject());
+			}
+		}
+		subjects.add(extractionModel.subject);
+
+		Set<String> closure = new TreeSet<String>();
+		Set<Table> toIgnore = new HashSet<Table>();
+		for (Table subject: subjects) {
+			for (Table table: subject.closure(toIgnore)) {
+				closure.add(Quoting.unquotedTableName(table, executionContext));
+				toIgnore.add(table);
+			}
+		}
+
+		int row = 0;
+		for (String tableName: closure) {
+			if (separator == null) {
+				System.out.println(tableName);
+			} else {
+				if (row++ > 0) {
+					System.out.print(separator);
+				}
+				System.out.print(tableName);
+			}
+		}
+		if (separator != null) {
+			System.out.println();
+		}
+	}
+
+	private static void updateDataModelFolder(CommandLine commandLine, String extractionModelFileName,
+			ExecutionContext executionContext) throws IOException {
+		if (extractionModelFileName != null && "datamodel".equals(commandLine.datamodelFolder)) {
+			String datamodelFolder = ExtractionModel.loadDatamodelFolder(extractionModelFileName, executionContext);
+			if (datamodelFolder != null) {
+				executionContext.setCurrentModelSubfolder(datamodelFolder);
+			}
 		}
 	}
 
 	/**
 	 * Render the data model.
-	 * 
+	 *
 	 * @param schema schema to analyze
 	 */
-	private static void renderDataModel(List<String> arguments, boolean withClosures, String schema, ExecutionContext executionContext) throws Exception {
+	private static void renderDataModel(List<String> arguments, String schema, ExecutionContext executionContext) throws Exception {
 		DataModel dataModel = new DataModel(executionContext);
 		for (String rm : arguments.subList(1, arguments.size())) {
 			if (dataModel.getRestrictionModel() == null) {
@@ -340,155 +408,4 @@ public class Jailer {
 		renderer.render(dataModel, arguments.subList(1, arguments.size()));
 	}
 
-	/**
-	 * Prints shortest association between two tables.
-	 */
-	private static void findAssociation(String from, String to, List<String> restModels, boolean undirected, ExecutionContext executionContext) throws Exception {
-		DataModel dataModel = new DataModel(executionContext);
-		for (String rm : restModels) {
-			if (dataModel.getRestrictionModel() == null) {
-				dataModel.setRestrictionModel(new RestrictionModel(dataModel, executionContext));
-			}
-			URL modelURL = new File(rm).toURI().toURL();
-			dataModel.getRestrictionModel().addRestrictionDefinition(modelURL, new HashMap<String, String>());
-		}
-		Table source = dataModel.getTable(from);
-		if (source == null) {
-			throw new RuntimeException("unknown table: '" + from);
-		}
-		Table destination = dataModel.getTable(to);
-		if (destination == null) {
-			throw new RuntimeException("unknown table: '" + to);
-		}
-
-		System.out.println();
-		System.out.println("Shortest path from " + source.getName() + " to " + destination.getName() + ":");
-
-		Map<Table, Table> successor = new HashMap<Table, Table>();
-		Map<Table, Association> outgoingAssociation = new HashMap<Table, Association>();
-		List<Table> agenda = new ArrayList<Table>();
-		agenda.add(destination);
-
-		while (!agenda.isEmpty()) {
-			Table table = agenda.remove(0);
-			for (Association association : incomingAssociations(table, undirected)) {
-				if (!successor.containsKey(association.source)) {
-					successor.put(association.source, table);
-					outgoingAssociation.put(association.source, association);
-					agenda.add(association.source);
-					if (association.source.equals(source)) {
-						agenda.clear();
-						break;
-					}
-				}
-			}
-		}
-		if (successor.containsKey(source)) {
-			String joinedSelect = "Select * From " + source.getName();
-			System.out.println("    " + source.getName());
-			for (Table table = source; !table.equals(destination); table = successor.get(table)) {
-				Association association = outgoingAssociation.get(table);
-				System.out.println("    " + association);
-				joinedSelect += " join "
-						+ association.destination.getName()
-						+ " on "
-						+ (association.reversed ? SqlUtil.replaceAliases(association.getJoinCondition(), association.destination.getName(), association.source
-								.getName()) : SqlUtil.replaceAliases(association.getJoinCondition(), association.source.getName(), association.destination
-								.getName()));
-			}
-			System.out.println();
-			System.out.println();
-			System.out.println("SQL query:");
-			System.out.println("    " + joinedSelect);
-		} else {
-			System.out.println("tables are not associated");
-		}
-	}
-
-	/**
-	 * Prints restricted data-model.
-	 */
-	private static void printDataModel(List<String> restrictionModels, boolean printClosures, ExecutionContext executionContext) throws Exception {
-		DataModel dataModel = new DataModel(executionContext);
-		if (printClosures) {
-			DataModel.printClosures = true;
-		}
-		for (String rm : restrictionModels.subList(1, restrictionModels.size())) {
-			if (dataModel.getRestrictionModel() == null) {
-				dataModel.setRestrictionModel(new RestrictionModel(dataModel, executionContext));
-			}
-			URL modelURL = new File(rm).toURI().toURL();
-			dataModel.getRestrictionModel().addRestrictionDefinition(modelURL, new HashMap<String, String>());
-		}
-
-		BufferedReader in = new BufferedReader(new StringReader(dataModel.toString()));
-		String line;
-		while ((line = in.readLine()) != null) {
-			System.out.println(line);
-			CancellationHandler.checkForCancellation(null);
-		}
-
-		printCycles(dataModel);
-		printComponents(dataModel, executionContext);
-	}
-
-	/**
-	 * Searches cycles in a data-model and prints out all tables involved in a
-	 * cycle.
-	 * 
-	 * @param dataModel
-	 *            the data-model
-	 */
-	private static void printCycles(DataModel dataModel) {
-		Set<Table> independentTables;
-		Set<Table> tables = new HashSet<Table>(dataModel.getTables());
-		do {
-			independentTables = dataModel.getIndependentTables(tables);
-			tables.removeAll(independentTables);
-		} while (!independentTables.isEmpty());
-		if (tables.isEmpty()) {
-			System.out.println("no cyclic dependencies" + SubsettingEngine.asString(tables));
-		} else {
-			System.out.println("tables in dependent-cycle: " + SubsettingEngine.asString(tables));
-		}
-	}
-
-	/**
-	 * Searches components in a data-model and prints out all components.
-	 * 
-	 * @param dataModel
-	 *            the data-model
-	 */
-	private static void printComponents(DataModel dataModel, ExecutionContext executionContext) {
-		List<Set<Table>> components = new ArrayList<Set<Table>>();
-		Set<Table> tables = new HashSet<Table>(dataModel.getTables());
-		while (!tables.isEmpty()) {
-			Table table = tables.iterator().next();
-			Set<Table> closure = table.closure(new HashSet<Table>(), false);
-			components.add(closure);
-			tables.removeAll(closure);
-		}
-		System.out.println(components.size() + " components: ");
-		for (Set<Table> component : components) {
-			System.out.println(new PrintUtil().tableSetAsString(component));
-		}
-	}
-
-	/**
-	 * Collects all non-ignored associations with a given table as destination.
-	 * 
-	 * @param table
-	 *            the table
-	 * @return all non-ignored associations with table as destination
-	 */
-	private static Collection<Association> incomingAssociations(Table table, boolean undirected) {
-		Collection<Association> result = new ArrayList<Association>();
-		for (Association association : table.associations) {
-			if (association.reversalAssociation.getJoinCondition() != null || (undirected && association.getJoinCondition() != null)) {
-				result.add(association.reversalAssociation);
-			}
-		}
-		return result;
-	}
-
 }

src/main/engine/net/sf/jailer/JailerVersion.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@ package net.sf.jailer;
 
 /**
  * The Jailer Version.
- * 
+ *
  * @author Ralf Wisser
  */
 public class JailerVersion {
@@ -25,7 +25,7 @@ public class JailerVersion {
 	/**
 	 * The Jailer version.
 	 */
-	public static final String VERSION = "9.4";
+	public static final String VERSION = "10.3.4";
 
 	/**
 	 * The Jailer working tables version.

src/main/engine/net/sf/jailer/api/Importer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/api/Subsetter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -38,7 +38,7 @@ import net.sf.jailer.subsetting.SubsettingEngine;
 
 /**
  * Generates a subset of a relational database that respects foreign key constraints.
- * 
+ *
  * @author Ralf Wisser
  */
 public class Subsetter {
@@ -63,7 +63,7 @@ public class Subsetter {
 
 	/**
 	 * Creates a new Subsetter with all mandatory attributes.
-	 * 
+	 *
 	 * @param dataSource the data-source to connect with the source database
 	 * @param dbms the DBMS of the source database
 	 * @param dataModel URL of the current data model (the datamodel's base folder)
@@ -87,7 +87,7 @@ public class Subsetter {
 
 	/**
 	 * Creates a new Subsetter with all mandatory attributes.
-	 * 
+	 *
 	 * @param dataSource the data-source to connect with the source database
 	 * @param dbms the DBMS of the source database
 	 * @param dataModel the current data model (the datamodel's base folder)
@@ -112,14 +112,14 @@ public class Subsetter {
 		}
 		setScriptFormat(scriptFormat);
 	}
-	
+
 	/**
 	 * Generates the export-script.
-	 * @param whereClause if not <code>null</code>, overrides the extraction model's subject condition 
+	 * @param whereClause if not <code>null</code>, overrides the extraction model's subject condition
 	 * @param exportScriptFile the export-script file (compressed if it ends with '.zip' or '.gz')
-	 * 
+	 *
 	 * @return export statistic
-	 * 
+	 *
 	 * @throws InconsistentSubsettingResultException if {@link ExecutionContext#isAbortInCaseOfInconsistency()} and the number of exported rows differs from that of the collected ones
 	 */
 	public ExportStatistic execute(String whereClause, File exportScriptFile) throws SQLException, IOException {
@@ -128,13 +128,13 @@ public class Subsetter {
 
 	/**
 	 * Generates the export- and/or delete-script.
-	 * 
-	 * @param whereClause if not <code>null</code>, overrides the extraction model's subject condition 
+	 *
+	 * @param whereClause if not <code>null</code>, overrides the extraction model's subject condition
 	 * @param exportScriptFile the export-script file (compressed if it ends with '.zip' or '.gz'), optional
 	 * @param deleteScriptFile the delete-script file (compressed if it ends with '.zip' or '.gz'), optional
-	 * 
+	 *
 	 * @return export statistic
-	 * 
+	 *
 	 * @throws InconsistentSubsettingResultException if {@link ExecutionContext#isAbortInCaseOfInconsistency()} and the number of exported rows differs from that of the collected ones
 	 */
 	public ExportStatistic execute(String whereClause, File exportScriptFile, File deleteScriptFile) throws SQLException, IOException {
@@ -161,10 +161,10 @@ public class Subsetter {
 			}
 			return new SubsettingEngine(executionContext).export(
 					whereClause,
-					getExtractionModelURL(), 
+					getExtractionModelURL(),
 					exportScriptFile == null? null : exportScriptFile.getAbsolutePath(),
 					deleteScriptFile == null? null : deleteScriptFile.getAbsolutePath(),
-					getDataSource(), sourceDBMS, false, getScriptFormat(), getModelPoolSize());
+					getDataSource(), sourceDBMS, getScriptFormat(), getModelPoolSize());
 		} catch (SAXException e) {
 			throw new RuntimeException(e);
 		}
@@ -172,7 +172,7 @@ public class Subsetter {
 
 	/**
 	 * Gets the data-source to connect with the database.
-	 * 
+	 *
 	 * @return the data-source to connect with the database
 	 */
 	public DataSource getDataSource() {
@@ -181,7 +181,7 @@ public class Subsetter {
 
 	/**
 	 * Sets the data-source to connect with the database.
-	 * 
+	 *
 	 * @param dataSource the data-source to connect with the database
 	 */
 	public void setDataSource(DataSource dataSource) {
@@ -190,7 +190,7 @@ public class Subsetter {
 
 	/**
 	 * Gets the DBMS of the database.
-	 * 
+	 *
 	 * @return the DBMS
 	 */
 	public DBMS getDbms() {
@@ -200,7 +200,7 @@ public class Subsetter {
 	/**
 	 * Sets the DBMS of the database. <br>
 	 * Note that it's not necessary to set the DBMS if {@link BasicDataSource} is used.
-	 * 
+	 *
 	 * @param dbms the DBMS
 	 */
 	public void setDbms(DBMS dbms) {
@@ -213,7 +213,7 @@ public class Subsetter {
 	public URL getDataModelURL() {
 		return executionContext.getDataModelURL();
 	}
-	
+
 	/**
 	 * Sets URL of the current data model (the datamodel's base folder)
 	 */
@@ -223,9 +223,9 @@ public class Subsetter {
 
 	/**
 	 * Sets the current data model as {@link File}
-	 * 
+	 *
 	 * @param datamodelBaseFolder represents the folder. Will be converted to an URL an set as datamodel URL
-	 * 
+	 *
 	 * @see #setDataModelURL(URL)
 	 */
 	public void setDataModelBaseFolder(File datamodelBaseFolder) {
@@ -238,7 +238,7 @@ public class Subsetter {
 
 	/**
 	 * Gets the URL of the extraction model.
-	 * 
+	 *
 	 * @return the URL of the extraction model
 	 */
 	public URL getExtractionModelURL() {
@@ -247,7 +247,7 @@ public class Subsetter {
 
 	/**
 	 * Sets the URL of the extraction model.
-	 * 
+	 *
 	 * @param extractionModelURL the URL of the extraction model
 	 */
 	public void setExtractionModelURL(URL extractionModelURL) {
@@ -256,16 +256,16 @@ public class Subsetter {
 
 	/**
 	 * Gets the script format.
-	 * 
+	 *
 	 * @return the script format
 	 */
 	public ScriptFormat getScriptFormat() {
 		return executionContext.getScriptFormat();
 	}
-	
+
 	/**
 	 * Sets the script format.
-	 * 
+	 *
 	 * @return the script format
 	 */
 	public void setScriptFormat(ScriptFormat scriptFormat) {
@@ -425,7 +425,7 @@ public class Subsetter {
 
 	/**
 	 * Gets IsolationLevel.
-	 * 
+	 *
 	 * @see Connection#setTransactionIsolation(int)
 	 */
 	public Integer getIsolationLevel() {
@@ -434,7 +434,7 @@ public class Subsetter {
 
 	/**
 	 * Sets IsolationLevel.
-	 * 
+	 *
 	 * @see Connection#setTransactionIsolation(int)
 	 */
 	public void setIsolationLevel(Integer isolationLevel) {
@@ -551,26 +551,63 @@ public class Subsetter {
 	}
 
 	/**
-	 * If <code>true</code>, Use primary keys to determine row identity (instead
-	 * of rowid-column)
-	 *
-	 * @return <code>true</code> if Use primary keys to determine row identity
-	 *         (instead of rowid-column)
+	 * @deprecated use {@link #getUseRowid()}
 	 */
 	public boolean getNoRowid() {
-		return executionContext.getNoRowid();
+		return !getUseRowid();
 	}
 
 	/**
-	 * If <code>true</code>, Use primary keys to determine row identity (instead
-	 * of rowid-column)
-	 *
-	 * @param noRowid
-	 *            <code>true</code> if Use primary keys to determine row
-	 *            identity (instead of rowid-column)
+	 * @deprecated use {@link #setUseRowid(boolean)}
 	 */
 	public void setNoRowid(boolean noRowid) {
-		executionContext.setNoRowid(noRowid);
+		setUseRowid(!noRowid);
+	}
+
+	/**
+	 * If <code>true</code>, use rowid/ctid-column to determine row identity (instead
+	 * of primary keys)
+	 *
+	 * @return if <code>true</code> use rowid/ctid-column to determine row identity
+	 *         (instead of primary keys)
+	 */
+	public boolean getUseRowid() {
+		return executionContext.getUseRowid();
+	}
+
+	/**
+	 * If <code>true</code>, use rowid/ctid-column to determine row identity (instead
+	 * of primary keys)
+	 *
+	 * @param useRowid
+	 *            if <code>true</code> use rowid/ctid-column to determine row
+	 *            identity (instead of primary keys)
+	 */
+	public void setUseRowid(boolean useRowid) {
+		executionContext.setUseRowid(useRowid);
+	}
+
+	/**
+	 * If <code>true</code>, use rowid/ctid-column only for tables without primary key.
+	 *
+	 * @return <code>true</code> if use rowid/ctid-column only for tables without primary key
+	 *
+	 * @see #getUseRowIdsOnlyForTablesWithoutPK()
+	 */
+	public boolean getUseRowIdsOnlyForTablesWithoutPK() {
+		return executionContext.getUseRowIdsOnlyForTablesWithoutPK();
+	}
+
+	/**
+	 * If <code>true</code>, use rowid/ctid-column only for tables without primary key.
+	 *
+	 * @param useRowIdsOnlyForTablesWithoutPK
+	 *            <code>true</code> if use rowid/ctid-column only for tables without primary key
+	 *
+	 * @see #setUseRowid(boolean)
+	 */
+	public void setUseRowIdsOnlyForTablesWithoutPK(boolean useRowIdsOnlyForTablesWithoutPK) {
+		executionContext.setUseRowIdsOnlyForTablesWithoutPK(useRowIdsOnlyForTablesWithoutPK);;
 	}
 
 	/**
@@ -595,7 +632,7 @@ public class Subsetter {
 
 	/**
 	 * Gets parameters.
-	 * 
+	 *
 	 * @return parameters
 	 */
 	public Map<String, String> getParameters() {
@@ -604,7 +641,7 @@ public class Subsetter {
 
 	/**
 	 * Sets a parameter.
-	 * 
+	 *
 	 * @param name parameter name
 	 * @param value value
 	 */
@@ -614,7 +651,7 @@ public class Subsetter {
 
 	/**
 	 * Gets schema map for destination database.
-	 * 
+	 *
 	 * @return schema map
 	 */
 	public Map<String, String> getSchemaMapping() {
@@ -623,7 +660,7 @@ public class Subsetter {
 
 	/**
 	 * Sets schema map for destination database.
-	 * 
+	 *
 	 * @param schemaMapping schema map
 	 */
 	public void setSchemaMapping(Map<String, String> schemaMapping) {
@@ -632,7 +669,7 @@ public class Subsetter {
 
 	/**
 	 * Sets schema map for source database.
-	 * 
+	 *
 	 * @param sourceSchemaMapping the sourceSchemaMapping to set
 	 */
 	public void setSourceSchemaMapping(Map<String, String> sourceSchemaMapping) {
@@ -641,7 +678,7 @@ public class Subsetter {
 
 	/**
 	 * Gets schema map for source database.
-	 * 
+	 *
 	 * @return the sourceSchemaMapping to set
 	 */
 	public Map<String, String> getSourceSchemaMapping() {
@@ -649,9 +686,7 @@ public class Subsetter {
 	}
 
 	/**
-	 * If <code>true</code>, the Subsetter throws an 
-	 * 
-	 * @return if <code>true</code>, abort the process if the result is inconsistent due to insufficient transaction isolation
+	 * @return if <code>true</code>, the Subsetter throws an exception if the result is inconsistent due to insufficient transaction isolation
 	 */
 	public boolean isAbortInCaseOfInconsistency() {
 		return executionContext.isAbortInCaseOfInconsistency();
@@ -681,7 +716,7 @@ public class Subsetter {
 	/**
 	 * Gets the {@link ExecutionContext}. <br>
 	 * Use this to set parameters that are not accessible via this facade.
-	 * 
+	 *
 	 * @return the {@link ExecutionContext}
 	 */
 	public ExecutionContext getExecutionContext() {
@@ -689,7 +724,7 @@ public class Subsetter {
 	}
 
 	private final ExecutionContext executionContext;
-	
+
 	private int modelPoolSize = 10;
 	private URL extractionModelURL;
 	private DataSource dataSource;

src/main/engine/net/sf/jailer/api_example/APIExample.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/configuration/Configuration.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -153,6 +153,10 @@ public class Configuration {
 
 	private int columnsPerIFMTable = 8;
 
+	/**
+	 * Time in seconds waiting for an idle database connection to be tested if it is still valid.
+	 */
+	private int databaseConnectionInteractiveTimeout = 240;
 
 	private String additionalSQLKeywords;
 
@@ -180,6 +184,20 @@ public class Configuration {
 		return doMinimizeUPK;
 	}
 	
+	/**
+	 * Gets time in seconds waiting for an idle database connection to be tested if it is still valid.
+	 */
+	public int getDatabaseConnectionInteractiveTimeout() {
+		return databaseConnectionInteractiveTimeout;
+	}
+
+	/**
+	 * Sets time in seconds waiting for an idle database connection to be tested if it is still valid.
+	 */
+	public void setDatabaseConnectionInteractiveTimeout(int databaseConnectionInteractiveTimeout) {
+		this.databaseConnectionInteractiveTimeout = databaseConnectionInteractiveTimeout;
+	}
+
 	/**
 	 * The configuration.
 	 */

src/main/engine/net/sf/jailer/configuration/DBMS.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Pattern;
 
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlTransient;
@@ -33,7 +34,7 @@ import net.sf.jailer.database.SqlScriptBasedStatisticRenovator;
 
 /**
  * Describes a specific DBMS.
- * 
+ *
  * @author Ralf Wisser
  */
 public class DBMS {
@@ -56,7 +57,7 @@ public class DBMS {
 
 	/**
 	 * Gets all DBMSes.
-	 * 
+	 *
 	 * @return array of all DBMSes
 	 */
 	public static DBMS[] values() {
@@ -68,7 +69,7 @@ public class DBMS {
 	 */
 	public DBMS() {
 	}
-	
+
 	/**
 	 * Copy constructor.
 	 */
@@ -135,7 +136,7 @@ public class DBMS {
 
 	/**
 	 * Gets DBMS specific configuration.
-	 * 
+	 *
 	 * @param dbmsId the DBMS id
 	 * @return the DBMS with given id, or the default DBMS if id is <code>null</code>
 	 */
@@ -146,7 +147,7 @@ public class DBMS {
 		if (perDBMS.containsKey(dbmsId)) {
 			return perDBMS.get(dbmsId);
 		}
-		List<DBMS> cs = Configuration.getInstance().getDBMS();  
+		List<DBMS> cs = Configuration.getInstance().getDBMS();
 		for (DBMS c: cs) {
 			if (dbmsId.equals(c.getId())) {
 				perDBMS.put(dbmsId, c);
@@ -155,7 +156,7 @@ public class DBMS {
 		}
 		throw new RuntimeException("Unknown DBMS: \"" + dbmsId + "\"");
 	}
-	
+
 	/**
 	 * Holds configurations.
 	 */
@@ -182,50 +183,52 @@ public class DBMS {
 		FIREBIRD = forDBMS("FIREBIRD");
 		DERBY = forDBMS("DERBY");
 	}
-	
+
 	private String id;
 	private String familyId;
 	private String displayName;
-	
+
 	/**
 	 * DB-URL pattern of DBMS for which this holds the configuration.
 	 */
 	private String urlPattern;
-	
+
 	/**
 	 * Test-query for the DBMS for which this holds the configuration.
 	 */
 	private String testQuery;
-	
+
 	/**
 	 * The {@link SqlScriptBasedStatisticRenovator}.
 	 */
 	private SqlScriptBasedStatisticRenovator statisticRenovator;
-	
+
 	/**
 	 * Replacement map for column types used for DDL generation.
 	 */
 	private Map<String, String> typeReplacement = new HashMap<String, String>();
-	
+
+	private Map<String, String> sqlExpressionRule = new HashMap<String, String>();
+
 	/**
 	 * Replacement map for column types used for DDL generation. Only used if DBMS accept it.
 	 */
 	private Map<String, String> experimentalTypeReplacement = new HashMap<String, String>();
-	
+
 	/**
 	 * Replacement map for special characters in string literals.
 	 */
 	private Map<String, String> stringLiteralEscapeSequences;
-	
+
 	/**
 	 * Suffix of SQL-Select statement to limit number of rows.
 	 */
 	private String sqlLimitSuffix;
-	
+
 	private Integer varcharLengthLimit = null;
 
 	private String tableProperties = "";
-	
+
 	/**
 	 * DB-Query to get DDL of a table
 	 */
@@ -268,7 +271,7 @@ public class DBMS {
 	public void setDdlCall(String ddlCall) {
 		this.ddlCall = ddlCall;
 	}
-	
+
 	/**
 	 * Maps characters to escape sequences according to {@link #stringLiteralEscapeSequences}.
 	 */
@@ -276,17 +279,17 @@ public class DBMS {
 	{ charToEscapeSequence.put('\'', "''"); }
 	private char[] keysOfCharToEscapeSequence = new char[] { '\'' };
 	private String ncharPrefix = null;
-	
+
 	/**
 	 * Set of type names for which no data must be exported.
 	 */
 	private Set<String> exportBlocks = new HashSet<String>();
-	
+
 	/**
 	 * <code>true</code> if DBMS supports identity-type (MS-SQL)
 	 */
 	private boolean identityInserts = false;
-	
+
 	private String emptyCLOBValue = null;
 	private String emptyNCLOBValue = null;
 	private String emptyBLOBValue = null;
@@ -322,13 +325,13 @@ public class DBMS {
 	private String explainPrepare = null;
 	private String explainQuery = null;
 	private String explainCleanup = null;
-	
+
 	private String functionSourceQuery;
 	private String procedureSourceQuery;
 	private String packageSourceQuery;
 	private String packageNamesQuery;
 	private String defaultSchemaQuery;
-	
+
 	private Integer fetchSize = null;
 
 	private List<DatabaseObjectRenderingDescription> objectRenderers = new ArrayList<DatabaseObjectRenderingDescription>();
@@ -336,6 +339,67 @@ public class DBMS {
 
 	private LimitTransactionSizeInfo limitTransactionSize = new LimitTransactionSizeInfo();
 
+	private String clobTypesRE;
+	private String nClobTypesRE;
+	private String blobTypesRE;
+	private Pattern clobTypesPattern;
+	private Pattern nClobTypesPattern;
+	private Pattern blobTypesPattern;
+
+	public boolean isClobType(String typeWithLength) {
+		if (clobTypesRE == null) {
+			return false;
+		}
+		if (clobTypesPattern == null) {
+			clobTypesPattern = Pattern.compile(clobTypesRE, Pattern.CASE_INSENSITIVE);
+		}
+		return clobTypesPattern.matcher(typeWithLength).matches();
+	}
+
+	public boolean isNClobType(String typeWithLength) {
+		if (nClobTypesRE == null) {
+			return false;
+		}
+		if (nClobTypesPattern == null) {
+			nClobTypesPattern = Pattern.compile(nClobTypesRE, Pattern.CASE_INSENSITIVE);
+		}
+		return nClobTypesPattern.matcher(typeWithLength).matches();
+	}
+
+	public boolean isBlobType(String typeWithLength) {
+		if (blobTypesRE == null) {
+			return false;
+		}
+		if (blobTypesPattern == null) {
+			blobTypesPattern = Pattern.compile(blobTypesRE, Pattern.CASE_INSENSITIVE);
+		}
+		return blobTypesPattern.matcher(typeWithLength).matches();
+	}
+
+	public String getClobTypesRE() {
+		return clobTypesRE;
+	}
+
+	public void setClobTypesRE(String clobTypesRE) {
+		this.clobTypesRE = clobTypesRE;
+	}
+
+	public String getnClobTypesRE() {
+		return nClobTypesRE;
+	}
+
+	public void setnClobTypesRE(String nClobTypesRE) {
+		this.nClobTypesRE = nClobTypesRE;
+	}
+
+	public String getBlobTypesRE() {
+		return blobTypesRE;
+	}
+
+	public void setBlobTypesRE(String blobTypesRE) {
+		this.blobTypesRE = blobTypesRE;
+	}
+
 	/**
 	 * @return the virtualColumnsQuery
 	 */
@@ -422,7 +486,7 @@ public class DBMS {
 	}
 
 	private String rowidType = null;
-	
+
 	/**
 	 * @return the sqlDialect
 	 */
@@ -441,7 +505,7 @@ public class DBMS {
 	 * Manages session local temporary tables.
 	 */
 	private DefaultTemporaryTableManager sessionTemporaryTableManager = null;
-	
+
 	/**
 	 * Manages transaction local temporary tables.
 	 */
@@ -469,23 +533,23 @@ public class DBMS {
 	public Set<String> getExportBlocks() {
 		return exportBlocks;
 	}
-	
+
 	public void setExportBlocks(Set<String> exportBlocks) {
 		this.exportBlocks = exportBlocks;
 	}
-	
+
 	/**
 	 * Gets the {@link SqlScriptBasedStatisticRenovator}.
-	 * 
+	 *
 	 * @return the {@link SqlScriptBasedStatisticRenovator}
 	 */
 	public SqlScriptBasedStatisticRenovator getStatisticRenovator() {
 		return statisticRenovator;
 	}
-	
+
 	/**
 	 * Sets the {@link SqlScriptBasedStatisticRenovator}.
-	 * 
+	 *
 	 * @param statisticRenovator the {@link SqlScriptBasedStatisticRenovator}
 	 */
 	public void setStatisticRenovator(SqlScriptBasedStatisticRenovator statisticRenovator) {
@@ -503,7 +567,7 @@ public class DBMS {
 	public void setBinaryPattern(String binaryPattern) {
 		this.binaryPattern = binaryPattern;
 	}
-	
+
 	/**
 	 * Sets replacement map for column types used for DDL generation.
 	 */
@@ -576,7 +640,7 @@ public class DBMS {
 	public void setSessionTemporaryTableManager(DefaultTemporaryTableManager tableManager) {
 		sessionTemporaryTableManager = tableManager;
 	}
-	
+
 	/**
 	 * Sets manager for transaction local temporary tables.
 	 */
@@ -638,14 +702,25 @@ public class DBMS {
 	public Map<String, String> getStringLiteralEscapeSequences() {
 		return stringLiteralEscapeSequences;
 	}
-	
+
 	/**
 	 * Converts a string to a string literal according to the {@link #getStringLiteralEscapeSequences()}.
-	 * 
+	 *
 	 * @param string the string to convert
 	 * @return the string literal
 	 */
 	public String convertToStringLiteral(String string) {
+		return convertToStringLiteral(string, null);
+	}
+	
+	/**
+	 * Converts a string to a string literal according to the {@link #getStringLiteralEscapeSequences()}.
+	 *
+	 * @param string the string to convert
+	 * @param prefix literal prefix (optional)
+	 * @return the string literal
+	 */
+	public String convertToStringLiteral(String string, String prefix) {
 		boolean esc = false;
 		for (char c: keysOfCharToEscapeSequence) {
 			if (string.indexOf(c) >= 0) {
@@ -654,16 +729,22 @@ public class DBMS {
 			}
 		}
 		if (!esc) {
+			if (prefix != null) {
+				return prefix + string;
+			}
 			return string;
 		}
-		
+
 		StringBuilder qvalue = new StringBuilder();
 		int l = string.length();
-		
+
 		for (int i = 0; i < l; ++i) {
 			char c = string.charAt(i);
 			String es = charToEscapeSequence.get(c);
 			if (es != null) {
+				if (prefix != null && es.startsWith("'") && es.endsWith("'") && es.length() > 2) {
+					es = es.substring(0, es.length() - 1) + prefix + "'";
+				}
 				qvalue.append(es);
 			} else {
 				qvalue.append(c);
@@ -693,7 +774,7 @@ public class DBMS {
 	public String getSqlLimitSuffix() {
 		return sqlLimitSuffix;
 	}
-	
+
 	public Integer getVarcharLengthLimit() {
 		return varcharLengthLimit;
 	}
@@ -824,10 +905,10 @@ public class DBMS {
 	public void setTableProperties(String tableProperties) {
 		this.tableProperties = tableProperties;
 	}
-	
+
 	/**
 	 * Gets the JDBC properties.
-	 * 
+	 *
 	 * @return the jdbcProperties
 	 */
 	public Map<String, String> getJdbcProperties() {
@@ -836,7 +917,7 @@ public class DBMS {
 
 	/**
 	 * Sets the JDBC properties.
-	 * 
+	 *
 	 * @param jdbcProperties the jdbcProperties to set
 	 */
 	public void setJdbcProperties(Map<String, String> jdbcProperties) {
@@ -856,7 +937,7 @@ public class DBMS {
 	public void setIdentifierQuoteString(String identifierQuoteString) {
 		this.identifierQuoteString = identifierQuoteString;
 	}
-	
+
 	public String getTestQuery() {
 		return testQuery;
 	}
@@ -1186,7 +1267,7 @@ public class DBMS {
 
 	/**
 	 * Gets fetch size.
-	 * 
+	 *
 	 * @return fetch size
 	 */
 	public Integer getFetchSize() {
@@ -1195,13 +1276,21 @@ public class DBMS {
 
 	/**
 	 * Sets fetch size.
-	 * 
+	 *
 	 * @param fetchSize fetch size
 	 */
 	public void setFetchSize(Integer fetchSize) {
 		this.fetchSize = fetchSize;
 	}
 
+	public Map<String, String> getSqlExpressionRule() {
+		return sqlExpressionRule;
+	}
+
+	public void setSqlExpressionRule(Map<String, String> sqlExpressionRule) {
+		this.sqlExpressionRule = sqlExpressionRule;
+	}
+
 	/* (non-Javadoc)
 	 * @see java.lang.Object#hashCode()
 	 */

src/main/engine/net/sf/jailer/configuration/DatabaseObjectRenderingDescription.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/configuration/LimitTransactionSizeInfo.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/configuration/LocalDatabaseConfiguration.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/configuration/jailer.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <configuration>
-    
+
     <!-- Configuration of the local database -->
     <localDatabase>
         <databaseFolder>local</databaseFolder>
@@ -13,17 +13,17 @@
         <user></user>
         <password></password>
     </localDatabase>
-    
+
     <!--
     nullColumnPlaceholder:
-    
+
 	For DBUnit flat XML files, null columns can carry a placeholder value
 	which can later be replaced using a ReplacementDataSet.
 	In Flat XML files the first row of a table defines the metadata. If a null
 	column is omitted, none of the following rows can have this column! This is the
 	default behaviour of Jailer.
 	Thus, define a null placeholder and replace it with a null value when you load
-	the data set with DBUnit.     
+	the data set with DBUnit.
     -->
 	<!--
 	<nullColumnPlaceholder>[NULL]</nullColumnPlaceholder>
@@ -31,19 +31,19 @@
 
 	<!--  number of columns per import-filter-mapping-table -->
 	<columnsPerIFMTable>8</columnsPerIFMTable>
-    
+
     <!--
 		If minimize-UPK is true, the UPK don't preserve order. This minimizes
 		the size of the UPK.
 	-->
 	<doMinimizeUPK>false</doMinimizeUPK>
 
-    <!-- HTML renderer -->	
+    <!-- HTML renderer -->
     <renderer>
         <maxDepth>0</maxDepth>
         <outputFolder>render</outputFolder>
     </renderer>
-    
+
     <!-- additional SQL keywords -->
 	<additionalSQLKeywords>sensitive, datetime</additionalSQLKeywords>
 
@@ -60,8 +60,11 @@
 	</urlRewriteRule>
 	-->
 
+	<!-- time in seconds waiting for an idle database connection to be tested if it is still valid -->
+	<databaseConnectionInteractiveTimeout>240</databaseConnectionInteractiveTimeout>
+
 	<!--
-		Also include 'null's in the generated upsert statements. 
+		Also include 'null's in the generated upsert statements.
 		Useful if a column contains 'null', but its default value is non-null.
 	<generateUpsertStatementsWithoutNulls>false</generateUpsertStatementsWithoutNulls>
 	-->
@@ -108,6 +111,7 @@
             <createTableSuffix>ON COMMIT PRESERVE ROWS</createTableSuffix>
             <dropTablePrefix>DROP TABLE </dropTablePrefix>
             <indexTablePrefix></indexTablePrefix>
+            <needsExclusiveAccess>true</needsExclusiveAccess>
         </sessionTemporaryTableManager>
         <identityInserts>false</identityInserts>
         <datePattern>'to_date('''yyyy-MM-dd''', ''YYYY-MM-DD'')'</datePattern>
@@ -120,10 +124,10 @@
         <toNClob>to_nclob('%s')</toNClob>
         <embeddedLobSizeLimit>3980</embeddedLobSizeLimit>
         <binaryPattern>hextoraw('%s')</binaryPattern>
-        
+
         <!--  <avoidLeftJoin>true</avoidLeftJoin> -->
         <avoidLeftJoin>false</avoidLeftJoin>
-        
+
         <limitTransactionSize>
         	<limit>30000</limit>
         	<additionalWhereCondition>rownum &lt;= %1$s</additionalWhereCondition>
@@ -179,15 +183,15 @@ union all
 		(
 		Select i.INDEX_NAME CNAME, i.TABLE_NAME TNAME, c.COLUMN_NAME COLNAME, c.COLUMN_POSITION POS
 		from USER_INDEXES i, USER_IND_COLUMNS c
-		where 
+		where
 		  i.TABLE_OWNER = '%1$s'
 		  and i.UNIQUENESS = 'UNIQUE'
 		  and i.TABLE_NAME not like 'BIN$%%'
 		  and i.TABLE_NAME = c.TABLE_NAME
 		  and i.INDEX_NAME = c.INDEX_NAME
 		  and i.INDEX_NAME not in
-		  	(Select CONSTRAINT_NAME 
-		  	from user_constraints 
+		  	(Select CONSTRAINT_NAME
+		  	from user_constraints
 		  	where CONSTRAINT_TYPE = 'P'
 			and STATUS = 'ENABLED'
 			and TABLE_NAME not like 'BIN$%%'
@@ -230,7 +234,7 @@ union all
     <dbms>
         <id>MSSQL</id>
         <displayName>MS SQL Server</displayName>
-        <urlPattern>jdbc:sqlserver.*|jdbc:inetdae.*|jdbc:JTurbo.*|jdbc:weblogic:mssqlserver.*|jdbc:jtds:sqlserver.*</urlPattern>
+        <urlPattern>jdbc:sqlserver.*|jdbc:inetdae.*|jdbc:JTurbo.*|jdbc:weblogic:mssqlserver.*|jdbc:jtds:sqlserver.*|jdbc:JSQLConnect.*</urlPattern>
         <sqlDialect>
             <needsValuesKeywordForDeletes>false</needsValuesKeywordForDeletes>
             <supportsInClauseForDeletes>false</supportsInClauseForDeletes>
@@ -256,7 +260,7 @@ union all
             </entry>
         </stringLiteralEscapeSequences>
         <sqlLimitSuffix>TOP %s</sqlLimitSuffix>
-        
+
         <!-- NOLOGGING, TABLESPACE-spec., etc. -->
         <tableProperties></tableProperties>
 
@@ -269,8 +273,9 @@ union all
 			<createIndexSuffix></createIndexSuffix>
 			<indexTablePrefix></indexTablePrefix>
 			<dropTablePrefix>DROP TABLE </dropTablePrefix>
+            <needsExclusiveAccess>false</needsExclusiveAccess>
 		</sessionTemporaryTableManager>
-        
+
         <ncharPrefix>N</ncharPrefix>
         <exportBlocks>timestamp</exportBlocks>
         <exportBlocks>TIMESTAMP</exportBlocks>
@@ -284,6 +289,9 @@ union all
                 <value>DATETIME</value>
             </entry>
         </typeReplacement>
+        <clobTypesRE>varchar\(max\)|text\(.*\)</clobTypesRE>
+        <nClobTypesRE>nvarchar\(max\)|xml\(.*\)|ntext\(.*\)</nClobTypesRE>
+        <blobTypesRE>varbinary\(max\)|image\(.*\)</blobTypesRE>
         <identityInserts>true</identityInserts>
         <timestampPattern>'convert(datetime, '''yyyy-MM-dd'T'HH:mm:ss.SSS''', 126)'</timestampPattern>
         <timestampWithNanoTypeName>datetime2</timestampWithNanoTypeName>
@@ -294,7 +302,7 @@ union all
         <toBlob>0x%s</toBlob>
         <toClob>'%s'</toClob>
         <toNClob>N'%s'</toNClob>
-        <embeddedLobSizeLimit>32000</embeddedLobSizeLimit>
+        <embeddedLobSizeLimit>7980</embeddedLobSizeLimit>
         <binaryPattern>0x%s</binaryPattern>
         <avoidLeftJoin>false</avoidLeftJoin>
         <supportsSchemasInIndexDefinitions>false</supportsSchemasInIndexDefinitions>
@@ -315,7 +323,7 @@ WHERE SCHEMA_NAME(tbl.schema_id)='%s'
 		<viewTextOrDDLQuery>SELECT VIEW_DEFINITION FROM INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA = '%1$s' and TABLE_NAME = '%2$s'</viewTextOrDDLQuery>
         <virtualColumnsQuery>SELECT sysobjects.name AS TableName, syscolumns.name AS ColumnName FROM syscolumns JOIN sysobjects ON syscolumns.id = sysobjects.id AND sysobjects.xtype = 'U' WHERE syscolumns.iscomputed = 1</virtualColumnsQuery>
 		<userDefinedColumnsQuery>SELECT distinct DOMAIN_NAME from INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND DOMAIN_NAME is not null</userDefinedColumnsQuery>
-<!-- 
+<!--
 		<importedKeysQuery>SELECT null, PKCU.TABLE_SCHEMA, PKCU.TABLE_NAME, PKCU.COLUMN_NAME, null, KCU.TABLE_SCHEMA, KCU.TABLE_NAME, KCU.COLUMN_NAME, KCU.ORDINAL_POSITION, null, null, RC.CONSTRAINT_NAME, RC.UNIQUE_CONSTRAINT_NAME, null FROM INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS RC JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE KCU ON KCU.CONSTRAINT_CATALOG = RC.CONSTRAINT_CATALOG AND KCU.CONSTRAINT_SCHEMA = RC.CONSTRAINT_SCHEMA AND KCU.CONSTRAINT_NAME = RC.CONSTRAINT_NAME AND KCU.TABLE_CATALOG = RC.CONSTRAINT_CATALOG JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE PKCU ON PKCU.CONSTRAINT_CATALOG = RC.UNIQUE_CONSTRAINT_CATALOG AND PKCU.CONSTRAINT_SCHEMA = RC.UNIQUE_CONSTRAINT_SCHEMA AND PKCU.CONSTRAINT_NAME = RC.UNIQUE_CONSTRAINT_NAME AND PKCU.TABLE_CATALOG = RC.UNIQUE_CONSTRAINT_CATALOG AND PKCU.ORDINAL_POSITION = KCU.ORDINAL_POSITION WHERE PKCU.TABLE_SCHEMA = '${SCHEMA}' ORDER BY KCU.ORDINAL_POSITION</importedKeysQuery>
         <primaryKeysQuery>SELECT null, KCU.TABLE_SCHEMA, KCU.TABLE_NAME, KCU.COLUMN_NAME, KCU.ORDINAL_POSITION, C.CONSTRAINT_NAME   FROM   INFORMATION_SCHEMA.TABLE_CONSTRAINTS C   JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE KCU     ON KCU.CONSTRAINT_CATALOG = C.CONSTRAINT_CATALOG        AND KCU.CONSTRAINT_SCHEMA = C.CONSTRAINT_SCHEMA   AND KCU.TABLE_NAME = C.TABLE_NAME    AND KCU.CONSTRAINT_NAME = C.CONSTRAINT_NAME   WHERE C.CONSTRAINT_TYPE = 'PRIMARY KEY'       AND  KCU.TABLE_SCHEMA = '${SCHEMA}'   ORDER BY KCU.ORDINAL_POSITION</primaryKeysQuery>
 -->
@@ -328,7 +336,7 @@ WHERE SCHEMA_NAME(tbl.schema_id)='%s'
 Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
 (
 	Select KCU.CONSTRAINT_NAME CNAME, KCU.TABLE_NAME TNAME, KCU.COLUMN_NAME COLNAME, 'PK' CTYPE, null DETAIL, KCU.ORDINAL_POSITION POS, 1 TYPEPOS
-    From 
+    From
         INFORMATION_SCHEMA.TABLE_CONSTRAINTS C
     join
         INFORMATION_SCHEMA.KEY_COLUMN_USAGE KCU
@@ -341,7 +349,7 @@ Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
         AND KCU.TABLE_SCHEMA = '%1$s'
 union all
 	Select C.CONSTRAINT_NAME CNAME, KCU.TABLE_NAME TNAME, KCU.COLUMN_NAME COLNAME, 'Unique' CTYPE, null DETAIL, KCU.ORDINAL_POSITION POS, 2 TYPEPOS
-    From 
+    From
         INFORMATION_SCHEMA.TABLE_CONSTRAINTS C
     join
         INFORMATION_SCHEMA.KEY_COLUMN_USAGE KCU
@@ -354,7 +362,7 @@ union all
         AND KCU.TABLE_SCHEMA = '%1$s'
 	union all
 	Select C.CONSTRAINT_NAME CNAME, C.TABLE_NAME TNAME, CU.COLUMN_NAME COLNAME, 'Check' CTYPE, CHECK_CLAUSE DETAIL, 1 POS, 3 TYPEPOS
-	From 
+	From
 		INFORMATION_SCHEMA.CHECK_CONSTRAINTS CC
 	join
 		INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE CU
@@ -371,7 +379,7 @@ union all
 ) pk order by TYPEPOS, CNAME, TNAME, POS, COLNAME
 		</constraintsQuery>
     </dbms>
-    
+
     <!-- for MySQL -->
     <dbms>
         <id>MySQL</id>
@@ -501,7 +509,7 @@ union all
             </entry>
         </stringLiteralEscapeSequences>
         <sqlLimitSuffix>LIMIT 0, %s</sqlLimitSuffix>
-        
+
         <!-- NOLOGGING, TABLESPACE-spec., etc. -->
         <tableProperties></tableProperties>
 
@@ -524,7 +532,7 @@ union all
         <defaultSchemaQuery>Select DATABASE()</defaultSchemaQuery>
 		<estimatedRowCountQuery>select TABLE_NAME, max(CARDINALITY) from INFORMATION_SCHEMA.STATISTICS where TABLE_SCHEMA = '%s' group by TABLE_NAME</estimatedRowCountQuery>
 		<viewTextOrDDLQuery>SELECT VIEW_DEFINITION FROM INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA = '%1$s' and TABLE_NAME = '%2$s'</viewTextOrDDLQuery>
-        
+
 		<virtualColumnsQuery>SELECT TABLE_NAME, COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '${SCHEMA}' AND EXTRA LIKE '%VIRTUAL%'</virtualColumnsQuery>
 <!--
         <importedKeysQuery>SELECT PKCU.TABLE_SCHEMA, null, PKCU.TABLE_NAME, PKCU.COLUMN_NAME, KCU.TABLE_SCHEMA, null, KCU.TABLE_NAME, KCU.COLUMN_NAME, KCU.ORDINAL_POSITION,        null, null, RC.CONSTRAINT_NAME, RC.UNIQUE_CONSTRAINT_NAME, null  FROM INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS RC  JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE KCU    ON KCU.CONSTRAINT_CATALOG = RC.CONSTRAINT_CATALOG        AND KCU.CONSTRAINT_SCHEMA = RC.CONSTRAINT_SCHEMA       AND KCU.CONSTRAINT_NAME = RC.CONSTRAINT_NAME  JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE PKCU    ON PKCU.CONSTRAINT_CATALOG = RC.UNIQUE_CONSTRAINT_CATALOG        AND PKCU.CONSTRAINT_SCHEMA = RC.UNIQUE_CONSTRAINT_SCHEMA       AND PKCU.CONSTRAINT_NAME = RC.UNIQUE_CONSTRAINT_NAME AND PKCU.TABLE_NAME = RC.REFERENCED_TABLE_NAME AND PKCU.ORDINAL_POSITION = KCU.ORDINAL_POSITION  WHERE PKCU.TABLE_SCHEMA = '${SCHEMA}'  ORDER BY KCU.ORDINAL_POSITION</importedKeysQuery>
@@ -542,7 +550,7 @@ union all
 Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
 (
 	Select concat('pk_', KCU.TABLE_NAME) CNAME, KCU.TABLE_NAME TNAME, KCU.COLUMN_NAME COLNAME, 'PK' CTYPE, null DETAIL, KCU.ORDINAL_POSITION POS, 1 TYPEPOS
-    From 
+    From
         INFORMATION_SCHEMA.TABLE_CONSTRAINTS C
     join
         INFORMATION_SCHEMA.KEY_COLUMN_USAGE KCU
@@ -555,7 +563,7 @@ Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
         AND KCU.TABLE_SCHEMA = '%1$s'
 union all
 	Select C.CONSTRAINT_NAME CNAME, KCU.TABLE_NAME TNAME, KCU.COLUMN_NAME COLNAME, 'Unique' CTYPE, null DETAIL, KCU.ORDINAL_POSITION POS, 2 TYPEPOS
-    From 
+    From
         INFORMATION_SCHEMA.TABLE_CONSTRAINTS C
     join
         INFORMATION_SCHEMA.KEY_COLUMN_USAGE KCU
@@ -708,7 +716,7 @@ union all
             </entry>
         </stringLiteralEscapeSequences>
         <sqlLimitSuffix>LIMIT 0, %s</sqlLimitSuffix>
-        
+
         <!-- NOLOGGING, TABLESPACE-spec., etc. -->
         <tableProperties></tableProperties>
 
@@ -731,7 +739,7 @@ union all
         <defaultSchemaQuery>Select DATABASE()</defaultSchemaQuery>
 		<estimatedRowCountQuery>select TABLE_NAME, max(CARDINALITY) from INFORMATION_SCHEMA.STATISTICS where TABLE_SCHEMA = '%s' group by TABLE_NAME</estimatedRowCountQuery>
 		<viewTextOrDDLQuery>SELECT VIEW_DEFINITION FROM INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA = '%1$s' and TABLE_NAME = '%2$s'</viewTextOrDDLQuery>
-        
+
 		<virtualColumnsQuery>SELECT TABLE_NAME, COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '${SCHEMA}' AND EXTRA LIKE '%VIRTUAL%'</virtualColumnsQuery>
 <!--
         <importedKeysQuery>SELECT PKCU.TABLE_SCHEMA, null, PKCU.TABLE_NAME, PKCU.COLUMN_NAME, KCU.TABLE_SCHEMA, null, KCU.TABLE_NAME, KCU.COLUMN_NAME, KCU.ORDINAL_POSITION,        null, null, RC.CONSTRAINT_NAME, RC.UNIQUE_CONSTRAINT_NAME, null  FROM INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS RC  JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE KCU    ON KCU.CONSTRAINT_CATALOG = RC.CONSTRAINT_CATALOG        AND KCU.CONSTRAINT_SCHEMA = RC.CONSTRAINT_SCHEMA       AND KCU.CONSTRAINT_NAME = RC.CONSTRAINT_NAME  JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE PKCU    ON PKCU.CONSTRAINT_CATALOG = RC.UNIQUE_CONSTRAINT_CATALOG        AND PKCU.CONSTRAINT_SCHEMA = RC.UNIQUE_CONSTRAINT_SCHEMA       AND PKCU.CONSTRAINT_NAME = RC.UNIQUE_CONSTRAINT_NAME AND PKCU.TABLE_NAME = RC.REFERENCED_TABLE_NAME AND PKCU.ORDINAL_POSITION = KCU.ORDINAL_POSITION  WHERE PKCU.TABLE_SCHEMA = '${SCHEMA}'  ORDER BY KCU.ORDINAL_POSITION</importedKeysQuery>
@@ -749,7 +757,7 @@ union all
 Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
 (
 	Select concat('pk_', KCU.TABLE_NAME) CNAME, KCU.TABLE_NAME TNAME, KCU.COLUMN_NAME COLNAME, 'PK' CTYPE, null DETAIL, KCU.ORDINAL_POSITION POS, 1 TYPEPOS
-    From 
+    From
         INFORMATION_SCHEMA.TABLE_CONSTRAINTS C
     join
         INFORMATION_SCHEMA.KEY_COLUMN_USAGE KCU
@@ -762,7 +770,7 @@ Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
         AND KCU.TABLE_SCHEMA = '%1$s'
 union all
 	Select C.CONSTRAINT_NAME CNAME, KCU.TABLE_NAME TNAME, KCU.COLUMN_NAME COLNAME, 'Unique' CTYPE, null DETAIL, KCU.ORDINAL_POSITION POS, 2 TYPEPOS
-    From 
+    From
         INFORMATION_SCHEMA.TABLE_CONSTRAINTS C
     join
         INFORMATION_SCHEMA.KEY_COLUMN_USAGE KCU
@@ -833,6 +841,7 @@ union all
             <createTableSuffix>ON COMMIT PRESERVE ROWS NOT LOGGED</createTableSuffix>
             <dropTablePrefix>DROP TABLE SESSION.</dropTablePrefix>
             <indexTablePrefix>SESSION.</indexTablePrefix>
+            <needsExclusiveAccess>true</needsExclusiveAccess>
         </sessionTemporaryTableManager>
         <identityInserts>false</identityInserts>
         <emptyBLOBValue>blob('')</emptyBLOBValue>
@@ -846,7 +855,7 @@ union all
         <avoidLeftJoin>false</avoidLeftJoin>
         <useInlineViewsInDataBrowser>true</useInlineViewsInDataBrowser>
         <estimatedRowCountQuery>SELECT TABNAME, CARD FROM SYSSTAT.TABLES WHERE TABSCHEMA = '%s'</estimatedRowCountQuery>
-        <viewTextOrDDLQuery>SELECT TEXT FROM SYSCAT.VIEWS WHERE VIEWSCHEMA = '%1$s' and VIEWNAME = '%2$s'</viewTextOrDDLQuery> 
+        <viewTextOrDDLQuery>SELECT TEXT FROM SYSCAT.VIEWS WHERE VIEWSCHEMA = '%1$s' and VIEWNAME = '%2$s'</viewTextOrDDLQuery>
         <virtualColumnsQuery>SELECT TABNAME, COLNAME from syscat.columns WHERE TABSCHEMA='${SCHEMA}' AND GENERATED='A'</virtualColumnsQuery>
         <importedKeysQuery> SELECT null, REF.REFTABSCHEMA, REF.REFTABNAME, PKCOL.COLNAME,        null, REF.TABSCHEMA, REF.TABNAME, FKCOL.COLNAME, FKCOL.COLSEQ,        null, null, REF.CONSTNAME, REF.REFKEYNAME, null FROM syscat.references REF, syscat.keycoluse PKCOL, syscat.keycoluse FKCOL WHERE REF.CONSTNAME = FKCOL.CONSTNAME   AND REF.TABSCHEMA = FKCOL.TABSCHEMA   AND REF.TABNAME = FKCOL.TABNAME   AND REF.REFKEYNAME = PKCOL.CONSTNAME   AND REF.REFTABSCHEMA = PKCOL.TABSCHEMA   AND REF.REFTABNAME = PKCOL.TABNAME   AND FKCOL.COLSEQ = PKCOL.COLSEQ   AND REF.REFTABSCHEMA = '${SCHEMA}'   ORDER BY FKCOL.COLSEQ</importedKeysQuery>
         <primaryKeysQuery>   SELECT null, CONST.TABSCHEMA, CONST.TABNAME, PKCOL.COLNAME, PKCOL.COLSEQ, CONST.CONSTNAME FROM syscat.tabconst CONST, syscat.keycoluse PKCOL WHERE CONST.CONSTNAME = PKCOL.CONSTNAME   AND CONST.TABSCHEMA = PKCOL.TABSCHEMA   AND CONST.TABNAME = PKCOL.TABNAME   AND CONST.TYPE = 'P'   AND CONST.TABSCHEMA = '${SCHEMA}'   ORDER BY PKCOL.COLSEQ</primaryKeysQuery>
@@ -854,14 +863,14 @@ union all
         <procedureDetailNeedsSpecificName>true</procedureDetailNeedsSpecificName>
         <functionSourceQuery>SELECT 'Source', cast(ROUTINE_DEFINITION as varchar) FROM SYSIBM.ROUTINES where ROUTINE_SCHEMA = '%1$s' and SPECIFIC_NAME = '%2$s' and ROUTINE_DEFINITION is not null</functionSourceQuery>
         <procedureSourceQuery>SELECT 'Source', cast(ROUTINE_DEFINITION as varchar) FROM SYSIBM.ROUTINES where ROUTINE_SCHEMA = '%1$s' and SPECIFIC_NAME = '%2$s' and ROUTINE_DEFINITION is not null</procedureSourceQuery>
-        
+
         <explainCreateExplainTable>CALL SYSPROC.SYSINSTALLOBJECTS('EXPLAIN', 'C',CAST (NULL AS VARCHAR(128)), CAST (NULL AS VARCHAR(128)))</explainCreateExplainTable>
         <explainPrepare>explain plan for %1$s</explainPrepare>
         <explainQuery><![CDATA[
 WITH tree(operator_ID, level, path, explain_time, cycle)
 AS
 (
-SELECT 1 operator_id 
+SELECT 1 operator_id
      , 0 level
      , CAST('001' AS VARCHAR(1000)) path
      , max(explain_time) explain_time
@@ -883,19 +892,19 @@ SELECT s.source_id
    AND tree.cycle = 0
    AND level < 100
 )
-SELECT * 
+SELECT *
   FROM (
 SELECT "Explain Plan"
   FROM (
 SELECT CAST(   LPAD(id,        MAX(LENGTH(id))        OVER(), ' ')
-            || ' | ' 
+            || ' | '
             || RPAD(operation, MAX(LENGTH(operation)) OVER(), ' ')
-            || ' | ' 
+            || ' | '
             || LPAD(rows,      MAX(LENGTH(rows))      OVER(), ' ')
-            || ' | ' 
-            -- Don't show ActualRows columns if there are no actuals available at all 
+            || ' | '
+            -- Don't show ActualRows columns if there are no actuals available at all
             || CASE WHEN COUNT(ActualRows) OVER () > 1 -- the heading 'ActualRows' is always present, so "1" means no OTHER values
-                    THEN LPAD(ActualRows, MAX(LENGTH(ActualRows)) OVER(), ' ') || ' | ' 
+                    THEN LPAD(ActualRows, MAX(LENGTH(ActualRows)) OVER(), ' ') || ' | '
                     ELSE ''
                END
             || LPAD(cost,      MAX(LENGTH(cost))      OVER(), ' ')
@@ -916,10 +925,10 @@ SELECT CAST(tree.operator_id as VARCHAR(254)) ID
        || CASE WHEN tree.cycle = 1
                THEN '(cycle) '
                ELSE ''
-          END     
+          END
        || COALESCE (
              TRIM(O.Operator_Type)
-          || COALESCE(' (' || argument || ')', '') 
+          || COALESCE(' (' || argument || ')', '')
           || ' '
           || COALESCE(S.Object_Name,'')
           , ''
@@ -987,7 +996,7 @@ SELECT CAST(tree.operator_id as VARCHAR(254)) ID
     ON (    o.operator_id  = tree.operator_id
         AND o.explain_time = tree.explain_time
         AND o.explain_requester = SESSION_USER
-       ) 
+       )
   LEFT JOIN (SELECT LISTAGG (CASE argument_type
                              WHEN 'UNIQUE' THEN
                                   CASE WHEN argument_value = 'TRUE'
@@ -998,13 +1007,13 @@ SELECT CAST(tree.operator_id as VARCHAR(254)) ID
                                   CASE WHEN argument_value = 'TRUE'
                                        THEN 'TOP-N'
                                   ELSE NULL
-                                  END   
+                                  END
                              WHEN 'SCANDIR' THEN
                                   CASE WHEN argument_value != 'FORWARD'
                                        THEN argument_value
                                   ELSE NULL
-                                  END                     
-                             ELSE argument_value     
+                                  END
+                             ELSE argument_value
                              END
                            , ' ') argument
                   , operator_id
@@ -1036,7 +1045,7 @@ SELECT CAST (LPAD(CASE WHEN operator_id = LAG  (operator_id)
                 , MAX(LENGTH(operator_id )+4) OVER()
                 , ' ')
              || how_applied
-             || ' ' 
+             || ' '
              || predicate_text
           AS VARCHAR(100)) "Predicate Information"
      , 'P' || LPAD(id_order, 5, '0') || pred_order path
@@ -1062,12 +1071,12 @@ ORDER BY path
 )]]>
 		</explainQuery>
         <explainCleanup></explainCleanup>
-        
+
         <constraintsQuery>
 Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
 (
 	Select CONST.CONSTNAME CNAME, CONST.TABNAME TNAME, PKCOL.COLNAME COLNAME, 'PK' CTYPE, null DETAIL, PKCOL.COLSEQ POS, 1 TYPEPOS
-	From 
+	From
 	    syscat.tabconst CONST,
         syscat.keycoluse PKCOL
     WHERE
@@ -1088,7 +1097,7 @@ Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
 			and i.TABSCHEMA = '%1$s'
 	union all
 		Select C.CONSTNAME CNAME, C.TABNAME TNAME, ' ' COLNAME, 'Check' CTYPE, cast(TEXT as varchar) DETAIL, 1 POS, 3 TYPEPOS
-		From 
+		From
 			syscat.checks C
 		where
 			C.TYPE = 'C'
@@ -1160,6 +1169,7 @@ Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
             <createTableSuffix>ON COMMIT PRESERVE ROWS NOT LOGGED</createTableSuffix>
             <dropTablePrefix>DROP TABLE SESSION.</dropTablePrefix>
             <indexTablePrefix>SESSION.</indexTablePrefix>
+            <needsExclusiveAccess>true</needsExclusiveAccess>
         </sessionTemporaryTableManager>
         <identityInserts>false</identityInserts>
         <emptyBLOBValue>blob('')</emptyBLOBValue>
@@ -1172,7 +1182,7 @@ Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
         <binaryPattern>x'%s'</binaryPattern>
         <avoidLeftJoin>false</avoidLeftJoin>
         <useInlineViewsInDataBrowser>true</useInlineViewsInDataBrowser>
-        <viewTextOrDDLQuery>SELECT TEXT FROM SYSCAT.VIEWS WHERE VIEWSCHEMA = '%1$s' and VIEWNAME = '%2$s'</viewTextOrDDLQuery> 
+        <viewTextOrDDLQuery>SELECT TEXT FROM SYSCAT.VIEWS WHERE VIEWSCHEMA = '%1$s' and VIEWNAME = '%2$s'</viewTextOrDDLQuery>
         <estimatedRowCountQuery>SELECT TABNAME, CARD FROM SYSSTAT.TABLES WHERE TABSCHEMA = '%s'</estimatedRowCountQuery>
         <virtualColumnsQuery>SELECT TABNAME, COLNAME from syscat.columns WHERE TABSCHEMA='${SCHEMA}' AND GENERATED='A'</virtualColumnsQuery>
         <importedKeysQuery>SELECT null, REF.REFTABSCHEMA, REF.REFTABNAME, PKCOL.COLNAME, null, REF.TABSCHEMA, REF.TABNAME, FKCOL.COLNAME, FKCOL.COLSEQ, null, null, REF.CONSTNAME, REF.REFKEYNAME, null FROM syscat.references REF, syscat.keycoluse PKCOL, syscat.keycoluse FKCOL WHERE REF.CONSTNAME = FKCOL.CONSTNAME   AND REF.TABSCHEMA = FKCOL.TABSCHEMA   AND REF.TABNAME = FKCOL.TABNAME   AND REF.REFKEYNAME = PKCOL.CONSTNAME   AND REF.REFTABSCHEMA = PKCOL.TABSCHEMA   AND REF.REFTABNAME = PKCOL.TABNAME   AND FKCOL.COLSEQ = PKCOL.COLSEQ   AND REF.REFTABSCHEMA = '${SCHEMA}' ORDER BY FKCOL.COLSEQ</importedKeysQuery>
@@ -1222,6 +1232,7 @@ Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
             <createTableSuffix> WITH NO LOG</createTableSuffix>
             <dropTablePrefix>DROP TABLE </dropTablePrefix>
             <indexTablePrefix></indexTablePrefix>
+            <needsExclusiveAccess>true</needsExclusiveAccess>
         </sessionTemporaryTableManager>
         <identityInserts>false</identityInserts>
         <timestampPattern>'cast('''yyyy-MM-dd HH.mm.ss'.${NANO}'' as datetime year to fraction)'</timestampPattern>
@@ -1231,7 +1242,7 @@ Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
         <useInlineViewsInDataBrowser>true</useInlineViewsInDataBrowser>
         <identifierQuoteString>"</identifierQuoteString>
     </dbms>
-    
+
     <!-- for PostgreSQL -->
     <dbms>
         <id>POSTGRESQL</id>
@@ -1247,6 +1258,8 @@ Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
         <statisticRenovator>
             <scriptFileName>script/postgres/update_statistics.sql</scriptFileName>
         </statisticRenovator>
+        <rowidName>ctid</rowidName>
+        <rowidType>tid</rowidType>
         <typeReplacement>
             <entry>
                 <key>DOUBLE</key>
@@ -1281,12 +1294,74 @@ Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
                 <value>varchar</value>
             </entry>
         </typeReplacement>
+        <sqlExpressionRule>
+       	    <entry>
+                <key>point</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>interval</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>inet</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>int4range</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>int8range</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>box</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>cidr</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>circle</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>line</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>lseg</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>macaddr</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>path</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>polygon</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>money</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>tid</key>
+                <value>'$1'::$2</value>
+            </entry>
+        </sqlExpressionRule>
         <sqlLimitSuffix>LIMIT %s</sqlLimitSuffix>
         <fetchSize>4096</fetchSize>
 
         <!-- NOLOGGING, TABLESPACE-spec., etc. -->
         <tableProperties>CREATE UNLOGGED TABLE</tableProperties>
-        
+
         <sessionTemporaryTableManager>
             <createIndexPrefix>CREATE INDEX </createIndexPrefix>
             <createIndexSuffix></createIndexSuffix>
@@ -1294,6 +1369,7 @@ Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
             <createTableSuffix></createTableSuffix>
             <dropTablePrefix>-- DROP TABLE </dropTablePrefix>
             <indexTablePrefix></indexTablePrefix>
+            <needsExclusiveAccess>false</needsExclusiveAccess>
         </sessionTemporaryTableManager>
         <identityInserts>false</identityInserts>
         <datePattern>''yyyy-MM-dd'''::date'</datePattern>
@@ -1324,7 +1400,7 @@ WHERE p.proname = '%2$s' AND n.nspname = '%1$s'
 Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
 (
 	Select KCU.CONSTRAINT_NAME CNAME, KCU.TABLE_NAME TNAME, KCU.COLUMN_NAME COLNAME, 'PK' CTYPE, null DETAIL, KCU.ORDINAL_POSITION POS, 1 TYPEPOS
-    From 
+    From
         INFORMATION_SCHEMA.TABLE_CONSTRAINTS C
     join
         INFORMATION_SCHEMA.KEY_COLUMN_USAGE KCU
@@ -1337,7 +1413,7 @@ Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
         AND KCU.TABLE_SCHEMA = '%1$s'
 union all
 	Select C.CONSTRAINT_NAME CNAME, KCU.TABLE_NAME TNAME, KCU.COLUMN_NAME COLNAME, 'Unique' CTYPE, null DETAIL, KCU.ORDINAL_POSITION POS, 2 TYPEPOS
-    From 
+    From
         INFORMATION_SCHEMA.TABLE_CONSTRAINTS C
     join
         INFORMATION_SCHEMA.KEY_COLUMN_USAGE KCU
@@ -1350,7 +1426,7 @@ union all
         AND KCU.TABLE_SCHEMA = '%1$s'
 	union all
 	Select C.CONSTRAINT_NAME CNAME, C.TABLE_NAME TNAME, CU.COLUMN_NAME COLNAME, 'Check' CTYPE, CHECK_CLAUSE DETAIL, 1 POS, 3 TYPEPOS
-	From 
+	From
 		INFORMATION_SCHEMA.CHECK_CONSTRAINTS CC
 	join
 		INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE CU
@@ -1399,6 +1475,8 @@ union all
         <statisticRenovator>
             <scriptFileName>script/postgres/update_statistics.sql</scriptFileName>
         </statisticRenovator>
+        <rowidName>ctid</rowidName>
+        <rowidType>tid</rowidType>
         <typeReplacement>
             <entry>
                 <key>DOUBLE</key>
@@ -1433,12 +1511,74 @@ union all
                 <value>varchar</value>
             </entry>
         </typeReplacement>
+        <sqlExpressionRule>
+       	    <entry>
+                <key>point</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>interval</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>inet</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>int4range</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>int8range</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>box</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>cidr</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>circle</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>line</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>lseg</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>macaddr</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>path</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>polygon</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>money</key>
+                <value>'$1'::$2</value>
+            </entry>
+       	    <entry>
+                <key>tid</key>
+                <value>'$1'::$2</value>
+            </entry>
+        </sqlExpressionRule>
         <sqlLimitSuffix>LIMIT %s</sqlLimitSuffix>
         <fetchSize>4096</fetchSize>
 
         <!-- NOLOGGING, TABLESPACE-spec., etc. -->
         <tableProperties>CREATE UNLOGGED TABLE</tableProperties>
-        
+
         <sessionTemporaryTableManager>
             <createIndexPrefix>CREATE INDEX </createIndexPrefix>
             <createIndexSuffix></createIndexSuffix>
@@ -1446,6 +1586,7 @@ union all
             <createTableSuffix></createTableSuffix>
             <dropTablePrefix>-- DROP TABLE </dropTablePrefix>
             <indexTablePrefix></indexTablePrefix>
+            <needsExclusiveAccess>false</needsExclusiveAccess>
         </sessionTemporaryTableManager>
         <identityInserts>false</identityInserts>
         <datePattern>''yyyy-MM-dd'''::date'</datePattern>
@@ -1476,7 +1617,7 @@ WHERE p.proname = '%2$s' AND n.nspname = '%1$s'
 Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
 (
 	Select KCU.CONSTRAINT_NAME CNAME, KCU.TABLE_NAME TNAME, KCU.COLUMN_NAME COLNAME, 'PK' CTYPE, null DETAIL, KCU.ORDINAL_POSITION POS, 1 TYPEPOS
-    From 
+    From
         INFORMATION_SCHEMA.TABLE_CONSTRAINTS C
     join
         INFORMATION_SCHEMA.KEY_COLUMN_USAGE KCU
@@ -1489,7 +1630,7 @@ Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
         AND KCU.TABLE_SCHEMA = '%1$s'
 union all
 	Select C.CONSTRAINT_NAME CNAME, KCU.TABLE_NAME TNAME, KCU.COLUMN_NAME COLNAME, 'Unique' CTYPE, null DETAIL, KCU.ORDINAL_POSITION POS, 2 TYPEPOS
-    From 
+    From
         INFORMATION_SCHEMA.TABLE_CONSTRAINTS C
     join
         INFORMATION_SCHEMA.KEY_COLUMN_USAGE KCU
@@ -1502,7 +1643,7 @@ union all
         AND KCU.TABLE_SCHEMA = '%1$s'
 	union all
 	Select C.CONSTRAINT_NAME CNAME, C.TABLE_NAME TNAME, CU.COLUMN_NAME COLNAME, 'Check' CTYPE, CHECK_CLAUSE DETAIL, 1 POS, 3 TYPEPOS
-	From 
+	From
 		INFORMATION_SCHEMA.CHECK_CONSTRAINTS CC
 	join
 		INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE CU
@@ -1550,10 +1691,10 @@ union all
         <statisticRenovator>
             <scriptFileName>script/cloudscape/update_statistics.sql</scriptFileName>
         </statisticRenovator>
-        
+
         <!-- NOLOGGING, TABLESPACE-spec., etc. -->
         <tableProperties></tableProperties>
-        
+
         <identityInserts>false</identityInserts>
         <embeddedLobSizeLimit>3980</embeddedLobSizeLimit>
         <binaryPattern>x'%s'</binaryPattern>
@@ -1561,7 +1702,7 @@ union all
         <useInlineViewsInDataBrowser>true</useInlineViewsInDataBrowser>
         <identifierQuoteString>"</identifierQuoteString>
     </dbms>
-    
+
     <!-- for Firebird -->
     <dbms>
         <id>FIREBIRD</id>
@@ -1582,10 +1723,10 @@ union all
                 <value>DOUBLE PRECISION</value>
             </entry>
         </typeReplacement>
-        
+
         <!-- NOLOGGING, TABLESPACE-spec., etc. -->
         <tableProperties></tableProperties>
-        
+
         <identityInserts>false</identityInserts>
         <embeddedLobSizeLimit>3980</embeddedLobSizeLimit>
         <binaryPattern>x'%s'</binaryPattern>
@@ -1593,7 +1734,7 @@ union all
         <useInlineViewsInDataBrowser>true</useInlineViewsInDataBrowser>
         <identifierQuoteString>"</identifierQuoteString>
     </dbms>
-    
+
     <!-- for Derby -->
     <dbms>
         <id>DERBY</id>
@@ -1608,10 +1749,10 @@ union all
         <statisticRenovator>
             <scriptFileName>script/derby/update_statistics.sql</scriptFileName>
         </statisticRenovator>
-        
+
         <!-- NOLOGGING, TABLESPACE-spec., etc. -->
         <tableProperties></tableProperties>
-        
+
         <identityInserts>false</identityInserts>
         <embeddedLobSizeLimit>3980</embeddedLobSizeLimit>
         <binaryPattern>x'%s'</binaryPattern>
@@ -1619,7 +1760,7 @@ union all
         <useInlineViewsInDataBrowser>true</useInlineViewsInDataBrowser>
         <identifierQuoteString>"</identifierQuoteString>
     </dbms>
-    
+
     <!-- for SyBASE -->
     <dbms>
         <id>SYBASE</id>
@@ -1635,10 +1776,10 @@ union all
             <scriptFileName>script/sybase/update_statistics.sql</scriptFileName>
         </statisticRenovator>
         <sqlLimitSuffix>TOP %s</sqlLimitSuffix>
-        
+
         <!-- NOLOGGING, TABLESPACE-spec., etc. -->
         <tableProperties></tableProperties>
-        
+
         <sessionTemporaryTableManager>
 			<dmlTableReferencePrefix>#</dmlTableReferencePrefix>
 			<ddlTableReferencePrefix>#</ddlTableReferencePrefix>
@@ -1648,6 +1789,7 @@ union all
 			<createIndexSuffix></createIndexSuffix>
 			<indexTablePrefix></indexTablePrefix>
 			<dropTablePrefix>DROP TABLE </dropTablePrefix>
+            <needsExclusiveAccess>true</needsExclusiveAccess>
         </sessionTemporaryTableManager>
         <exportBlocks>timestamp</exportBlocks>
         <exportBlocks>TIMESTAMP</exportBlocks>
@@ -1661,6 +1803,9 @@ union all
                 <value>DATETIME</value>
             </entry>
         </typeReplacement>
+        <clobTypesRE>varchar\(max\)|text\(.*\)</clobTypesRE>
+        <nClobTypesRE>nvarchar\(max\)|xml\(.*\)|ntext\(.*\)</nClobTypesRE>
+        <blobTypesRE>varbinary\(max\)|image\(.*\)</blobTypesRE>
         <identityInserts>true</identityInserts>
         <embeddedLobSizeLimit>3980</embeddedLobSizeLimit>
         <binaryPattern>0x%s</binaryPattern>
@@ -1673,7 +1818,7 @@ union all
         	<afterSelect>TOP(%1$s)</afterSelect>
         </limitTransactionSize>
     </dbms>
-    
+
     <!-- for SQLite -->
     <dbms>
         <id>SQLITE</id>
@@ -1696,10 +1841,10 @@ union all
             </entry>
         </typeReplacement>
         <sqlLimitSuffix>LIMIT %s</sqlLimitSuffix>
-        
+
         <!-- NOLOGGING, TABLESPACE-spec., etc. -->
         <tableProperties></tableProperties>
-        
+
         <sessionTemporaryTableManager>
             <createIndexPrefix>CREATE INDEX </createIndexPrefix>
             <createIndexSuffix></createIndexSuffix>
@@ -1707,6 +1852,7 @@ union all
             <createTableSuffix></createTableSuffix>
             <dropTablePrefix>DROP TABLE </dropTablePrefix>
             <indexTablePrefix></indexTablePrefix>
+            <needsExclusiveAccess>true</needsExclusiveAccess>
         </sessionTemporaryTableManager>
         <identityInserts>false</identityInserts>
         <embeddedLobSizeLimit>3980</embeddedLobSizeLimit>
@@ -1715,7 +1861,7 @@ union all
         <useInlineViewsInDataBrowser>true</useInlineViewsInDataBrowser>
         <identifierQuoteString>"</identifierQuoteString>
     </dbms>
-    
+
     <!-- for HSQLDB -->
     <dbms>
         <id>HSQL</id>
@@ -1728,10 +1874,10 @@ union all
             <upsertMode>FROM_JL_DUAL</upsertMode>
         </sqlDialect>
         <sqlLimitSuffix>LIMIT %s</sqlLimitSuffix>
-        
+
         <!-- NOLOGGING, TABLESPACE-spec., etc. -->
         <tableProperties>NOLOGGING</tableProperties>
-        
+
         <identityInserts>false</identityInserts>
         <embeddedLobSizeLimit>3980</embeddedLobSizeLimit>
         <binaryPattern>x'%s'</binaryPattern>
@@ -1739,7 +1885,7 @@ union all
         <useInlineViewsInDataBrowser>true</useInlineViewsInDataBrowser>
         <identifierQuoteString>"</identifierQuoteString>
     </dbms>
-    
+
     <!-- for H2 -->
     <dbms>
         <id>H2</id>
@@ -1778,7 +1924,7 @@ union all
             </entry>
         </stringLiteralEscapeSequences>
         <sqlLimitSuffix>LIMIT %s</sqlLimitSuffix>
-        
+
         <!-- NOLOGGING, TABLESPACE-spec., etc. -->
         <tableProperties></tableProperties>
 
@@ -1798,21 +1944,21 @@ union all
 Select CNAME, TNAME, COLNAME, CTYPE, DETAIL From
 (
 	Select C.CONSTRAINT_NAME CNAME, C.TABLE_NAME TNAME, C.COLUMN_LIST COLNAME, 'PK' CTYPE, null DETAIL, 1 POS, 1 TYPEPOS
-	From 
+	From
 		INFORMATION_SCHEMA.CONSTRAINTS C
 	where
 		C.CONSTRAINT_TYPE = 'PRIMARY KEY'
 		AND C.TABLE_SCHEMA = '%1$s'
 	union all
 	Select I.INDEX_NAME CNAME, I.TABLE_NAME TNAME, I.COLUMN_NAME COLNAME, 'Unique' CTYPE, null DETAIL, I.ORDINAL_POSITION POS, 2 TYPEPOS
-	From 
+	From
 		INFORMATION_SCHEMA.INDEXES I
 	where
 		I.INDEX_TYPE_NAME = 'UNIQUE INDEX'
 		AND I.TABLE_SCHEMA = '%1$s'
 	union all
 	Select C.CONSTRAINT_NAME CNAME, C.TABLE_NAME TNAME, case when C.COLUMN_LIST is null then '' else C.COLUMN_LIST end COLNAME, 'Check' CTYPE, CHECK_EXPRESSION DETAIL, 1 POS, 3 TYPEPOS
-	From 
+	From
 		INFORMATION_SCHEMA.CONSTRAINTS C
 	where
 		C.CONSTRAINT_TYPE = 'CHECK'

src/main/engine/net/sf/jailer/database/BasicDataSource.java

@@ -21,6 +21,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -200,16 +201,27 @@ public class BasicDataSource implements DataSource {
 		}
 	}
 
-	private static Set<String> registeredDriverClassNames = new HashSet<String>();
-	
+	private static Set<String> registeredDriverClassNames = Collections.synchronizedSet(new HashSet<String>());
+
+	private DriverShim currentDriver;
+	private static Map<Class<Driver>, DriverShim> drivers = new IdentityHashMap<Class<Driver>, BasicDataSource.DriverShim>();
+
+	@SuppressWarnings("deprecation")
 	private void loadDriver(URL[] jdbcDriverURL) {
 		ClassLoader classLoaderForJdbcDriver = addJarToClasspath(jdbcDriverURL);
 		try {
 			if (classLoaderForJdbcDriver != null) {
-				Driver d;
 				try {
-					d = (Driver) Class.forName(driverClassName, true, classLoaderForJdbcDriver).newInstance();
-					DriverManager.registerDriver(new DriverShim(d));
+					@SuppressWarnings("unchecked")
+					Class<Driver> driverClass = (Class<Driver>) Class.forName(driverClassName, true, classLoaderForJdbcDriver);
+					synchronized (drivers) {
+						currentDriver = drivers.get(driverClass);
+						if (currentDriver == null) {
+							currentDriver = new DriverShim((Driver) driverClass.newInstance());
+							drivers.put(driverClass, currentDriver);
+						}
+					}
+					DriverManager.registerDriver(currentDriver);
 					registeredDriverClassNames.add(driverClassName);
 				} catch (InstantiationException e) {
 					throw new RuntimeException(e);
@@ -335,30 +347,39 @@ public class BasicDataSource implements DataSource {
 				}
 			}
 		}
-		
+
 		Map<String, String> jdbcProperties = theDbms.getJdbcProperties();
-		if (con == null && jdbcProperties != null) {
+		if (con == null) {
+			java.util.Properties info = new java.util.Properties();
+			if (dbUser != null) {
+				info.put("user", dbUser);
+			}
+			if (dbPassword != null) {
+				info.put("password", dbPassword);
+			}
+			if (jdbcProperties != null) {
+				for (Map.Entry<String, String> entry: jdbcProperties.entrySet()) {
+					info.put(entry.getKey(), entry.getValue());
+				}
+			}
 			try {
-				 java.util.Properties info = new java.util.Properties();
-				 if (dbUser != null) {
-					 info.put("user", dbUser);
-				 }
-				 if (dbPassword != null) {
-					 info.put("password", dbPassword);
-				 }
-				 for (Map.Entry<String, String> entry: jdbcProperties.entrySet()) {
-					 info.put(entry.getKey(), entry.getValue());
-				 }
-				 con = DriverManager.getConnection(dbUrl, info);
-			} catch (SQLException e2) {
-				// ignore
+				if (currentDriver != null) {
+					con = currentDriver.connect(dbUrl, info);
+				}
+			} catch (SQLException e) {
+				try {
+					if (currentDriver.acceptsURL(dbUrl)) {
+						throw e;
+					}
+				} catch (SQLException e2) {
+					// fall back
+				}
+			}
+			if (con == null) {
+				con = DriverManager.getConnection(dbUrl, info);
 			}
 		}
-		
-		if (con == null) {
-			con = DriverManager.getConnection(dbUrl, dbUser, dbPassword);
-		}
-		
+
 		if (maxPoolSize == 0) {
 			return con;
 		}

src/main/engine/net/sf/jailer/database/DB2ShellScriptBasedStatisticRenovator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/database/DMLTransformer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -53,9 +53,9 @@ import net.sf.jailer.util.SqlScriptExecutor;
 import net.sf.jailer.util.SqlUtil;
 
 /**
- * A {@link ResultSetReader} that writes the read rows as DML-statements 
+ * A {@link ResultSetReader} that writes the read rows as DML-statements
  * into the export-script.
- * 
+ *
  * @author Ralf Wisser
  */
 public class DMLTransformer extends AbstractResultSetReader {
@@ -64,12 +64,12 @@ public class DMLTransformer extends AbstractResultSetReader {
 	 * The table to read from.
 	 */
 	private final Table table;
-	
+
 	/**
 	 * The file to write to.
 	 */
 	private final OutputStreamWriter scriptFileWriter;
-	
+
 	/**
 	 * Number of columns.
 	 */
@@ -85,7 +85,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 	 */
 	private List<String> lobColumns = null;
 	private boolean[] isLobColumn;
-	
+
 	/**
 	 * Literals for empty lob values.
 	 */
@@ -95,7 +95,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 	 * Whether or not the table has columns of type CLOB or BLOB.
 	 */
 	private boolean tableHasLobs = false;
-	
+
 	/**
 	 * Lob columns indexes.
 	 */
@@ -110,17 +110,17 @@ public class DMLTransformer extends AbstractResultSetReader {
 	 * For building compact insert-statements.
 	 */
 	private final StatementBuilder insertStatementBuilder;
-	
+
 	/**
 	 * For building compact update-statements.
 	 */
 	private final StatementBuilder updateStatementBuilder;
-	
+
 	/**
 	 * Terminator of current {@link #updateStatementBuilder}
 	 */
 	private String updateStatementBuilderTerminator;
-	
+
 	/**
 	 * For building compact insert-parts of upsert-statements.
 	 */
@@ -135,7 +135,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 	 * Maximum length of SQL values list (for generated inserts).
 	 */
 	private final int maxBodySize;
-	
+
 	/**
 	 * Counts the exported LOBs. (GUI support)
 	 */
@@ -145,12 +145,12 @@ public class DMLTransformer extends AbstractResultSetReader {
 	 * For quoting of column names.
 	 */
 	protected final Quoting quoting;
-	
+
 	/**
 	 * If table has identity column (MSSQL/Sybase)
 	 */
 	private boolean tableHasIdentityColumn;
-	
+
 	/**
 	 * Current session;
 	 */
@@ -170,35 +170,35 @@ public class DMLTransformer extends AbstractResultSetReader {
 	 * The execution context.
 	 */
 	private final ExecutionContext executionContext;
-	
+
 	/**
 	 * Transforms {@link Filter} into SQL-expressions.
 	 */
 	private final ImportFilterTransformer importFilterTransformer;
-	
+
 	private final Set<String> primaryKeyColumnNames;
 	private final Set<String> nullableColumnNames;
-	
+
 	/**
 	 * Factory.
 	 */
 	public static class Factory implements TransformerFactory {
-		
+
 		private final int maxBodySize;
 		private final boolean upsertOnly;
 		private final OutputStreamWriter scriptFileWriter;
 		private final Session session;
 		private final DBMS targetDBMSConfiguration;
 		private ImportFilterTransformer importFilterTransformer;
-		
+
 		/**
 		 * The execution context.
 		 */
 		private final ExecutionContext executionContext;
-		
+
 		/**
 		 * Constructor.
-		 * 
+		 *
 		 * @param scriptFileWriter the file to write to
 		 * @param maxBodySize maximum length of SQL values list (for generated inserts)
 		 * @param upsertOnly use 'upsert' statements for all entities
@@ -213,9 +213,9 @@ public class DMLTransformer extends AbstractResultSetReader {
 		}
 
 		/**
-		 * Creates transformer (as {@link ResultSetReader} which 
+		 * Creates transformer (as {@link ResultSetReader} which
 		 * transforms rows of a given table into an external representation.
-		 * 
+		 *
 		 * @param table the table
 		 * @return a transformer
 		 */
@@ -223,7 +223,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 		public ResultSetReader create(Table table) throws SQLException {
 			return new DMLTransformer(table, scriptFileWriter, upsertOnly, maxBodySize, session, targetDBMSConfiguration, importFilterTransformer, executionContext);
 		}
-		
+
 		/**
 		 * Sets the {@link ImportFilterTransformer}.
 		 */
@@ -231,21 +231,21 @@ public class DMLTransformer extends AbstractResultSetReader {
 			this.importFilterTransformer = importFilterManager;
 		}
 
-	};
+	}
 
 	private final List<Column> selectionClause;
-	
+
 	/**
 	 * Constructor.
-	 * 
+	 *
 	 * @param table the table to read from
 	 * @param scriptFileWriter the file to write to
 	 * @param maxBodySize maximum length of SQL values list (for generated inserts)
 	 * @param upsertOnly use 'upsert' statements for all entities
 	 * @param session the session
 	 * @param targetDBMSConfiguration configuration of the target DBMS
-	 * @param executionContext 
-	 * @param importFilterTransformer2 
+	 * @param executionContext
+	 * @param importFilterTransformer2
 	 */
 	protected DMLTransformer(Table table, OutputStreamWriter scriptFileWriter, boolean upsertOnly, int maxBodySize, Session session, DBMS targetDBMSConfiguration, ImportFilterTransformer importFilterTransformer, ExecutionContext executionContext) throws SQLException {
 		this.executionContext = executionContext;
@@ -259,7 +259,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 		this.updateStatementBuilder = new StatementBuilder(maxBodySize);
 		this.quoting = createQuoting(session);
 		this.importFilterTransformer = importFilterTransformer;
-		if (targetDBMSConfiguration != null && targetDBMSConfiguration != session.dbms) {
+		if (targetDBMSConfiguration != session.dbms) {
 			if (targetDBMSConfiguration.getIdentifierQuoteString() != null) {
 				this.quoting.setIdentifierQuoteString(targetDBMSConfiguration.getIdentifierQuoteString());
 			}
@@ -290,9 +290,9 @@ public class DMLTransformer extends AbstractResultSetReader {
 	protected Quoting createQuoting(Session session) throws SQLException {
 		return Quoting.getQuoting(session);
 	}
-	
+
 	private Map<Integer, String> columnTypeFromDatamodel = new HashMap<Integer, String>();
-	
+
 	/**
 	 * Reads result-set and writes into export-script.
 	 */
@@ -308,9 +308,10 @@ public class DMLTransformer extends AbstractResultSetReader {
 			labelCSL = "";
 			tableHasLobs = false;
 			for (int i = 1; i <= columnCount; ++i) {
-				String mdColumnLabel = quoting.quote(getMetaData(resultSet).getColumnLabel(i));
-				int mdColumnType = getMetaData(resultSet).getColumnType(i);
-				
+				// TODO get rid of #getColumnLabel(int). Pass selection-clause as list of columns (for all Transformers).
+				String mdColumnLabel = SqlUtil.columnLabel(quoting, session, targetDBMSConfiguration, table, getMetaData(resultSet).getColumnLabel(i));
+				// int mdColumnType = getMetaData(resultSet).getColumnType(i);
+				int mdColumnType = SqlUtil.getColumnType(session.dbms, resultSet, getMetaData(resultSet), i, null);
 				if ((mdColumnType == Types.BLOB || mdColumnType == Types.CLOB || mdColumnType == Types.NCLOB || mdColumnType == Types.SQLXML) && !DBMS.SQLITE.equals(targetDBMSConfiguration)) {
 					tableHasLobs = true;
 					isLobColumn[i] = true;
@@ -363,7 +364,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 					valueList.append(", ");
 				}
 				f = false;
-				String cVal = isSmallLob? (String) content : 
+				String cVal = isSmallLob? (String) content :
 					 convertToSql(cellContentConverter, resultSet, i, content, 0, null);
 				if (!isSmallLob && content != null && emptyLobValue[i] != null) {
 					cVal = emptyLobValue[i];
@@ -373,7 +374,9 @@ public class DMLTransformer extends AbstractResultSetReader {
 			}
 			if (table.getUpsert() || upsertOnly) {
 				if (table.getNonVirtualPKColumns(session).isEmpty()) {
-					throw new DataModel.NoPrimaryKeyException(table, "has no primary key. Upsert statement can not be generated.");
+					throw new DataModel.NoPrimaryKeyException(table, "has no " +
+							(table.primaryKey != null && table.primaryKey.getColumns() != null && !table.primaryKey.getColumns().isEmpty()? "non-virtual " : "") +
+							"primary key. Upsert statement can not be generated.");
 				}
 
 				Map<String, String> val = new HashMap<String, String>();
@@ -394,7 +397,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 					String suffix = null;
 					if (DBMS.POSTGRESQL.equals(targetDBMSConfiguration)) {
 						int mdColumnType = getMetaData(resultSet).getColumnType(i);
-						if (mdColumnType == Types.TIME || (content == null && ((currentDialect.getUpdateMode() == UPDATE_MODE.PG || !generateUpsertStatementsWithoutNulls)))) { 
+						if (mdColumnType == Types.TIME || (content == null && ((currentDialect.getUpdateMode() == UPDATE_MODE.PG || !generateUpsertStatementsWithoutNulls)))) {
 							// explicit cast needed
 							if (mdColumnType == Types.OTHER) {
 								String type = null;
@@ -440,14 +443,14 @@ public class DMLTransformer extends AbstractResultSetReader {
 						columnsWONull.append(columnLabel[i]);
 					}
 				}
-				
+
 				String insertHead = "Insert into " + qualifiedTableName(table) + "(" + columnsWONull + ") ";
 				f = true;
 				StringBuffer whereForTerminator = new StringBuffer("");
 				StringBuffer whereForTerminatorWONull = new StringBuffer("");
 				StringBuffer where = new StringBuffer("");
 				StringBuffer whereWOAlias = new StringBuffer("");
-				
+
 				// assemble 'where' for sub-select and update
 				for (Column pk: table.getNonVirtualPKColumns(session)) {
 					if (!f) {
@@ -483,13 +486,13 @@ public class DMLTransformer extends AbstractResultSetReader {
 				}
 
 				if (currentDialect.getUpsertMode() == UPSERT_MODE.MERGE && !tableHasLobs) {
-					// MERGE INTO JL_TMP T USING (SELECT 1 c1, 2 c2 from dual) incoming 
-					// ON (T.c1 = incoming.c1) 
-					// WHEN MATCHED THEN UPDATE SET T.c2 = incoming.c2 
+					// MERGE INTO JL_TMP T USING (SELECT 1 c1, 2 c2 from dual) incoming
+					// ON (T.c1 = incoming.c1)
+					// WHEN MATCHED THEN UPDATE SET T.c2 = incoming.c2
 					// WHEN NOT MATCHED THEN INSERT (T.c1, T.c2) VALUES (incoming.c1, incoming.c2)
 					insertHead = "MERGE INTO " + qualifiedTableName(table) + " T USING(";
 					StringBuffer terminator = new StringBuffer(") Q ON(" + whereForTerminator + ") ");
-					
+
 					StringBuffer sets = new StringBuffer();
 					StringBuffer tSchema = new StringBuffer();
 					StringBuffer iSchema = new StringBuffer();
@@ -497,7 +500,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 						if (columnLabel[i] == null) {
 							continue;
 						}
-						if  (!isPrimaryKeyColumn(columnLabel[i])) {
+						if (!isPrimaryKeyColumn(columnLabel[i])) {
 							if (sets.length() > 0) {
 								sets.append(", ");
 							}
@@ -522,7 +525,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 						sb = new StatementBuilder(maxBodySize);
 						upsertInsertStatementBuilder.put(insertHead, sb);
 					}
-				
+
 					String item = "Select " + valueList + " from dual";
 					if (!sb.isAppendable(insertHead)) {
 						writeToScriptFile(sb.build(), true);
@@ -536,13 +539,13 @@ public class DMLTransformer extends AbstractResultSetReader {
 					StringBuffer terminator = new StringBuffer(") as Q(" + columnsWONull + ") Where not exists (Select * from " + qualifiedTableName(table) + " T "
 							+ "Where ");
 					terminator.append(whereForTerminatorWONull + ");" + PrintUtil.LINE_SEPARATOR);
-					
+
 					StatementBuilder sb = upsertInsertStatementBuilder.get(insertHead);
 					if (sb == null) {
 						sb = new StatementBuilder(maxBodySize);
 						upsertInsertStatementBuilder.put(insertHead, sb);
 					}
-				
+
 					String item = (maxBodySize > 1? PrintUtil.LINE_SEPARATOR + " " : "") + "(" + valuesWONull + ")";
 					if (!sb.isAppendable(insertHead)) {
 						writeToScriptFile(sb.build(), true);
@@ -553,45 +556,45 @@ public class DMLTransformer extends AbstractResultSetReader {
 					StringBuffer terminator = new StringBuffer(") as Q " + PrintUtil.LINE_SEPARATOR + "Where not exists (Select * from " + qualifiedTableName(table) + " T "
 							+ "Where ");
 					terminator.append(whereForTerminatorWONull + ");" + PrintUtil.LINE_SEPARATOR);
-					
+
 					StatementBuilder sb = upsertInsertStatementBuilder.get(insertHead);
 					if (sb == null) {
 						sb = new StatementBuilder(maxBodySize);
 						upsertInsertStatementBuilder.put(insertHead, sb);
 					}
-				
+
 					String item;
 					item = valuesWONull.toString();
 					if (!sb.isAppendable(insertHead)) {
 						writeToScriptFile(sb.build(), true);
 					}
 					if (sb.isEmpty()) {
-						item = namedValuesWONull.toString();	
+						item = namedValuesWONull.toString();
 					}
 					sb.append(insertHead, item, " union all " + PrintUtil.LINE_SEPARATOR + " Select ", terminator.toString());
 				} else {
-					String item = "Select " + valuesWONull + " From " + 
-						(currentDialect.getUpsertMode() == UPSERT_MODE.FROM_DUAL || 
+					String item = "Select " + valuesWONull + " From " +
+						(currentDialect.getUpsertMode() == UPSERT_MODE.FROM_DUAL ||
 						 currentDialect.getUpsertMode() == UPSERT_MODE.MERGE? // oracle table with lobs
 								 "dual" : currentDialect.getUpsertMode() == UPSERT_MODE.FROM_SYSDUMMY1? "sysibm.sysdummy1" : SQLDialect.DUAL_TABLE);
 					StringBuffer terminator = new StringBuffer(" Where not exists (Select * from " + qualifiedTableName(table) + " T "
 							+ "Where ");
 					terminator.append(where + ");" + PrintUtil.LINE_SEPARATOR);
-					
+
 					StatementBuilder sb = upsertInsertStatementBuilder.get(insertHead);
 					if (sb == null) {
 						sb = new StatementBuilder(1 /* insertStatementBuilder.getMaxBodySize() */);
 						upsertInsertStatementBuilder.put(insertHead, sb);
 					}
-				
+
 					if (!sb.isAppendable(insertHead)) {
 						writeToScriptFile(sb.build(), true);
 					}
 					sb.append(insertHead, item, ", ", terminator.toString());
 				}
-				
+
 				// TODO refactoring, method is too long
-				
+
 				if (currentDialect.getUpsertMode() != UPSERT_MODE.MERGE || tableHasLobs) {
 					if (currentDialect.getUpdateMode() == UPDATE_MODE.PG && DBMS.POSTGRESQL.equals(session.dbms)) {
 						StringBuilder item = new StringBuilder(" (");
@@ -626,8 +629,8 @@ public class DMLTransformer extends AbstractResultSetReader {
 								writeToScriptFile(updateStatementBuilder.build(), true);
 							}
 							updateStatementBuilder.append(
-									headAsString, 
-									itemAsString, 
+									headAsString,
+									itemAsString,
 									", " + PrintUtil.LINE_SEPARATOR,
 									terminator + ")" + PrintUtil.LINE_SEPARATOR + "Where " + whereForTerminator + ";" + PrintUtil.LINE_SEPARATOR);
 						}
@@ -694,8 +697,8 @@ public class DMLTransformer extends AbstractResultSetReader {
 							}
 							updateStatementBuilderTerminator = terminatorAsString;
 							updateStatementBuilder.append(
-									headAsString, 
-									itemAsString, 
+									headAsString,
+									itemAsString,
 									" union all " + PrintUtil.LINE_SEPARATOR,
 									terminatorAsString);
 						}
@@ -737,7 +740,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 					}
 					String item;
 					if (insertStatementBuilder.isEmpty()) {
-						item = PrintUtil.LINE_SEPARATOR + " Select " + namedValues + " From DUAL";	
+						item = PrintUtil.LINE_SEPARATOR + " Select " + namedValues + " From DUAL";
 					} else {
 						item = PrintUtil.LINE_SEPARATOR + " Select " + valueList + " From DUAL";
 					}
@@ -758,7 +761,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 					insertStatementBuilder.append(insertSchema, item, ", ", ";" + PrintUtil.LINE_SEPARATOR);
 				}
 			}
-			
+
 			exportLobs(table, resultSet, smallLobsPerIndex.keySet());
 		} catch (IOException e) {
 			throw new RuntimeException(e);
@@ -767,7 +770,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 
 	/**
 	 * Converts cell content to SQL literals.
-	 * 
+	 *
 	 * @param cellContentConverter converter
 	 * @param resultSet points to current row
 	 * @param i current result set index
@@ -776,15 +779,18 @@ public class DMLTransformer extends AbstractResultSetReader {
 	 */
 	protected String convertToSql(CellContentConverter cellContentConverter, ResultSet resultSet, int i, Object content, int callerId, String suffix) throws SQLException {
 		String cVal = cellContentConverter.toSql(content);
+		if (i > selectionClause.size()) {
+			throw new IllegalStateException("Table \"" + table.getName() + "\": Too many columns in the result set. Check the filter definitions.");
+		}
 		Column column = selectionClause.get(i - 1);
 		Filter filter = column.getFilter();
-		
+
 		if (filter != null && importFilterTransformer != null) {
 			if (!filter.isApplyAtExport()) {
 				return importFilterTransformer.transform(column, cVal);
 			}
 		}
-		
+
 		if (content != null && filter != null && filter.getExpression().trim().startsWith(Filter.LITERAL_PREFIX)) {
 			return content.toString();
 		}
@@ -802,7 +808,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 				return cVal + " /*" + filter.getReason() + "*/";
 			}
 		}
-		
+
 		return cVal;
 	}
 
@@ -810,7 +816,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 
 	/**
 	 * Gets qualified table name.
-	 * 
+	 *
 	 * @param t the table
 	 * @return qualified name of t
 	 */
@@ -828,7 +834,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 
 	/**
 	 * Checks if columns is part of primary key.
-	 * 
+	 *
 	 * @param column the column
 	 * @return <code>true</code> if column is part of primary key
 	 */
@@ -842,7 +848,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 
 	/**
 	 * Exports the (c|b)lob content.
-	 * 
+	 *
 	 * @param resultSet export current row
 	 */
 	private void exportLobs(Table table, ResultSet resultSet, Set<Integer> smallLobsIndexes) throws IOException, SQLException {
@@ -852,7 +858,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 				if (smallLobsIndexes.contains(lobColumnIndexes.get(i))) {
 					continue;
 				}
-				Object lob = resultSet.getObject(lobColumnIndexes.get(i));
+				Object lob = cellContentConverter.getObject(resultSet, lobColumnIndexes.get(i));
 				Map<String, String> val = new HashMap<String, String>();
 				for (int j = 1; j <= columnCount; ++j) {
 					if (columnLabel[j] == null) {
@@ -868,7 +874,9 @@ public class DMLTransformer extends AbstractResultSetReader {
 				boolean f = true;
 				StringBuffer where = new StringBuffer("");
 				if (table.getNonVirtualPKColumns(session).isEmpty()) {
-					throw new DataModel.NoPrimaryKeyException(table, "has no primary key. Update statement to import CLOB/BLOB/XML can not be generated.");
+					throw new DataModel.NoPrimaryKeyException(table, "has no " +
+							(table.primaryKey != null && table.primaryKey.getColumns() != null && !table.primaryKey.getColumns().isEmpty()? "non-virtual " : "") +
+							"primary key. Update statement to import CLOB/BLOB/XML can not be generated.");
 				}
 				for (Column pk: table.getNonVirtualPKColumns(session)) {
 					if (!f) {
@@ -976,7 +984,7 @@ public class DMLTransformer extends AbstractResultSetReader {
 			throw new RuntimeException(e);
 		}
 	}
-	
+
 	/**
 	 * Flushes the export-reader.
 	 */
@@ -994,12 +1002,12 @@ public class DMLTransformer extends AbstractResultSetReader {
 			}
 		}
 	}
-	
+
 	/**
 	 * Table which is currently enabled for identity-inserts.
 	 */
 	private static Table identityInsertTable = null;
-	
+
 	/**
 	 * Writes into script.
 	 */
@@ -1022,5 +1030,5 @@ public class DMLTransformer extends AbstractResultSetReader {
 			}
 		}
 	}
-	
+
 }

src/main/engine/net/sf/jailer/database/DefaultTemporaryTableManager.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@ package net.sf.jailer.database;
 
 /**
  * Default implementation of {@link TemporaryTableManager}.
- * 
+ *
  * @author Ralf Wisser
  */
 public class DefaultTemporaryTableManager implements TemporaryTableManager {
@@ -31,37 +31,40 @@ public class DefaultTemporaryTableManager implements TemporaryTableManager {
 	 * Prefix of references to a temporary table in DDL statements.
 	 */
 	private String ddlTableReferencePrefix = "";
-	
+
 	/**
 	 * Prefix of DDL statement to create temporary table.
 	 */
 	private String createTablePrefix;
-	
+
 	/**
 	 * Suffix of DDL statement to create temporary table.
 	 */
 	private String createTableSuffix;
-	
+
 	/**
 	 * Prefix of DDL statement to create temporary index.
 	 */
 	private String createIndexPrefix;
-	
+
 	/**
 	 * Suffix of DDL statement to create temporary index.
 	 */
 	private String createIndexSuffix;
-	
+
 	/**
 	 * Prefix of table name to be used in DDL for creating temporary index.
 	 */
 	private String indexTablePrefix;
-	
+
 	/**
 	 * Prefix to be used in DDL for dropping temporary tables.
 	 */
 	private String dropTablePrefix;
-	
+
+	// Whether temp tables are shared between sessions.
+	private boolean needsExclusiveAccess = true;
+
 	/**
 	 * Gets prefix of DDL statement to create temporary table.
 	 */
@@ -77,14 +80,14 @@ public class DefaultTemporaryTableManager implements TemporaryTableManager {
 	public String getIndexTablePrefix() {
 		return indexTablePrefix;
 	}
-	
+
 	/**
 	 * Sets prefix of table name to be used in DDL for creating temporary index.
 	 */
 	public void setIndexTablePrefix(String v) {
 		this.indexTablePrefix = v;
 	}
-	
+
 	/**
 	 * Gets suffix of DDL statement to create temporary table.
 	 */
@@ -92,7 +95,7 @@ public class DefaultTemporaryTableManager implements TemporaryTableManager {
 	public String getCreateTableSuffix() {
 		return createTableSuffix;
 	}
-	
+
 	/**
 	 * Gets prefix of DDL statement to create temporary index.
 	 */
@@ -129,7 +132,7 @@ public class DefaultTemporaryTableManager implements TemporaryTableManager {
 	public void setCreateTableSuffix(String v) {
 		createTableSuffix = v;
 	}
-	
+
 	/**
 	 * Gets prefix of DDL statement to create temporary index.
 	 */
@@ -143,7 +146,7 @@ public class DefaultTemporaryTableManager implements TemporaryTableManager {
 	public void setCreateIndexSuffix(String v) {
 		createIndexSuffix = v;
 	}
-	
+
 	/**
 	 * Gets prefix to be used in DDL for dropping temporary tables.
 	 */
@@ -174,9 +177,27 @@ public class DefaultTemporaryTableManager implements TemporaryTableManager {
 	public void setDdlTableReferencePrefix(String ddlTableReferencePrefix) {
 		this.ddlTableReferencePrefix = ddlTableReferencePrefix;
 	}
-	
+
 	public String getDdlTableReferencePrefix() {
 		return ddlTableReferencePrefix;
 	}
 
+	/**
+	 * Whether temp tables are shared between sessions.
+	 *
+	 * @return whether temp tables are shared between sessions
+	 */
+	public boolean isNeedsExclusiveAccess() {
+		return needsExclusiveAccess;
+	}
+
+	/**
+	 * Whether temp tables are shared between sessions.
+	 *
+	 * @param needsExclusiveAccess whether temp tables are shared between sessions
+	 */
+	public void setNeedsExclusiveAccess(boolean needsExclusiveAccess) {
+		this.needsExclusiveAccess = needsExclusiveAccess;
+	}
+
 }

src/main/engine/net/sf/jailer/database/DeletionTransformer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -118,7 +118,7 @@ public class DeletionTransformer extends AbstractResultSetReader {
 		public ResultSetReader create(Table table) throws SQLException {
 			return new DeletionTransformer(table, scriptFileWriter, maxBodySize, session, targetDBMSConfiguration, executionContext);
 		}
-	};
+	}
 
 	/**
 	 * Constructor.

src/main/engine/net/sf/jailer/database/ImportFilterTransformer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/database/InlineViewBuilder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/database/InlineViewStyle.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -86,7 +86,7 @@ public enum InlineViewStyle {
 			return sb.toString();
 		}
 	},
-	DB2("(values (1, '2', 3), (4, '5', 6)) %s(A, B, C)") {
+	Db2("(values (1, '2', 3), (4, '5', 6)) %s(A, B, C)") {
 		@Override
 		public String head(String[] columnNames) throws SQLException {
 			return "(values ";
@@ -293,6 +293,39 @@ public enum InlineViewStyle {
 			StringBuilder sb = new StringBuilder(" from sysmaster:\"informix\".sysdual) " + name);
 			return sb.toString();
 		}
+	},
+	FIREBIRD("(Select 1 A, '2' B, 3 C from RDB$DATABASE Union all "
+			+ "Select 4, '5', 6 from RDB$DATABASE) %s") {
+		@Override
+		public String head(String[] columnNames) throws SQLException {
+			return "(Select ";
+		}
+
+		@Override
+		public String item(String[] values, String[] columnNames, int rowNumber) throws SQLException {
+			StringBuilder sb = new StringBuilder();
+			for (int i = 1; i <= columnNames.length; ++i) {
+				if (i > 1) {
+					sb.append(", ");
+				}
+				sb.append(values[i - 1]);
+				if (rowNumber == 0) {
+					sb.append(" " + columnNames[i - 1]);
+				}
+			}
+			return sb.toString();
+		}
+
+		@Override
+		public String separator() throws SQLException {
+			return " from RDB$DATABASE Union all Select ";
+		}
+
+		@Override
+		public String terminator(String name, String[] columnNames) throws SQLException {
+			StringBuilder sb = new StringBuilder(" from RDB$DATABASE) " + name);
+			return sb.toString();
+		}
 	};
 
 	public final String example;

src/main/engine/net/sf/jailer/database/LocalDatabase.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -50,14 +50,26 @@ public class LocalDatabase {
 		this.databaseFolder = new File(Configuration.getInstance().getTempFileFolder() + File.separator + UUID.randomUUID().toString()).getAbsolutePath();
 		new File(databaseFolder).mkdirs();
 		BasicDataSource dataSource;
+		URL[] urlArray;
+		FileNotFoundException urlException = null;
 		try {
-			dataSource = new BasicDataSource(driverClassName, urlPattern.replace("%s", databaseFolder + File.separator + "local"), user, password, 0, new URL[0]);
-		} catch (Exception e) {
-			dataSource = new BasicDataSource(driverClassName, urlPattern.replace("%s", databaseFolder + File.separator + "local"), user, password, 0, ClasspathUtil.toURLArray(jarfile, null, null, null));
+			urlArray = ClasspathUtil.toURLArray(jarfile, null, null, null);
+		} catch (FileNotFoundException e) {
+			urlArray = new URL[0];
+			urlException = new FileNotFoundException(e.getMessage() + ". This file is required for using WorkingTableScope.LOCAL_DATABASE. Make the file available or add it to the classpath.");
+		}
+		if (urlException != null) {
+			try {
+				dataSource = new BasicDataSource(driverClassName, urlPattern.replace("%s", databaseFolder + File.separator + "local"), user, password, 0, urlArray);
+			} catch (Exception e) {
+				throw urlException;
+			}
+		} else {
+			dataSource = new BasicDataSource(driverClassName, urlPattern.replace("%s", databaseFolder + File.separator + "local"), user, password, 0, urlArray);
 		}
 		session = new Session(dataSource, dataSource.dbms, Connection.TRANSACTION_READ_UNCOMMITTED, null, false, true);
 	}
-	
+
 	/**
 	 * Shut local database down. Remove all database files.
 	 */

src/main/engine/net/sf/jailer/database/PrimaryKeyValidator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -35,20 +35,20 @@ import net.sf.jailer.util.Quoting;
 
 /**
  * Validates all primary keys of a set of tables.
- * 
+ *
  * @author Ralf Wisser
  */
 public abstract class PrimaryKeyValidator {
 
 	private final Object cancellationContext;
-	
+
 	public PrimaryKeyValidator(Object cancellationContext) {
 		this.cancellationContext = cancellationContext;
 	}
 
 	/**
 	 * Validates all primary keys of a set of tables.
-	 * 
+	 *
 	 * @param session the session
 	 * @param tables the tables
 	 * @throws SQLException if a pk is invalid
@@ -72,7 +72,6 @@ public abstract class PrimaryKeyValidator {
 			try {
 				ResultSet resultSet = JDBCMetaDataBasedModelElementFinder.getPrimaryKeys(
 						session,
-						session.getMetaData(),
 						Quoting.staticUnquote(table.getSchema(defaultSchema)),
 						Quoting.staticUnquote(table.getUnqualifiedName()),
 						true);
@@ -101,21 +100,15 @@ public abstract class PrimaryKeyValidator {
 				jobListToAddTo = jobsUDPK;
 			}
 
-			jobListToAddTo.add(new JobManager.Job() {
-				@Override
-				public void run() throws SQLException {
-					checkUniqueness(session, table, Quoting.getQuoting(session));
-					numDone.getAndIncrement();
-					updateProgressBar();
-				}
+			jobListToAddTo.add(() -> {
+				checkUniqueness(session, table, Quoting.getQuoting(session));
+				numDone.getAndIncrement();
+				updateProgressBar();
 			});
-			jobListToAddTo.add(new JobManager.Job() {
-				@Override
-				public void run() throws SQLException {
-					checkNoNull(session, table, Quoting.getQuoting(session));
-					numDone.getAndIncrement();
-					updateProgressBar();
-				}
+			jobListToAddTo.add(() -> {
+				checkNoNull(session, table, Quoting.getQuoting(session));
+				numDone.getAndIncrement();
+				updateProgressBar();
 			});
 		}
 
@@ -135,8 +128,16 @@ public abstract class PrimaryKeyValidator {
 		CancellationHandler.checkForCancellation(cancellationContext);
 	}
 
-	private void throwIfErrorFound() throws SqlException {
+	private synchronized void throwIfErrorFound() throws SqlException {
+		errorMessage.append(errorMessageLowPrio);
+		errorStatements.append(errorStatementsLowPrio);
+		errorMessageLowPrio.setLength(0);
+		errorStatementsLowPrio.setLength(0);
 		if (errorMessage.length() > 0) {
+			if (numErrors.get() == 1) {
+				errorMessage = new StringBuilder(errorMessage.toString().replaceFirst("1\\. ", ""));
+				errorStatements = new StringBuilder(errorStatements.toString().replaceFirst("1\\. ", ""));
+			}
 			SqlException e = new SqlException("Invalid Primary Key", errorMessage.toString(), errorStatements.toString(), null);
 			e.setFormatted(true);
 			throw e;
@@ -157,11 +158,11 @@ public abstract class PrimaryKeyValidator {
 			session.executeQuery(sql, new Session.AbstractResultSetReader() {
 				@Override
 				public void readCurrentRow(ResultSet resultSet) throws SQLException {
-					addError("Primary key of table \"" + table.getName() + "\" is not unique.", sql.toString());
+					addError(null, "Primary key of table \"" + table.getName() + "\" is not unique.", sql.toString());
 				}
 			}, null, cancellationContext, 1, true);
 		} catch (SqlException e) {
-			addError("Table \"" + table.getName() + "\": " + e.message, sql.toString());
+			addError(table, "Table \"" + table.getName() + "\": " + e.message, sql.toString());
 		}
 	}
 
@@ -182,28 +183,41 @@ public abstract class PrimaryKeyValidator {
 				session.executeQuery(sql, new Session.AbstractResultSetReader() {
 					@Override
 					public void readCurrentRow(ResultSet resultSet) throws SQLException {
-						addError("Primary key of table \"" + table.getName() + "\" contains null.", sql.toString());
+						addError(null, "Primary key of table \"" + table.getName() + "\" contains null.", sql.toString());
 					}
 				}, null, cancellationContext, 1, true);
 			} catch (SqlException e) {
-				addError("Table \"" + table.getName() + "\": " + e.message, sql.toString());
+				addError(table, "Table \"" + table.getName() + "\": " + e.message, sql.toString());
 			}
 		}
 	}
 
 	private StringBuilder errorMessage = new StringBuilder();
 	private StringBuilder errorStatements = new StringBuilder();
+	private StringBuilder errorMessageLowPrio = new StringBuilder();
+	private StringBuilder errorStatementsLowPrio = new StringBuilder();
 
+	private Set<Table> errTables = new HashSet<Table>();
+	
 	protected AtomicInteger numErrors = new AtomicInteger();
 	protected AtomicInteger numDone = new AtomicInteger();
 	protected AtomicInteger numTotal = new AtomicInteger();
-	
+
 	protected abstract void updateProgressBar();
-	
-	private void addError(String message, String sql) {
-		errorMessage.append("- " + message + "\n");
-		errorStatements.append("- " + sql + "\n");
-		numErrors.getAndIncrement();
+
+	private synchronized void addError(Table table, String message, String sql) {
+		if (table != null) {
+			if (!errTables.add(table)) {
+				return;
+			}
+			int numError = 1 + numErrors.getAndIncrement();
+			errorMessageLowPrio.append(numError + ". " + message + "\n");
+			errorStatementsLowPrio.append(numError + ". " + sql + "\n");
+		} else {
+			int numError = 1 + numErrors.getAndIncrement();
+			errorMessage.append(numError + ". " + message + "\n");
+			errorStatements.append(numError + ". " + sql + "\n");
+		}
 		updateProgressBar();
 	}
 

src/main/engine/net/sf/jailer/database/SQLDialect.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/database/Session.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -43,14 +43,17 @@ import javax.sql.DataSource;
 
 import org.apache.log4j.Logger;
 
+import net.sf.jailer.configuration.Configuration;
 import net.sf.jailer.configuration.DBMS;
+import net.sf.jailer.util.CancellationException;
 import net.sf.jailer.util.CancellationHandler;
 import net.sf.jailer.util.CellContentConverter;
+import net.sf.jailer.util.LogUtil;
 
 /**
  * Manages database sessions on a 'per thread' basis.
  * Executes SQL-Statements in the context of a session.
- * 
+ *
  * @author Ralf Wisser
  */
 public class Session {
@@ -59,43 +62,48 @@ public class Session {
 	 * Hold a connection for each thread.
 	 */
 	protected ThreadLocal<Connection> connection = new ThreadLocal<Connection>();
-	
+
 	/**
 	 * Holds all connections.
 	 */
 	private final List<Connection> connections = Collections.synchronizedList(new ArrayList<Connection>());
-	
+
 	/**
 	 * The session in which temporary tables lives, if any.
 	 */
 	private Connection temporaryTableSession = null;
-		
+
 	/**
 	 * Shared scope of temporary tables.
 	 */
 	private WorkingTableScope temporaryTableScope;
-	
+
 	/**
 	 * Scope of temporary tables.
 	 */
 	public final WorkingTableScope scope;
-	
+
 	/**
-	 * No SQL-Exceptions will be logged in silent mode. 
+	 * No SQL-Exceptions will be logged in silent mode.
 	 */
 	private boolean silent = false;
 	
+	/**
+	 * Last known activity time per connection.
+	 */
+	private Map<Connection, Long> lastConnectionActiviyTimeStamp = Collections.synchronizedMap(new HashMap<Connection, Long>());
+
 	private final boolean transactional;
 	public final boolean local;
-	
+
 	/**
 	 * Reads a JDBC-result-set.
 	 */
 	public interface ResultSetReader {
-	
+
 		/**
 		 * Reads current row of a result-set.
-		 * 
+		 *
 		 * @param resultSet the result-set
 		 */
 		void readCurrentRow(ResultSet resultSet) throws SQLException;
@@ -105,26 +113,26 @@ public class Session {
 		 */
 		void close() throws SQLException;
 	}
-	
+
 	/**
 	 * Reads a JDBC-result-set.
 	 * Caches a {@link ResultSetMetaData}.
 	 */
 	public static abstract class AbstractResultSetReader implements ResultSetReader {
-	
+
 		private ResultSet owner;
 		private ResultSetMetaData metaData;
 
 		private ResultSet cccOwner;
 		private Session cccSession;
 		private CellContentConverter cellContentConverter;
-		
+
 		/**
 		 * Gets and cache meta data of a result set.
-		 * 
+		 *
 		 * @param resultSet
 		 * @return meta data of resultSet
-		 * @throws SQLException 
+		 * @throws SQLException
 		 */
 		protected ResultSetMetaData getMetaData(ResultSet resultSet) throws SQLException {
 			if (owner == resultSet) {
@@ -134,13 +142,13 @@ public class Session {
 			metaData = resultSet.getMetaData();
 			return metaData;
 		}
-		
+
 		/**
 		 * Gets and cache CellContentConverter for the result set.
-		 * 
+		 *
 		 * @param resultSet
 		 * @return meta data of resultSet
-		 * @throws SQLException 
+		 * @throws SQLException
 		 */
 		protected CellContentConverter getCellContentConverter(ResultSet resultSet, Session session, DBMS targetDBMSConfiguration) throws SQLException {
 			if (cccOwner == resultSet && cccSession == session) {
@@ -154,7 +162,7 @@ public class Session {
 
 		/**
 		 * Does nothing.
-		 * @throws SQLException 
+		 * @throws SQLException
 		 */
 		@Override
 		public void close() throws SQLException {
@@ -162,15 +170,15 @@ public class Session {
 
 		/**
 		 * Initializes the reader.
-		 * 
+		 *
 		 * @param resultSet the result set to read.
 		 * @throws SQLException
 		 */
 		public void init(ResultSet resultSet) throws SQLException {
 		}
-		
+
 	}
-	
+
 	/**
 	 * The logger.
 	 */
@@ -181,8 +189,8 @@ public class Session {
 	 */
 	public interface ConnectionFactory {
 		Connection getConnection() throws SQLException;
-	};
-	
+	}
+
 	/**
 	 * Connection factory.
 	 */
@@ -202,20 +210,20 @@ public class Session {
 	 * The DBMS.
 	 */
 	public final DBMS dbms;
-	
+
 	/**
 	 * The DBMS.
 	 */
 	public final String driverClassName;
-	
+
 	/**
 	 * The dbUrl (<code>null</code> if unknown)
 	 */
 	public final String dbUrl;
-	
+
 	/**
 	 * Constructor.
-	 * 
+	 *
 	 * @param dataSource the data source
 	 * @param dbms the DBMS
 	 */
@@ -225,7 +233,7 @@ public class Session {
 
 	/**
 	 * Constructor.
-	 * 
+	 *
 	 * @param dataSource the data source
 	 * @param dbms the DBMS
 	 */
@@ -235,7 +243,7 @@ public class Session {
 
 	/**
 	 * Constructor.
-	 * 
+	 *
 	 * @param dataSource the data source
 	 * @param dbms the DBMS
 	 * @param local <code>true</code> for the local entity-graph database
@@ -255,7 +263,31 @@ public class Session {
 			private Random random = new Random();
 			@Override
 			public synchronized Connection getConnection() throws SQLException {
-				@SuppressWarnings("resource")
+				Connection con = getConnection0();
+				Long ts = lastConnectionActiviyTimeStamp.get(con);
+				releaseConnection(con);
+				if (ts != null && con != null && con == connection.get() && con.getAutoCommit() && !Session.this.transactional) {
+					long idleTime = System.currentTimeMillis() - ts;
+					long databaseConnectionInteractiveTimeout = Configuration.getInstance().getDatabaseConnectionInteractiveTimeout() * 1000L;
+
+					if (idleTime >= databaseConnectionInteractiveTimeout) {
+						boolean valid;
+						try {
+							valid = con.isValid(4);
+						} catch (Throwable t) {
+							LogUtil.warn(t);
+							valid = true;
+						}
+						if (!valid) {
+							LogUtil.warn(new RuntimeException("invalid connection, reconnecting (" + idleTime + ")"));
+							reconnect();
+							return getConnection0();
+						}
+					}
+				}
+				return con;
+			}
+			private Connection getConnection0() throws SQLException {
 				Connection con = local? connection.get() : temporaryTableSession == null? connection.get() : temporaryTableSession;
 
 				if (con == null && Boolean.TRUE.equals(sharesConnection.get())) {
@@ -321,6 +353,16 @@ public class Session {
 		init();
 	}
 
+	/**
+	 * Releases a connection get from {@link #getConnection()}.
+	 * Indicated that the connection is no longer in use for the time being.
+	 * 
+	 * @param con the connection
+	 */
+	public void releaseConnection(Connection con) {
+		lastConnectionActiviyTimeStamp.put(con, System.currentTimeMillis());
+	}
+
 	protected void init() throws SQLException {
 		Connection connection = connectionFactory.getConnection();
 		logDriverInfo(connection);
@@ -368,12 +410,12 @@ public class Session {
 			temporaryTableSession = null;
 		}
 	}
-	
+
 	private Object silentLock = new Object();
-	
+
 	/**
 	 * No SQL-Exceptions will be logged in silent mode.
-	 * 
+	 *
 	 * @param silent <code>true</code> for silence
 	 */
 	public void setSilent(boolean silent) {
@@ -381,10 +423,10 @@ public class Session {
 			this.silent = silent;
 		}
 	}
-	
+
 	/**
 	 * No SQL-Exceptions will be logged in silent mode.
-	 * 
+	 *
 	 * @return silent <code>true</code> for silence
 	 */
 	public boolean getSilent() {
@@ -394,14 +436,14 @@ public class Session {
 	}
 
 	private static final ThreadLocal<Boolean> logStatements = new ThreadLocal<Boolean>();
-	
+
 	/**
 	 * Log statements?
 	 */
 	public void setLogStatements(boolean logStatements) {
 		Session.logStatements.set(logStatements);
 	}
-	
+
 	/**
 	 * Log statements?
 	 */
@@ -411,7 +453,7 @@ public class Session {
 
 	/**
 	 * Logs driver info
-	 * 
+	 *
 	 * @param connection connection to DB
 	 * @return the DBMS
 	 */
@@ -429,7 +471,7 @@ public class Session {
 
 	/**
 	 * Gets DB schema name.
-	 * 
+	 *
 	 * @return DB schema name (empty string if unknown)
 	 */
 	public String getSchema() {
@@ -438,7 +480,7 @@ public class Session {
 
 	/**
 	 * Executes a SQL-Query (SELECT).
-	 * 
+	 *
 	 * @param sqlQuery the query in SQL
 	 * @param reader the reader for the result
 	 * @param withExplicitCommit if <code>true</code>, switch of autocommit and commit explicitly
@@ -446,20 +488,20 @@ public class Session {
 	public long executeQuery(String sqlQuery, ResultSetReader reader, boolean withExplicitCommit) throws SQLException {
 		return executeQuery(sqlQuery, reader, null, null, 0, withExplicitCommit);
 	}
-	
+
 	/**
 	 * Executes a SQL-Query (SELECT).
-	 * 
+	 *
 	 * @param sqlQuery the query in SQL
 	 * @param reader the reader for the result
 	 */
 	public long executeQuery(String sqlQuery, ResultSetReader reader) throws SQLException {
 		return executeQuery(sqlQuery, reader, null, null, 0, false);
 	}
-	
+
 	/**
 	 * Executes a SQL-Query (SELECT).
-	 * 
+	 *
 	 * @param sqlQuery the query in SQL
 	 * @param reader the reader for the result
 	 * @param alternativeSQL query to be executed if sqlQuery fails
@@ -473,7 +515,7 @@ public class Session {
 
 	/**
 	 * Executes a SQL-Query (SELECT).
-	 * 
+	 *
 	 * @param sqlQuery the query in SQL
 	 * @param reader the reader for the result
 	 * @param alternativeSQL query to be executed if sqlQuery fails
@@ -486,7 +528,7 @@ public class Session {
 
 	/**
 	 * Executes a SQL-Query (SELECT) with timeout.
-	 * 
+	 *
 	 * @param theConnection connection to use
 	 * @param sqlQuery the query in SQL
 	 * @param reader the reader for the result
@@ -497,6 +539,28 @@ public class Session {
 	 * @param withExplicitCommit if <code>true</code>, switch of autocommit and commit explicitly
 	 */
 	private long executeQuery(Connection theConnection, String sqlQuery, ResultSetReader reader, String alternativeSQL, Object context, long limit, int timeout, boolean withExplicitCommit) throws SQLException {
+		if (!transactional) {
+			synchronized (theConnection) {
+				return executeQuery0(theConnection, sqlQuery, reader, alternativeSQL, context, limit, timeout, withExplicitCommit);
+			}
+		} else {
+			return executeQuery0(theConnection, sqlQuery, reader, alternativeSQL, context, limit, timeout, withExplicitCommit);
+		}
+	}
+
+	/**
+	 * Executes a SQL-Query (SELECT) with timeout.
+	 *
+	 * @param theConnection connection to use
+	 * @param sqlQuery the query in SQL
+	 * @param reader the reader for the result
+	 * @param alternativeSQL query to be executed if sqlQuery fails
+	 * @param context cancellation context
+	 * @param limit row limit, 0 for unlimited
+	 * @param timeout the timeout in sec
+	 * @param withExplicitCommit if <code>true</code>, switch of autocommit and commit explicitly
+	 */
+	private long executeQuery0(Connection theConnection, String sqlQuery, ResultSetReader reader, String alternativeSQL, Object context, long limit, int timeout, boolean withExplicitCommit) throws SQLException {
 		if (withExplicitCommit) {
 			synchronized (theConnection) {
 				if (theConnection.getAutoCommit()) {
@@ -519,17 +583,24 @@ public class Session {
 		long startTime = System.currentTimeMillis();
 		Statement statement = null;
 		try {
+			final String woSuffix = " /*!*/";
+			boolean wo = sqlQuery.endsWith(woSuffix);
+			if (wo) {
+				sqlQuery = sqlQuery.substring(0, sqlQuery.length() - woSuffix.length());
+			}
 			statement = theConnection.createStatement();
 			if (dbms != null) {
 				if (dbms.getFetchSize() != null) {
-					try {
-						statement.setFetchSize(dbms.getFetchSize());
-					} catch (Throwable t) {
-						// ignore
+					if (!wo || !DBMS.MySQL.equals(dbms)) {
+						try {
+							statement.setFetchSize(dbms.getFetchSize());
+						} catch (Throwable t) {
+							// ignore
+						}
 					}
 				}
 			}
-			CancellationHandler.begin(statement, context);
+			begin(statement, context);
 			ResultSet resultSet;
 			try {
 				if (limit > 0 && limit < Integer.MAX_VALUE - 1) {
@@ -544,10 +615,12 @@ public class Session {
 				}
 				resultSet = statement.executeQuery(sqlQuery);
 			} catch (SQLException e) {
+				checkKilled();
+				CancellationHandler.checkForCancellation(context);
+
 				if (alternativeSQL != null) {
 					_log.warn("query failed, using alternative query. Reason: " + e.getMessage());
 					_log.info(alternativeSQL);
-					CancellationHandler.checkForCancellation(context);
 					resultSet = statement.executeQuery(alternativeSQL);
 				} else {
 					throw e;
@@ -560,6 +633,7 @@ public class Session {
 				reader.readCurrentRow(resultSet);
 				++rc;
 				if (rc % 100 == 0) {
+					checkKilled();
 					CancellationHandler.checkForCancellation(context);
 				}
 				if (limit > 0 && rc >= limit) {
@@ -575,7 +649,7 @@ public class Session {
 				} catch (SQLException e) {
 					// ignore
 				}
-				CancellationHandler.end(statement, context);
+				end(statement, context);
 			}
 		}
 		if (getLogStatements()) {
@@ -586,7 +660,7 @@ public class Session {
 
 	/**
 	 * Executes a SQL-Query (SELECT) with timeout.
-	 * 
+	 *
 	 * @param sqlQuery the query in SQL
 	 * @param reader the reader for the result
 	 * @param alternativeSQL query to be executed if sqlQuery fails
@@ -600,7 +674,10 @@ public class Session {
 			_log.info(sqlQuery);
 		}
 		try {
-			return executeQuery(connectionFactory.getConnection(), sqlQuery, reader, alternativeSQL, context, limit, timeout, withExplicitCommit);
+			Connection con = connectionFactory.getConnection();
+			long result = executeQuery(con, sqlQuery, reader, alternativeSQL, context, limit, timeout, withExplicitCommit);
+			releaseConnection(con);
+			return result;
 		} catch (SQLException e) {
 			CancellationHandler.checkForCancellation(context);
 			if (!silent) {
@@ -609,13 +686,17 @@ public class Session {
 			if (e instanceof SqlException) {
 				throw e;
 			}
+			final String woSuffix = " /*!*/";
+			if (sqlQuery != null && sqlQuery.endsWith(woSuffix)) {
+				sqlQuery = sqlQuery.substring(0, sqlQuery.length() - woSuffix.length());
+			}
 			throw new SqlException("\"" + e.getMessage() + "\" in statement \"" + sqlQuery + "\"", sqlQuery, e);
 		}
 	}
 
 	/**
 	 * Executes a SQL-Query (SELECT).
-	 * 
+	 *
 	 * @param sqlFile file containing a query in SQL
 	 * @param reader the reader for the result
 	 * @param withExplicitCommit if <code>true</code>, switch of autocommit and commit explicitly
@@ -647,9 +728,9 @@ public class Session {
 
 	/**
 	 * Executes a SQL-Update (INSERT, DELETE or UPDATE).
-	 * 
+	 *
 	 * @param sqlUpdate the update in SQL
-	 * 
+	 *
 	 * @return update-count
 	 */
 	public int executeUpdate(String sqlUpdate) throws SQLException {
@@ -667,8 +748,9 @@ public class Session {
 				long startTime = System.currentTimeMillis();
 				Statement statement = null;
 				try {
-					statement = connectionFactory.getConnection().createStatement();
-					CancellationHandler.begin(statement, null);
+					Connection con = connectionFactory.getConnection();
+					statement = con.createStatement();
+					begin(statement, null);
 					if (serializeAccess) {
 						boolean acquired;
 						try {
@@ -703,14 +785,16 @@ public class Session {
 						}
 					}
 
-					CancellationHandler.end(statement, null);
+					end(statement, null);
+					releaseConnection(con);
 					ok = true;
 					if (getLogStatements()) {
 						_log.info("" + rowCount + " row(s) in " + (System.currentTimeMillis() - startTime) + " ms");
 					}
 				} catch (SQLException e) {
+					checkKilled();
 					CancellationHandler.checkForCancellation(null);
-					CancellationHandler.end(statement, null);
+					end(statement, null);
 
 					boolean isRetrieable = isRetrieable(e);
 					if (++failures > MAXIMUM_NUMBER_OF_FAILURES || !isRetrieable) {
@@ -745,33 +829,35 @@ public class Session {
 			throw e;
 		}
 	}
-	
+
 	/**
 	 * Executes a SQL-Update (INSERT, DELETE or UPDATE) with parameters.
-	 * 
+	 *
 	 * @param sqlUpdate the update in SQL
 	 * @param parameter the parameters
-	 * 
+	 *
 	 * @return update-count
 	 */
 	public int executeUpdate(String sqlUpdate, Object[] parameter) throws SQLException {
 		if (getLogStatements()) {
 			_log.info(sqlUpdate);
 		}
+		PreparedStatement statement = null;
 		try {
 			CancellationHandler.checkForCancellation(null);
 			int rowCount = 0;
 			long startTime = System.currentTimeMillis();
-			PreparedStatement statement = null;
 			try {
-				statement = connectionFactory.getConnection().prepareStatement(sqlUpdate);
-				CancellationHandler.begin(statement, null);
+				Connection con = connectionFactory.getConnection();
+				statement = con.prepareStatement(sqlUpdate);
+				begin(statement, null);
 				int i = 1;
 				for (Object p: parameter) {
 					statement.setObject(i++, p);
 				}
 				rowCount = statement.executeUpdate();
-				CancellationHandler.end(statement, null);
+				end(statement, null);
+				releaseConnection(con);
 				if (getLogStatements()) {
 					_log.info("" + rowCount + " row(s) in " + (System.currentTimeMillis() - startTime) + " ms");
 				}
@@ -782,6 +868,7 @@ public class Session {
 			}
 			return rowCount;
 		} catch (SQLException e) {
+			checkKilled();
 			CancellationHandler.checkForCancellation(null);
 			if (!silent) {
 				_log.error("Error executing statement", e);
@@ -801,19 +888,20 @@ public class Session {
 		PreparedStatement statement = null;
 		try {
 			statement = connectionFactory.getConnection().prepareStatement(sqlUpdate);
-			CancellationHandler.begin(statement, null);
+			begin(statement, null);
 			InputStreamReader inputStreamReader = new InputStreamReader(new FileInputStream(lobFile), "UTF-8");
 			statement.setCharacterStream(1, inputStreamReader, (int) length);
 			statement.execute();
 			inputStreamReader.close();
 		} catch (SQLException e) {
+			checkKilled();
 			CancellationHandler.checkForCancellation(null);
 			throw e;
 		} finally {
 			if (statement != null) {
 				try {
 					statement.close();
-					CancellationHandler.end(statement, null);
+					end(statement, null);
 				} catch (SQLException e) {
 				}
 			}
@@ -829,19 +917,20 @@ public class Session {
 		PreparedStatement statement = null;
 		try {
 			statement = connectionFactory.getConnection().prepareStatement(sqlUpdate);
-			CancellationHandler.begin(statement, null);
+			begin(statement, null);
 			InputStreamReader inputStreamReader = new InputStreamReader(new FileInputStream(lobFile), "UTF-8");
 			statement.setCharacterStream(1, inputStreamReader, (int) length);
 			statement.execute();
 			inputStreamReader.close();
 		} catch (SQLException e) {
+			checkKilled();
 			CancellationHandler.checkForCancellation(null);
 			throw e;
 		} finally {
 			if (statement != null) {
 				try {
 					statement.close();
-					CancellationHandler.end(statement, null);
+					end(statement, null);
 				} catch (SQLException e) {
 				}
 			}
@@ -857,19 +946,20 @@ public class Session {
 		PreparedStatement statement = null;
 		try {
 			statement = connectionFactory.getConnection().prepareStatement(sqlUpdate);
-			CancellationHandler.begin(statement, null);
+			begin(statement, null);
 			FileInputStream fileInputStream = new FileInputStream(lobFile);
 			statement.setBinaryStream(1, fileInputStream, (int) lobFile.length());
 			statement.execute();
 			fileInputStream.close();
 		} catch (SQLException e) {
+			checkKilled();
 			CancellationHandler.checkForCancellation(null);
 			throw e;
 		} finally {
 			if (statement != null) {
 				try {
 					statement.close();
-					CancellationHandler.end(statement, null);
+					end(statement, null);
 				} catch (SQLException e) {
 				}
 			}
@@ -878,7 +968,7 @@ public class Session {
 
 	/**
 	 * Executes a SQL-Statement without returning any result.
-	 * 
+	 *
 	 * @param sql the SQL-Statement
 	 */
 	public long execute(String sql) throws SQLException {
@@ -887,7 +977,7 @@ public class Session {
 
 	/**
 	 * Executes a SQL-Statement without returning any result.
-	 * 
+	 *
 	 * @param sql the SQL-Statement
 	 */
 	public long execute(String sql, Object cancellationContext, boolean acceptQueries) throws SQLException {
@@ -905,8 +995,9 @@ public class Session {
 				long startTime = System.currentTimeMillis();
 				Statement statement = null;
 				try {
-					statement = connectionFactory.getConnection().createStatement();
-					CancellationHandler.begin(statement, null);
+					Connection con = connectionFactory.getConnection();
+					statement = con.createStatement();
+					begin(statement, null);
 					if (serializeAccess) {
 						boolean acquired;
 						try {
@@ -957,14 +1048,16 @@ public class Session {
 						}
 					}
 
-					CancellationHandler.end(statement, null);
+					end(statement, null);
+					releaseConnection(con);
 					ok = true;
 					if (getLogStatements()) {
 						_log.info("" + rowCount + " row(s) in " + (System.currentTimeMillis() - startTime) + " ms");
 					}
 				} catch (SQLException e) {
+					checkKilled();
 					CancellationHandler.checkForCancellation(null);
-					CancellationHandler.end(statement, null);
+					end(statement, null);
 
 					boolean isRetrieable = isRetrieable(e);
 					if (++failures > MAXIMUM_NUMBER_OF_FAILURES || !isRetrieable) {
@@ -1004,25 +1097,31 @@ public class Session {
 		String sqlState = e.getSQLState();
 		boolean deadlock = sqlState != null && sqlState.matches("40.01"); // "serialization failure", see https://en.wikipedia.org/wiki/SQLSTATE
 		boolean crf = DBMS.ORACLE.equals(dbms) && e.getErrorCode() == 8176; // ORA-08176: consistent read failure; rollback data not available
-		
+
 		boolean isRetrieable = deadlock || crf;
 		return isRetrieable;
 	}
-	
+
 	/**
 	 * Cached Database Meta Data.
 	 */
 	private Map<Connection, DatabaseMetaData> metaData = Collections.synchronizedMap(new IdentityHashMap<Connection, DatabaseMetaData>());
-	
+
+	private boolean checkMetaData = true;
+
+	public void disableMetaDataChecking() {
+		checkMetaData = false;
+	}
+
 	/**
 	 * Gets DB meta data.
-	 * 
+	 *
 	 * @return DB meta data
 	 */
 	public DatabaseMetaData getMetaData() throws SQLException {
 		Connection con = connectionFactory.getConnection();
 		DatabaseMetaData mData = metaData.get(con);
-		if (mData != null) {
+		if (mData != null && checkMetaData) {
 			try {
 				// is meta data still valid?
 				mData.getIdentifierQuoteString();
@@ -1060,6 +1159,52 @@ public class Session {
 		return down.get();
 	}
 
+	/**
+	 * Cancels all currently running statements.
+	 */
+	public synchronized void killRunningStatements() {
+		final IdentityHashMap<Statement, Statement> toBeCanceled = new IdentityHashMap<Statement, Statement>(runningStatements);
+		runningStatements.clear();
+		++currentVersion;
+		for (final IdentityHashMap.Entry<Statement, Statement> statement: toBeCanceled.entrySet()) {
+			Thread thread = new Thread(new Runnable() {
+				@Override
+				public void run() {
+					try {
+						statement.getKey().cancel();
+					} catch (Exception e) {
+						// ignore
+					}
+				}
+			});
+			thread.setDaemon(true);
+			thread.start();
+		}
+	}
+
+	private long currentVersion = 0;
+	private static ThreadLocal<Long> runningVersion = new ThreadLocal<Long>();
+	private Map<Statement, Statement> runningStatements = new IdentityHashMap<Statement, Statement>();
+
+	private synchronized void begin(Statement statement, Object context) {
+		CancellationHandler.begin(statement, context);
+		runningStatements.put(statement, statement);
+		runningVersion.set(currentVersion);
+	}
+
+	private synchronized void end(Statement statement, Object context) {
+		CancellationHandler.end(statement, context);
+		runningStatements.remove(statement);
+		runningVersion.set(null);
+	}
+
+	private synchronized void checkKilled() {
+		Long v = runningVersion.get();
+		if (v != null && v != currentVersion) {
+			throw new CancellationException();
+		}
+	}
+
 	/**
 	 * Rolls back and closes all connections.
 	 */
@@ -1085,7 +1230,7 @@ public class Session {
 		 }
 		connection = new ThreadLocal<Connection>();
 	}
-	
+
 	/**
 	 * Commits all connections.
 	 */
@@ -1105,25 +1250,25 @@ public class Session {
 			}
 		 }
 	}
-	
+
 	/**
 	 * Gets optional schema for database analysis.
-	 * 
+	 *
 	 * @return optional schema for database analysis
 	 */
 	public String getIntrospectionSchema() {
 		return introspectionSchema;
 	}
-	
+
 	/**
 	 * Sets optional schema for database analysis.
-	 * 
+	 *
 	 * @param introspectionSchema optional schema for database analysis
 	 */
 	public void setIntrospectionSchema(String introspectionSchema) {
 		this.introspectionSchema = introspectionSchema;
 	}
-	
+
 	/**
 	 * Closes the session in which temporary tables lives, if any.
 	 */
@@ -1145,14 +1290,14 @@ public class Session {
 	 * CLI connection arguments (UI support)
 	 */
 	private List<String> cliArguments;
-	
+
 	/**
 	 * Connection password (UI support)
 	 */
 	private String password;
-	
+
 	private final Object CLI_LOCK = new String("CLI_LOCK");
-	
+
 	/**
 	 * Gets connection password (UI support)
 	 */
@@ -1170,7 +1315,7 @@ public class Session {
 			this.password = password;
 		}
 	}
-	
+
 	/**
 	 * Sets CLI connection arguments (UI support)
 	 */
@@ -1179,7 +1324,7 @@ public class Session {
 			this.cliArguments = args;
 		}
 	}
-	
+
 	/**
 	 * Gets CLI connection arguments (UI support)
 	 */
@@ -1191,19 +1336,19 @@ public class Session {
 
 	/**
 	 * Gets the connection for the current thread.
-	 * 
+	 *
 	 * @return the connection for the current thread
 	 */
 	public Connection getConnection() throws SQLException {
 		return connectionFactory.getConnection();
 	}
-	
+
 	private InlineViewStyle inlineViewStyle;
 	private boolean noInlineViewStyleFound = false;
-	
+
 	/**
 	 * Returns a suitable {@link InlineViewStyle} for this session.
-	 * 
+	 *
 	 * @return a suitable {@link InlineViewStyle} for this session or <code>null</code>, if no style is found
 	 */
 	public synchronized InlineViewStyle getInlineViewStyle() {
@@ -1217,12 +1362,12 @@ public class Session {
 		}
 		return inlineViewStyle;
 	}
-	
+
 	private Map<String, Object> sessionProperty = Collections.synchronizedMap(new HashMap<String, Object>());
 
 	/**
 	 * Sets a session property.
-	 * 
+	 *
 	 * @param owner the class that owns the property
 	 * @param name name of the property
 	 * @param property value of the property
@@ -1233,7 +1378,7 @@ public class Session {
 
 	/**
 	 * Gets a session property.
-	 * 
+	 *
 	 * @param owner the class that owns the property
 	 * @param name name of the property
 	 * @return value of the property
@@ -1241,10 +1386,10 @@ public class Session {
 	public Object getSessionProperty(Class<?> owner, String name) {
 		return sessionProperty.get(owner.getName() + "." + name);
 	}
-	
+
 	/**
 	 * Removes all session properties.
-	 * 
+	 *
 	 * @param owner the class that owns the properties
 	 */
 	public void removeSessionProperties(Class<?> owner) {
@@ -1258,8 +1403,8 @@ public class Session {
 
 	/**
 	 * Checks SQL query.
-	 * 
-	 * @param sql 
+	 *
+	 * @param sql
 	 * @return <code>true</code> iff sql is executable without errors
 	 */
 	public boolean checkQuery(String sql) {

src/main/engine/net/sf/jailer/database/ShellScriptBasedStatisticRenovator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/database/SqlException.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -29,13 +29,23 @@ public class SqlException extends SQLException {
 	public final String sqlStatement;
 	private boolean insufficientPrivileges = false;
 	private boolean isFormatted = false;
+	private final String sqlState;
 
 	public SqlException(String message, String sqlStatement, Throwable t) {
 		this(null, message, sqlStatement, t);
 	}
+	
+	public SqlException(String message, String sqlStatement, Throwable t, String sqlLState) {
+		this(null, message, sqlStatement, t, sqlLState);
+	}
 
 	public SqlException(String errorDialogTitle, String message, String sqlStatement, Throwable t) {
+		this(errorDialogTitle, message, sqlStatement, t, null);
+	}
+
+	public SqlException(String errorDialogTitle, String message, String sqlStatement, Throwable t, String sqlState) {
 		super(message, t);
+		this.sqlState = sqlState;
 		this.errorDialogTitle = errorDialogTitle;
 		this.message = t == null? message : t.getMessage();
 		this.sqlStatement = sqlStatement;
@@ -59,4 +69,16 @@ public class SqlException extends SQLException {
 		this.isFormatted = isFormatted;
 	}
 
+	@Override
+	public String getSQLState() {
+		if (sqlState != null) {
+			return sqlState;
+		}
+		Throwable cause = getCause();
+		if (cause instanceof SQLException) {
+			return ((SQLException) cause).getSQLState();
+		}
+		return super.getSQLState();
+	}
+
 }

src/main/engine/net/sf/jailer/database/SqlScriptBasedStatisticRenovator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/database/StatementBuilder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/database/StatisticRenovator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/database/TemporaryTableManager.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/database/UPDATE_MODE.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/database/UPSERT_MODE.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,7 +24,7 @@ public enum UPSERT_MODE {
 	FROM_JL_DUAL, // ("Select 1, 2 From $ where not exists(Select * from $ T where T.c1=1)")
 	FROM_SYSDUMMY1, // ("Select 1, 2 From sysibm.sysdummy1 where not exists(Select * from $ T where T.c1=1)")
 	UNION_ALL,
-	MERGE; // ("MERGE INTO $ T " +
+	MERGE // ("MERGE INTO $ T " +
 //						  "USING (SELECT 1 c1, 2 c2 from dual) incoming " +
 //						  "ON (T.c1 = incoming.c1) " +
 //						  "WHEN MATCHED THEN " +

src/main/engine/net/sf/jailer/database/UpdateTransformer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -153,7 +153,7 @@ public class UpdateTransformer extends AbstractResultSetReader {
 		this.importFilterTransformer = importFilterTransformer;
 		this.inSourceSchema = inSourceSchema;
 		this.reason = reason;
-		if (targetDBMSConfiguration != null && targetDBMSConfiguration != session.dbms) {
+		if (targetDBMSConfiguration != session.dbms) {
 			if (targetDBMSConfiguration.getIdentifierQuoteString() != null) {
 				this.quoting.setIdentifierQuoteString(targetDBMSConfiguration.getIdentifierQuoteString());
 			}
@@ -207,7 +207,7 @@ public class UpdateTransformer extends AbstractResultSetReader {
 			columnLabel = new String[columnCount + 1];
 			labelCSL = "";
 			for (int i = 1; i <= columnCount; ++i) {
-				String mdColumnLabel = quoting.quote(getMetaData(resultSet).getColumnLabel(i));
+				String mdColumnLabel = SqlUtil.columnLabel(quoting, session, targetDBMSConfiguration, table, getMetaData(resultSet).getColumnLabel(i));
 				
 				columnLabel[i] = mdColumnLabel;
 				if (labelCSL.length() > 0) {
@@ -301,7 +301,6 @@ public class UpdateTransformer extends AbstractResultSetReader {
 					whereWOAlias.append(" and ");
 				}
 				f = false;
-				whereForTerminator.append("T." + quoting.requote(pk.name) + "=Q." + quoting.requote(pk.name));
 				String value;
 				Boolean isNull;
 				String name = quoting.quote(pk.name);
@@ -341,7 +340,7 @@ public class UpdateTransformer extends AbstractResultSetReader {
 					if (columnLabel[i] == null) {
 						continue;
 					}
-					if  (!isPrimaryKeyColumn(columnLabel[i])) {
+					if (!isPrimaryKeyColumn(columnLabel[i])) {
 						if (columnNamesLower.contains(columnLabel[i].toLowerCase(Locale.ENGLISH))) {
 							if (sets.length() > 0) {
 								sets.append(", ");

src/main/engine/net/sf/jailer/database/WorkingTableScope.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,29 +17,29 @@ package net.sf.jailer.database;
 
 /**
  * Working-tables scopes.
- * 
+ *
  * @author Ralf Wisser
  */
 public enum WorkingTableScope {
-	
+
 	/**
 	 * Create the working-tables (JAILER_*) in the source database.
 	 */
 	GLOBAL,
-	
+
 	/**
 	 * Create the working-tables (JAILER_*) as temporary tables in the source database.
 	 */
 	SESSION_LOCAL,
-	
+
 	/**
-	 * Not supported.
+	 * Not supported. Do not use.
 	 */
 	TRANSACTION_LOCAL, // not supported
-	
+
 	/**
 	 * Create a local database (H2) for the working-tables (JAILER_*).
 	 */
 	LOCAL_DATABASE
-	
+
 }

src/main/engine/net/sf/jailer/datamodel/AggregationSchema.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

src/main/engine/net/sf/jailer/datamodel/Association.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2007 - 2020 Ralf Wisser.
+ * Copyright 2007 - 2021 Ralf Wisser.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,11 +17,12 @@
 package net.sf.jailer.datamodel;
 
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Pattern;
 
 import net.sf.jailer.restrictionmodel.RestrictionModel;
 import net.sf.jailer.util.Pair;
@@ -30,7 +31,7 @@ import net.sf.jailer.util.SqlUtil;
 
 /**
  * An association between database-tables.
- * 
+ *
  * @author Ralf Wisser
  */
 public class Association extends ModelElement {
@@ -104,7 +105,7 @@ public class Association extends ModelElement {
 
 	/**
 	 * Constructor.
-	 * 
+	 *
 	 * @param source
 	 *            the source table
 	 * @param destination
@@ -133,7 +134,7 @@ public class Association extends ModelElement {
 
 	/**
 	 * Constructor.
-	 * 
+	 *
 	 * @param source
 	 *            the source table
 	 * @param destination
@@ -174,7 +175,7 @@ public class Association extends ModelElement {
 	/**
 	 * Gets the restricted join-condition for joining source with destination
 	 * table.
-	 * 
+	 *
 	 * @return the restricted join-condition for joining source with destination
 	 *         table, <code>null</code> if association must be ignored
 	 */
@@ -193,7 +194,7 @@ public class Association extends ModelElement {
 
 	/**
 	 * Is this association ignored?
-	 * 
+	 *
 	 * @return <code>true</code> iff this association is ignored
 	 */
 	public boolean isIgnored() {
@@ -210,7 +211,7 @@ public class Association extends ModelElement {
 
 	/**
 	 * Gets the cardinality.
-	 * 
+	 *
 	 * @return the cardinality. <code>null</code> if cardinality is not known.
 	 */
 	public Cardinality getCardinality() {
@@ -260,7 +261,7 @@ public class Association extends ModelElement {
 
 	/**
 	 * Stringifies the join condition.
-	 * 
+	 *
 	 * @param restrictionSeparator
 	 *            separates join-condition from restriction condition in the
 	 *            result
@@ -285,7 +286,7 @@ public class Association extends ModelElement {
 
 	/**
 	 * Gets join-condition without any restrictions.
-	 * 
+	 *
 	 * @return join-condition as defined in data model
 	 */
 	public String getUnrestrictedJoinCondition() {
@@ -294,7 +295,7 @@ public class Association extends ModelElement {
 
 	/**
 	 * Gets restriction-condition.
-	 * 
+	 *
 	 * @return restriction-condition, <code>null</code> if association is not
 	 *         restricted
 	 */
@@ -339,7 +340,7 @@ public class Association extends ModelElement {
 
 	/**
 	 * Sets the name of the association.
-	 * 
+	 *
 	 * @param name
 	 *            the name of the association
 	 */
@@ -352,7 +353,7 @@ public class Association extends ModelElement {
 
 	/**
 	 * Gets the name of the association.
-	 * 
+	 *
 	 * @return the name of the association
 	 */
 	public String getName() {
@@ -362,7 +363,7 @@ public class Association extends ModelElement {
 	/**
 	 * Whether or not to insert source-rows before destination rows in order to
 	 * prevent foreign-key-constraint violation.
-	 * 
+	 *
 	 * @return the insertSourceBeforeDestination
 	 */
 	public boolean isInsertSourceBeforeDestination() {
@@ -375,7 +376,7 @@ public class Association extends ModelElement {
 	/**
 	 * Whether or not to insert destination-rows before source-rows in order to
 	 * prevent foreign-key-constraint violation.
-	 * 
+	 *
 	 * @return the insertDestinationBeforeSource
 	 */
 	public boolean isInsertDestinationBeforeSource() {
@@ -402,7 +403,7 @@ public class Association extends ModelElement {
 
 	/**
 	 * Appends condition to join-condition.
-	 * 
+	 *
 	 * @param condition
 	 *            the condition
 	 */
@@ -416,7 +417,7 @@ public class Association extends ModelElement {
 
 	/**
 	 * Gets the XML aggregation schema.
-	 * 
+	 *
 	 * @return the XML aggregation schema
 	 */
 	public AggregationSchema getAggregationSchema() {
@@ -425,7 +426,7 @@ public class Association extends ModelElement {
 
 	/**
 	 * Gets name of XML-tag used for aggregation.
-	 * 
+	 *
 	 * @return name of XML-tag used for aggregation
 	 */
 	public String getAggregationTagName() {
@@ -452,7 +453,7 @@ public class Association extends ModelElement {
 
 	/**
 	 * Sets the XML aggregation schema.
-	 * 
+	 *
 	 * @param aggregationSchema
 	 *            the XML aggregation schema
 	 */
@@ -465,7 +466,7 @@ public class Association extends ModelElement {
 
 	/**
 	 * Sets name of XML-tag used for aggregation.
-	 * 
+	 *
 	 * @param aggregationTagName
 	 *            name of XML-tag used for aggregation
 	 */
@@ -478,7 +479,7 @@ public class Association extends ModelElement {
 
 	/**
 	 * Gets unique ID.
-	 * 
+	 *
 	 * @return unique ID
 	 */
 	public int getId() {
@@ -498,7 +499,7 @@ public class Association extends ModelElement {
 	/**
 	 * Maps source-columns to destination-columns, if this represents an
 	 * equi-join. Otherwise it returns an empty map.
-	 * 
+	 *
 	 * @return map from source-columns to destination-columns, if this
 	 *         represents an equi-join
 	 */
@@ -509,16 +510,16 @@ public class Association extends ModelElement {
 	/**
 	 * Maps source-columns to destination-columns, if this represents an
 	 * equi-join. Otherwise it returns an empty map.
-	 * 
+	 *
 	 * @param assignments if not <code>null</code>, put column assignments into it
-	 * 
+	 *
 	 * @return map from source-columns to destination-columns, if this
 	 *         represents an equi-join
 	 */
 	public Map<Column, Column> createSourceToDestinationKeyMapping(Set<Pair<Column, Column>> assignments) {
 		String[] equations = getUnrestrictedJoinCondition().replaceAll("\\(|\\)", " ").trim()
 				.split("\\s*\\b(a|A)(n|N)(d|D)\\b\\s*");
-		Map<Column, Column> mapping = new HashMap<Column, Column>();
+		Map<Column, Column> mapping = new LinkedHashMap<Column, Column>();
 		Set<Column> destinationColumns = new HashSet<Column>();
 		boolean isValid = true;
 		for (String equation: equations) {
@@ -550,7 +551,7 @@ public class Association extends ModelElement {
 
 			sColumn = Quoting.normalizeIdentifier(sColumn);
 			dColumn = Quoting.normalizeIdentifier(dColumn);
-			
+
 			if (reversed) {
 				String h = sColumn;
 				sColumn = dColumn;
@@ -593,6 +594,17 @@ public class Association extends ModelElement {
 		}
 	}
 
+	public static final String NULL_FILTER_COMMENT_PREFIX = "foreign key to ";
+	private static final Pattern NULL_FILTER_PATTERN = Pattern.compile("(?i:\\s*(/\\*.*\\*/\\s*)?null(\\s*/\\*.*\\*/)?)\\s*");
+
+	private boolean isNullFilter(Filter filter) {
+		return filter.getExpression() != null
+				&&
+				((filter.isApplyAtExport() && NULL_FILTER_PATTERN.matcher(filter.getExpression()).matches())
+				||
+				Filter.EXCLUDED_VALUE.equals(filter.getExpression()));
+	}
+
 	public boolean hasNullableFK() {
 		if (!isInsertDestinationBeforeSource()) {
 			return false;
@@ -605,7 +617,7 @@ public class Association extends ModelElement {
 			if (!c.isNullable) {
 				return false;
 			}
-			if (c.getFilter() != null && !c.getFilter().isDerived() && !c.getFilter().isNullFilter()) {
+			if (c.getFilter() != null && !c.getFilter().isDerived() && !isNullFilter(c.getFilter())) {
 				return false;
 			}
 			for (Column pk: source.primaryKey.getColumns()) {
@@ -620,7 +632,17 @@ public class Association extends ModelElement {
 	public boolean fkHasNullFilter() {
 		Map<Column, Column> sdMap = createSourceToDestinationKeyMapping();
 		for (Column c: sdMap.keySet()) {
-			if (c.getFilter() == null || !c.getFilter().isNullFilter()) {
+			if (c.getFilter() == null || !isNullFilter(c.getFilter())) {
+				return false;
+			}
+		}
+		return true;
+	}
+
+	public boolean fkHasExcludeFilter() {
+		Map<Column, Column> sdMap = createSourceToDestinationKeyMapping();
+		for (Column c: sdMap.keySet()) {
+			if (c.getFilter() == null || !Filter.EXCLUDED_VALUE.equals(c.getFilter().getExpression())) {
 				return false;
 			}
 		}
@@ -632,15 +654,15 @@ public class Association extends ModelElement {
 		Map<Column, Column> sdMap = createSourceToDestinationKeyMapping();
 		for (Column c: sdMap.keySet()) {
 			if (set) {
-				if (c.getFilter() == null || !"null".equals(c.getFilter().getExpression())) {
+				if (c.getFilter() == null || !isNullFilter(c.getFilter())) {
+					c.setFilter(new Filter("null /* " + NULL_FILTER_COMMENT_PREFIX + getName() + " */", null, false, null));
 					changed = true;
 				}
-				c.setFilter(new Filter("null", null, false, null));
 			} else {
 				if (c.getFilter() != null) {
+					c.setFilter(null);
 					changed = true;
 				}
-				c.setFilter(null);
 			}
 		}
 		getDataModel().deriveFilters();
@@ -648,4 +670,14 @@ public class Association extends ModelElement {
 		return changed;
 	}
 
+	public boolean isRestrictedDependencyWithNulledFK() {
+		boolean restrictedDep = isInsertDestinationBeforeSource() && getRestrictionCondition() != null;
+		if (restrictedDep) {
+			if (fkHasNullFilter() && hasNullableFK() && !fkHasExcludeFilter()) {
+				return true;
+			}
+		}
+		return false;
+	}
+
 }