Extend SQL grammar to recognise 'without rowid' tables correctly

Since version 3.8.2 SQLite supports tables without the internal rowid
column added to the table. For these tables the primary key serves as a
replacement for the rowid column.

These changes update the grammar parser to correctly handle 'without
rowid' tables and also generate 'without rowid' SQL statements.

src/grammar/Sqlite3Lexer.hpp

@@ -2,7 +2,7 @@
 #define INC_Sqlite3Lexer_hpp_
 
 #include <antlr/config.hpp>
-/* $ANTLR 2.7.7 (20140222): "sqlite3.g" -> "Sqlite3Lexer.hpp"$ */
+/* $ANTLR 2.7.7 (20130425): "sqlite3.g" -> "Sqlite3Lexer.hpp"$ */
 #include <antlr/CommonToken.hpp>
 #include <antlr/InputBuffer.hpp>
 #include <antlr/BitSet.hpp>

src/grammar/Sqlite3Parser.hpp

@@ -2,7 +2,7 @@
 #define INC_Sqlite3Parser_hpp_
 
 #include <antlr/config.hpp>
-/* $ANTLR 2.7.7 (20140222): "sqlite3.g" -> "Sqlite3Parser.hpp"$ */
+/* $ANTLR 2.7.7 (20130425): "sqlite3.g" -> "Sqlite3Parser.hpp"$ */
 #include <antlr/TokenStream.hpp>
 #include <antlr/TokenBuffer.hpp>
 #include "sqlite3TokenTypes.hpp"
@@ -76,10 +76,10 @@ protected:
 private:
 	static const char* tokenNames[];
 #ifndef NO_STATIC_CONSTS
-	static const int NUM_TOKENS = 103;
+	static const int NUM_TOKENS = 105;
 #else
 	enum {
-		NUM_TOKENS = 103
+		NUM_TOKENS = 105
 	};
 #endif
 	

src/grammar/sqlite3.g

@@ -62,6 +62,7 @@ tokens {
   REPLACE="REPLACE";
   RESTRICT="RESTRICT";
   ROLLBACK="ROLLBACK";
+  ROWID="ROWID";
   SET="SET";
   TEMPORARY="TEMPORARY";
   TEMP="TEMP";
@@ -69,6 +70,7 @@ tokens {
   UNIQUE="UNIQUE";
   UPDATE="UPDATE";
   WHEN="WHEN";
+  WITHOUT="WITHOUT";
 
 //ast
 
@@ -252,7 +254,7 @@ keywordastablename
 createtable
   :
   CREATE (TEMP|TEMPORARY)? TABLE (IF_T NOT EXISTS)? (tablename | keywordastablename)
-  ( LPAREN columndef (COMMA columndef)* (COMMA tableconstraint)* RPAREN
+  ( LPAREN columndef (COMMA columndef)* (COMMA tableconstraint)* RPAREN (WITHOUT ROWID)?
   | AS selectstmt
   )
   {#createtable = #([CREATETABLE, "CREATETABLE"], #createtable);}

src/grammar/sqlite3TokenTypes.hpp

@@ -1,7 +1,7 @@
 #ifndef INC_sqlite3TokenTypes_hpp_
 #define INC_sqlite3TokenTypes_hpp_
 
-/* $ANTLR 2.7.7 (20140222): "sqlite3.g" -> "sqlite3TokenTypes.hpp"$ */
+/* $ANTLR 2.7.7 (20130425): "sqlite3.g" -> "sqlite3TokenTypes.hpp"$ */
 
 #ifndef CUSTOM_API
 # define CUSTOM_API
@@ -61,56 +61,58 @@ struct CUSTOM_API sqlite3TokenTypes {
 		REPLACE = 50,
 		RESTRICT = 51,
 		ROLLBACK = 52,
-		SET = 53,
-		TEMPORARY = 54,
-		TEMP = 55,
-		THEN = 56,
-		UNIQUE = 57,
-		UPDATE = 58,
-		WHEN = 59,
-		TYPE_NAME = 60,
-		COLUMNDEF = 61,
-		COLUMNCONSTRAINT = 62,
-		TABLECONSTRAINT = 63,
-		CREATETABLE = 64,
-		KEYWORDASTABLENAME = 65,
-		KEYWORDASCOLUMNNAME = 66,
-		DIGIT = 67,
-		DOT = 68,
-		ID = 69,
-		QUOTEDID = 70,
-		QUOTEDLITERAL = 71,
-		NUMERIC = 72,
-		NL = 73,
-		COMMENT = 74,
-		WS = 75,
-		STRINGLITERAL = 76,
-		LPAREN = 77,
-		RPAREN = 78,
-		COMMA = 79,
-		SEMI = 80,
-		PLUS = 81,
-		MINUS = 82,
-		STAR = 83,
-		TILDE = 84,
-		AMPERSAND = 85,
-		BITOR = 86,
-		OROP = 87,
-		EQUAL = 88,
-		EQUAL2 = 89,
-		GREATER = 90,
-		GREATEREQUAL = 91,
-		LOWER = 92,
-		LOWEREQUAL = 93,
-		UNEQUAL = 94,
-		UNEQUAL2 = 95,
-		BITWISELEFT = 96,
-		BITWISERIGHT = 97,
-		NO = 98,
-		SELECT = 99,
-		SLASH = 100,
-		PERCENT = 101,
-		IN = 102,
+		ROWID = 53,
+		SET = 54,
+		TEMPORARY = 55,
+		TEMP = 56,
+		THEN = 57,
+		UNIQUE = 58,
+		UPDATE = 59,
+		WHEN = 60,
+		WITHOUT = 61,
+		TYPE_NAME = 62,
+		COLUMNDEF = 63,
+		COLUMNCONSTRAINT = 64,
+		TABLECONSTRAINT = 65,
+		CREATETABLE = 66,
+		KEYWORDASTABLENAME = 67,
+		KEYWORDASCOLUMNNAME = 68,
+		DIGIT = 69,
+		DOT = 70,
+		ID = 71,
+		QUOTEDID = 72,
+		QUOTEDLITERAL = 73,
+		NUMERIC = 74,
+		NL = 75,
+		COMMENT = 76,
+		WS = 77,
+		STRINGLITERAL = 78,
+		LPAREN = 79,
+		RPAREN = 80,
+		COMMA = 81,
+		SEMI = 82,
+		PLUS = 83,
+		MINUS = 84,
+		STAR = 85,
+		TILDE = 86,
+		AMPERSAND = 87,
+		BITOR = 88,
+		OROP = 89,
+		EQUAL = 90,
+		EQUAL2 = 91,
+		GREATER = 92,
+		GREATEREQUAL = 93,
+		LOWER = 94,
+		LOWEREQUAL = 95,
+		UNEQUAL = 96,
+		UNEQUAL2 = 97,
+		BITWISELEFT = 98,
+		BITWISERIGHT = 99,
+		NO = 100,
+		SELECT = 101,
+		SLASH = 102,
+		PERCENT = 103,
+		IN = 104,
 		NULL_TREE_LOOKAHEAD = 3
 	};
 #ifdef __cplusplus

src/sqlitetypes.cpp

@@ -55,6 +55,7 @@ bool Field::isInteger() const
 void Table::clear()
 {
     m_fields.clear();
+    m_rowidColumn = "rowid";
 }
 
 Table::~Table()
@@ -94,6 +95,16 @@ int Table::findField(const QString &sname)
     return -1;
 }
 
+int Table::findPk() const
+{
+    for(int i = 0; i < m_fields.count(); ++i)
+    {
+        if(m_fields.at(i)->primaryKey())
+            return i;
+    }
+    return -1;
+}
+
 QStringList Table::fieldList() const
 {
     QStringList sl;
@@ -208,8 +219,13 @@ QString Table::sql() const
         if(pks_found)
             sql += pk + ")";
     }
+    sql += "\n)";
 
-    return sql + "\n);";
+    // without rowid
+    if(m_rowidColumn != "rowid")
+        sql += " WITHOUT ROWID";
+
+    return sql + ";";
 }
 
 namespace
@@ -262,7 +278,6 @@ Table CreateTableWalker::table()
         s = s->getNextSibling(); // first column name
         antlr::RefAST column = s;
         // loop columndefs
-        FieldVector pks;
         while(column != antlr::nullAST && column->getType() == sqlite3TokenTypes::COLUMNDEF)
         {
             FieldPtr f;
@@ -275,33 +290,46 @@ Table CreateTableWalker::table()
         // now we are finished or it is a tableconstraint
         while(s != antlr::nullAST)
         {
-            antlr::RefAST tc = s->getFirstChild();
-            // skip constraint name, if there is any
-            if(tc->getType() == sqlite3TokenTypes::CONSTRAINT)
-                tc = tc->getNextSibling()->getNextSibling();
+            // Is this a 'without rowid' definiton?
+            if(s->getType() != sqlite3TokenTypes::WITHOUT)
+            {
+                // It's not, so treat this as table constraints
 
-            switch(tc->getType())
-            {
-            case sqlite3TokenTypes::PRIMARY:
-            {
-                tc = tc->getNextSibling()->getNextSibling(); // skip primary and key
-                tc = tc->getNextSibling(); // skip LPAREN
-                do
+                antlr::RefAST tc = s->getFirstChild();
+                // skip constraint name, if there is any
+                if(tc->getType() == sqlite3TokenTypes::CONSTRAINT)
+                    tc = tc->getNextSibling()->getNextSibling();
+
+                switch(tc->getType())
                 {
-                    QString col = identifier(tc);
-                    int fieldindex = tab.findField(col);
-                    if(fieldindex != -1)
-                        tab.fields().at(fieldindex)->setPrimaryKey(true);
-                    tc = tc->getNextSibling(); // skip ident and comma
-                    tc = tc->getNextSibling();
-                } while(tc != antlr::nullAST && tc->getType() != sqlite3TokenTypes::RPAREN);
-            }
-            break;
-            default: break;
-            }
+                case sqlite3TokenTypes::PRIMARY:
+                {
+                    tc = tc->getNextSibling()->getNextSibling(); // skip primary and key
+                    tc = tc->getNextSibling(); // skip LPAREN
+                    do
+                    {
+                        QString col = identifier(tc);
+                        int fieldindex = tab.findField(col);
+                        if(fieldindex != -1)
+                            tab.fields().at(fieldindex)->setPrimaryKey(true);
+                        tc = tc->getNextSibling(); // skip ident and comma
+                        tc = tc->getNextSibling();
+                    } while(tc != antlr::nullAST && tc->getType() != sqlite3TokenTypes::RPAREN);
+                }
+                break;
+                default: break;
+                }
 
-            s = s->getNextSibling(); //COMMA or RPAREN
-            s = s->getNextSibling();
+                s = s->getNextSibling(); //COMMA or RPAREN
+                s = s->getNextSibling();
+            } else {
+                // It is
+
+                s = s->getNextSibling();    // WITHOUT
+                s = s->getNextSibling();    // ROWID
+
+                tab.setRowidColumn(tab.fields().at(tab.findPk())->name());
+            }
         }
     }
 

src/sqlitetypes.h

@@ -66,7 +66,7 @@ typedef QVector< FieldPtr > FieldVector;
 class Table
 {
 public:
-    Table(const QString& name): m_name(name) {}
+    Table(const QString& name): m_name(name), m_rowidColumn("rowid") {}
     virtual ~Table();
 
     void setName(const QString& name) { m_name = name; }
@@ -90,6 +90,8 @@ public:
     bool removeField(const QString& sFieldName);
     void setFields(const FieldVector& fields);
     void setField(int index, FieldPtr f) { m_fields[index] = f; }
+    void setRowidColumn(const QString& rowid) {  m_rowidColumn = rowid; }
+    QString rowidColumn() const { return m_rowidColumn; }
     void clear();
 
     /**
@@ -100,6 +102,8 @@ public:
      */
     int findField(const QString& sname);
 
+    int findPk() const;
+
     static Table parseSQL(const QString& sSQL);
 private:
     QStringList fieldList() const;
@@ -108,6 +112,7 @@ private:
 private:
     QString m_name;
     FieldVector m_fields;
+    QString m_rowidColumn;
 };
 
 /**