Merge branch 'stable'

# Conflicts:
#	src/services/notes.js
#	src/tools/generate_document.js

src/services/anonymization.js

@@ -18,7 +18,10 @@ async function anonymize() {
 
     await db.run("UPDATE notes SET title = 'title'");
     await db.run("UPDATE note_contents SET content = 'text'");
-    await db.run("UPDATE note_revisions SET title = 'title', content = 'text'");
+    await db.run("UPDATE note_revisions SET title = 'title'");
+    await db.run("UPDATE note_revision_contents SET content = 'title'");
+    await db.run("UPDATE attributes SET name = 'name', value = 'value' WHERE type = 'label'");
+    await db.run("UPDATE attributes SET name = 'name' WHERE type = 'relation'");
     await db.run("UPDATE branches SET prefix = 'prefix' WHERE prefix IS NOT NULL");
     await db.run(`UPDATE options SET value = 'anonymized' WHERE name IN 
                     ('documentSecret', 'encryptedDataKey', 'passwordVerificationHash', 

src/services/consistency_checks.js

@@ -626,12 +626,31 @@ async function runAllChecks() {
     return !unrecoveredConsistencyErrors;
 }
 
+async function showEntityStat(name, query) {
+    const map = await sql.getMap(query);
+
+    map[0] = map[0] || 0;
+    map[1] = map[1] || 0;
+
+    log.info(`${name} deleted: ${map[1]}, not deleted ${map[0]}`);
+}
+
+async function runDbDiagnostics() {
+    await showEntityStat("Notes", `SELECT isDeleted, count(noteId) FROM notes GROUP BY isDeleted`);
+    await showEntityStat("Note revisions", `SELECT isErased, count(noteRevisionId) FROM note_revisions GROUP BY isErased`);
+    await showEntityStat("Branches", `SELECT isDeleted, count(branchId) FROM branches GROUP BY isDeleted`);
+    await showEntityStat("Attributes", `SELECT isDeleted, count(attributeId) FROM attributes GROUP BY isDeleted`);
+    await showEntityStat("API tokens", `SELECT isDeleted, count(apiTokenId) FROM api_tokens GROUP BY isDeleted`);
+}
+
 async function runChecks() {
     let elapsedTimeMs;
 
     await syncMutexService.doExclusively(async () => {
         const startTime = new Date();
 
+        await runDbDiagnostics();
+
         await runAllChecks();
 
         elapsedTimeMs = Date.now() - startTime.getTime();
@@ -663,7 +682,7 @@ sqlInit.dbReady.then(() => {
     setInterval(cls.wrap(runChecks), 60 * 60 * 1000);
 
     // kickoff checks soon after startup (to not block the initial load)
-    setTimeout(cls.wrap(runChecks), 10 * 1000);
+    setTimeout(cls.wrap(runChecks), 20 * 1000);
 });
 
 module.exports = {};

src/services/handlers.js

@@ -81,7 +81,7 @@ eventService.subscribe(eventService.CHILD_NOTE_CREATED, async ({ parentNote, chi
 async function processInverseRelations(entityName, entity, handler) {
     if (entityName === 'attributes' && entity.type === 'relation') {
         const note = await entity.getNote();
-        const attributes = (await note.getAttributes(entity.name)).filter(relation => relation.type === 'relation-definition');
+        const attributes = (await note.getOwnedAttributes(entity.name)).filter(relation => relation.type === 'relation-definition');
 
         for (const attribute of attributes) {
             const definition = attribute.value;

src/services/migration.js

@@ -43,9 +43,6 @@ async function migrate() {
         try {
             log.info("Attempting migration to version " + mig.dbVersion);
 
-            // needs to happen outside of the transaction (otherwise it's a NO-OP)
-            await sql.execute("PRAGMA foreign_keys = OFF");
-
             await sql.transactional(async () => {
                 if (mig.type === 'sql') {
                     const migrationSql = fs.readFileSync(resourceDir.MIGRATIONS_DIR + "/" + mig.file).toString('utf8');
@@ -76,10 +73,6 @@ async function migrate() {
 
             utils.crash();
         }
-        finally {
-            // make sure foreign keys are enabled even if migration script disables them
-            await sql.execute("PRAGMA foreign_keys = ON");
-        }
     }
 
     if (await sqlInit.isDbUpToDate()) {

src/services/notes.js

@@ -57,7 +57,7 @@ function deriveMime(type, mime) {
 }
 
 async function copyChildAttributes(parentNote, childNote) {
-    for (const attr of await parentNote.getAttributes()) {
+    for (const attr of await parentNote.getOwnedAttributes()) {
         if (attr.name.startsWith("child:")) {
             await new Attribute({
                 noteId: childNote.noteId,
@@ -489,7 +489,7 @@ async function eraseDeletedNotes() {
         SET content = NULL,
             utcDateModified = '${utcNowDateTime}'
         WHERE noteRevisionId IN 
-            (SELECT noteRevisionId FROM note_revisions WHERE isErased = 0 AND noteId IN ((???)))`, noteIdsToErase);
+            (SELECT noteRevisionId FROM note_revisions WHERE isErased = 0 AND noteId IN (???))`, noteIdsToErase);
 
     await sql.executeMany(`
         UPDATE note_revisions 
@@ -523,7 +523,7 @@ async function duplicateNote(noteId, parentNoteId) {
         notePosition: origBranch ? origBranch.notePosition + 1 : null
     }).save();
 
-    for (const attribute of await origNote.getAttributes()) {
+    for (const attribute of await origNote.getOwnedAttributes()) {
         const attr = new Attribute(attribute);
         attr.attributeId = undefined; // force creation of new attribute
         attr.noteId = newNote.noteId;

src/services/sql_init.js

@@ -57,8 +57,6 @@ async function initDbConnection() {
             return;
         }
 
-        await sql.execute("PRAGMA foreign_keys = ON");
-
         const currentDbVersion = await getDbVersion();
 
         if (currentDbVersion > appInfo.dbVersion) {
@@ -175,9 +173,11 @@ async function isDbUpToDate() {
 }
 
 async function dbInitialized() {
-    await optionService.setOption('initialized', 'true');
+    if (!await isDbInitialized()) {
+        await optionService.setOption('initialized', 'true');
 
-    await initDbConnection();
+        await initDbConnection();
+    }
 }
 
 dbReady.then(async () => {