diff --git a/.eslintignore b/.eslintignore index 2315fe2b61..11deefda4d 100644 --- a/.eslintignore +++ b/.eslintignore @@ -990,6 +990,9 @@ packages/lib/models/Folder.test.js.map packages/lib/models/ItemChange.d.ts packages/lib/models/ItemChange.js packages/lib/models/ItemChange.js.map +packages/lib/models/ItemChange.test.d.ts +packages/lib/models/ItemChange.test.js +packages/lib/models/ItemChange.test.js.map packages/lib/models/MasterKey.d.ts packages/lib/models/MasterKey.js packages/lib/models/MasterKey.js.map @@ -1404,6 +1407,12 @@ packages/lib/services/rest/actionApi.desktop.js.map packages/lib/services/rest/routes/auth.d.ts packages/lib/services/rest/routes/auth.js packages/lib/services/rest/routes/auth.js.map +packages/lib/services/rest/routes/events.d.ts +packages/lib/services/rest/routes/events.js +packages/lib/services/rest/routes/events.js.map +packages/lib/services/rest/routes/events.test.d.ts +packages/lib/services/rest/routes/events.test.js +packages/lib/services/rest/routes/events.test.js.map packages/lib/services/rest/routes/folders.d.ts packages/lib/services/rest/routes/folders.js packages/lib/services/rest/routes/folders.js.map diff --git a/.gitignore b/.gitignore index 246a0ebe6b..4e63199dfb 100644 --- a/.gitignore +++ b/.gitignore @@ -975,6 +975,9 @@ packages/lib/models/Folder.test.js.map packages/lib/models/ItemChange.d.ts packages/lib/models/ItemChange.js packages/lib/models/ItemChange.js.map +packages/lib/models/ItemChange.test.d.ts +packages/lib/models/ItemChange.test.js +packages/lib/models/ItemChange.test.js.map packages/lib/models/MasterKey.d.ts packages/lib/models/MasterKey.js packages/lib/models/MasterKey.js.map @@ -1389,6 +1392,12 @@ packages/lib/services/rest/actionApi.desktop.js.map packages/lib/services/rest/routes/auth.d.ts packages/lib/services/rest/routes/auth.js packages/lib/services/rest/routes/auth.js.map +packages/lib/services/rest/routes/events.d.ts +packages/lib/services/rest/routes/events.js +packages/lib/services/rest/routes/events.js.map +packages/lib/services/rest/routes/events.test.d.ts +packages/lib/services/rest/routes/events.test.js +packages/lib/services/rest/routes/events.test.js.map packages/lib/services/rest/routes/folders.d.ts packages/lib/services/rest/routes/folders.js packages/lib/services/rest/routes/folders.js.map diff --git a/packages/app-cli/app/command-apidoc.js b/packages/app-cli/app/command-apidoc.js index 8da3a50c46..c66e45b589 100644 --- a/packages/app-cli/app/command-apidoc.js +++ b/packages/app-cli/app/command-apidoc.js @@ -381,6 +381,30 @@ async function fetchAllNotes() { } } + { + const tableFields = reg.db().tableFields('item_changes', { includeDescription: true }); + + lines.push('# Events'); + lines.push(''); + lines.push('This end point can be used to retrieve the latest note changes. Currently only note changes are tracked.'); + lines.push(''); + lines.push('## Properties'); + lines.push(''); + lines.push(this.createPropertiesTable(tableFields)); + lines.push(''); + lines.push('## GET /events'); + lines.push(''); + lines.push('Returns a paginated list of recent events. A `cursor` property should be provided, which tells from what point in time the events should be returned. The API will return a `cursor` property, to tell from where to resume retrieving events, as well as an `has_more` (tells if more changes can be retrieved) and `items` property, which will contain the list of events. Events are kept for up to 90 days.'); + lines.push(''); + lines.push('If no `cursor` property is provided, the API will respond with the latest change ID. That can be used to retrieve future events later on.'); + lines.push(''); + lines.push('The results are paginated so will need to may multiple calls to retrieve all the events. Use the `has_more` property to know if more can be retrieved.'); + lines.push(''); + lines.push('## GET /events/:id'); + lines.push(''); + lines.push('Returns the event with the given ID.'); + } + const outFilePath = args['file']; await shim.fsDriver().writeFile(outFilePath, lines.join('\n'), 'utf8'); diff --git a/packages/lib/JoplinDatabase.ts b/packages/lib/JoplinDatabase.ts index d3d8a7193b..f102f4afc1 100644 --- a/packages/lib/JoplinDatabase.ts +++ b/packages/lib/JoplinDatabase.ts @@ -259,6 +259,14 @@ export default class JoplinDatabase extends Database { folders: {}, resources: {}, tags: {}, + item_changes: { + type: 'The type of change - either 1 (created), 2 (updated) or 3 (deleted)', + created_time: 'When the event was generated', + item_type: 'The item type (see table above for the list of item types)', + item_id: 'The item ID', + before_change_item: 'Unused', + source: 'Unused', + }, }; const baseItems = ['notes', 'folders', 'tags', 'resources']; diff --git a/packages/lib/database.ts b/packages/lib/database.ts index 56a8159d52..aec89c72ac 100644 --- a/packages/lib/database.ts +++ b/packages/lib/database.ts @@ -77,7 +77,7 @@ export default class Database { throw new Error(`Invalid field format: ${field}`); } - escapeFields(fields: string[] | string): string[] | string { + public escapeFields(fields: string[] | string): string[] | string { if (fields == '*') return '*'; const output = []; @@ -87,6 +87,16 @@ export default class Database { return output; } + public escapeFieldsToString(fields: string[] | string): string { + if (fields === '*') return '*'; + + const output = []; + for (let i = 0; i < fields.length; i++) { + output.push(this.escapeField(fields[i])); + } + return output.join(','); + } + async tryCall(callName: string, inputSql: StringOrSqlQuery, inputParams: SqlParams) { let sql: string = null; let params: SqlParams = null; diff --git a/packages/lib/models/ItemChange.test.js b/packages/lib/models/ItemChange.test.ts similarity index 50% rename from packages/lib/models/ItemChange.test.js rename to packages/lib/models/ItemChange.test.ts index d94e3e5912..4e266cda99 100644 --- a/packages/lib/models/ItemChange.test.js +++ b/packages/lib/models/ItemChange.test.ts @@ -1,13 +1,13 @@ -const { revisionService, setupDatabaseAndSynchronizer, db, switchClient } = require('../testing/test-utils.js'); -const SearchEngine = require('../services/searchengine/SearchEngine').default; -const ResourceService = require('../services/ResourceService').default; -const ItemChangeUtils = require('../services/ItemChangeUtils').default; -const Note = require('../models/Note').default; -const ItemChange = require('../models/ItemChange').default; +import { revisionService, setupDatabaseAndSynchronizer, db, switchClient, msleep } from '../testing/test-utils'; +import SearchEngine from '../services/searchengine/SearchEngine'; +import ResourceService from '../services/ResourceService'; +import ItemChangeUtils from '../services/ItemChangeUtils'; +import Note from '../models/Note'; +import ItemChange from '../models/ItemChange'; -let searchEngine = null; +let searchEngine: SearchEngine = null; -describe('models_ItemChange', function() { +describe('models/ItemChange', function() { beforeEach(async (done) => { await setupDatabaseAndSynchronizer(1); @@ -27,17 +27,28 @@ describe('models_ItemChange', function() { const resourceService = new ResourceService(); await searchEngine.syncTables(); + // If we run this now, it should not delete any change because // the resource service has not yet processed the change - await ItemChangeUtils.deleteProcessedChanges(); + await ItemChangeUtils.deleteProcessedChanges(0); expect(await ItemChange.lastChangeId()).toBe(1); await resourceService.indexNoteResources(); - await ItemChangeUtils.deleteProcessedChanges(); + await ItemChangeUtils.deleteProcessedChanges(0); expect(await ItemChange.lastChangeId()).toBe(1); await revisionService().collectRevisions(); + + // If we don't set a TTL it will default to 90 days so it won't delete + // either. await ItemChangeUtils.deleteProcessedChanges(); + expect(await ItemChange.lastChangeId()).toBe(1); + + // All changes should be at least 4 ms old now + await msleep(4); + + // Now it should delete all changes older than 3 ms + await ItemChangeUtils.deleteProcessedChanges(3); expect(await ItemChange.lastChangeId()).toBe(0); })); diff --git a/packages/lib/models/ItemChange.ts b/packages/lib/models/ItemChange.ts index 3ce8ff553d..bd29acefec 100644 --- a/packages/lib/models/ItemChange.ts +++ b/packages/lib/models/ItemChange.ts @@ -1,8 +1,14 @@ import BaseModel, { ModelType } from '../BaseModel'; import shim from '../shim'; import eventManager from '../eventManager'; +import { ItemChangeEntity } from '../services/database/types'; const Mutex = require('async-mutex').Mutex; +export interface ChangeSinceIdOptions { + limit?: number; + fields?: string[]; +} + export default class ItemChange extends BaseModel { private static addChangeMutex_: any = new Mutex(); @@ -24,7 +30,7 @@ export default class ItemChange extends BaseModel { return BaseModel.TYPE_ITEM_CHANGE; } - static async add(itemType: ModelType, itemId: string, type: number, changeSource: any = null, beforeChangeItemJson: string = null) { + public static async add(itemType: ModelType, itemId: string, type: number, changeSource: any = null, beforeChangeItemJson: string = null) { if (changeSource === null) changeSource = ItemChange.SOURCE_UNSPECIFIED; if (!beforeChangeItemJson) beforeChangeItemJson = ''; @@ -57,14 +63,14 @@ export default class ItemChange extends BaseModel { } } - static async lastChangeId() { + public static async lastChangeId() { const row = await this.db().selectOne('SELECT max(id) as max_id FROM item_changes'); return row && row.max_id ? row.max_id : 0; } // Because item changes are recorded in the background, this function // can be used for synchronous code, in particular when unit testing. - static async waitForAllSaved() { + public static async waitForAllSaved() { return new Promise((resolve) => { const iid = shim.setInterval(() => { if (!ItemChange.saveCalls_.length) { @@ -75,8 +81,32 @@ export default class ItemChange extends BaseModel { }); } - static async deleteOldChanges(lowestChangeId: number) { + public static async deleteOldChanges(lowestChangeId: number, itemMinTtl: number) { if (!lowestChangeId) return; - return this.db().exec('DELETE FROM item_changes WHERE id <= ?', [lowestChangeId]); + + const cutOffDate = Date.now() - itemMinTtl; + + return this.db().exec(` + DELETE FROM item_changes + WHERE id <= ? + AND created_time <= ? + `, [lowestChangeId, cutOffDate]); } + + public static async changesSinceId(changeId: number, options: ChangeSinceIdOptions = null): Promise { + options = { + limit: 100, + fields: ['id', 'item_type', 'item_id', 'type', 'created_time'], + ...options, + }; + + return this.db().selectAll(` + SELECT ${this.db().escapeFieldsToString(options.fields)} + FROM item_changes + WHERE id > ? + ORDER BY id + LIMIT ? + `, [changeId, options.limit]); + } + } diff --git a/packages/lib/services/ItemChangeUtils.ts b/packages/lib/services/ItemChangeUtils.ts index 6d2aa1741e..1aa5474ca8 100644 --- a/packages/lib/services/ItemChangeUtils.ts +++ b/packages/lib/services/ItemChangeUtils.ts @@ -1,8 +1,10 @@ import Setting from '../models/Setting'; import ItemChange from '../models/ItemChange'; +const dayMs = 86400000; + export default class ItemChangeUtils { - static async deleteProcessedChanges() { + static async deleteProcessedChanges(itemMinTtl: number = dayMs * 90) { const lastProcessedChangeIds = [ Setting.value('resourceService.lastProcessedChangeId'), Setting.value('searchEngine.lastProcessedChangeId'), @@ -10,6 +12,6 @@ export default class ItemChangeUtils { ]; const lowestChangeId = Math.min(...lastProcessedChangeIds); - await ItemChange.deleteOldChanges(lowestChangeId); + await ItemChange.deleteOldChanges(lowestChangeId, itemMinTtl); } } diff --git a/packages/lib/services/rest/Api.ts b/packages/lib/services/rest/Api.ts index a0a310276d..97d84af183 100644 --- a/packages/lib/services/rest/Api.ts +++ b/packages/lib/services/rest/Api.ts @@ -9,6 +9,7 @@ import route_master_keys from './routes/master_keys'; import route_search from './routes/search'; import route_ping from './routes/ping'; import route_auth from './routes/auth'; +import route_events from './routes/events'; const { ltrimSlashes } = require('../../path-utils'); const md5 = require('md5'); @@ -43,6 +44,9 @@ interface RequestQuery { // Auth token auth_token?: string; + + // Event cursor + cursor?: string; } export interface Request { @@ -104,6 +108,7 @@ export default class Api { search: route_search, services: this.action_services.bind(this), auth: route_auth, + events: route_events, }; this.dispatch = this.dispatch.bind(this); diff --git a/packages/lib/services/rest/routes/events.test.ts b/packages/lib/services/rest/routes/events.test.ts new file mode 100644 index 0000000000..74de66ee4c --- /dev/null +++ b/packages/lib/services/rest/routes/events.test.ts @@ -0,0 +1,94 @@ +import { ModelType } from '../../../BaseModel'; +import ItemChange from '../../../models/ItemChange'; +import Note from '../../../models/Note'; +import { expectThrow, setupDatabaseAndSynchronizer, switchClient } from '../../../testing/test-utils'; +import { ItemChangeEntity } from '../../database/types'; +import Api, { RequestMethod } from '../Api'; + +let api: Api = null; + +describe('routes/events', function() { + + beforeEach(async (done) => { + api = new Api(); + await setupDatabaseAndSynchronizer(1); + await switchClient(1); + done(); + }); + + it('should retrieve the latest events', async () => { + let cursor = '0'; + + { + const response = await api.route(RequestMethod.GET, 'events', { cursor }); + expect(response.cursor).toBe('0'); + } + + const note1 = await Note.save({ title: 'toto' }); + await Note.save({ id: note1.id, title: 'tutu' }); + const note2 = await Note.save({ title: 'tata' }); + await ItemChange.waitForAllSaved(); + + { + const response = await api.route(RequestMethod.GET, 'events', { cursor }); + expect(response.cursor).toBe('3'); + expect(response.items.length).toBe(2); + expect(response.has_more).toBe(false); + expect(response.items.map((it: ItemChangeEntity) => it.item_id).sort()).toEqual([note1.id, note2.id].sort()); + + cursor = response.cursor; + } + + { + const response = await api.route(RequestMethod.GET, 'events', { cursor }); + expect(response.cursor).toBe(cursor); + expect(response.items.length).toBe(0); + expect(response.has_more).toBe(false); + } + + await Note.save({ id: note2.id, title: 'titi' }); + await ItemChange.waitForAllSaved(); + + { + const response = await api.route(RequestMethod.GET, 'events', { cursor }); + expect(response.cursor).toBe('4'); + expect(response.items.length).toBe(1); + expect(response.items[0].item_id).toBe(note2.id); + } + }); + + it('should limit the number of response items', async () => { + const promises = []; + for (let i = 0; i < 101; i++) { + promises.push(Note.save({ title: 'toto' })); + } + + await Promise.all(promises); + await ItemChange.waitForAllSaved(); + + const response1 = await api.route(RequestMethod.GET, 'events', { cursor: '0' }); + expect(response1.items.length).toBe(100); + expect(response1.has_more).toBe(true); + + const response2 = await api.route(RequestMethod.GET, 'events', { cursor: response1.cursor }); + expect(response2.items.length).toBe(1); + expect(response2.has_more).toBe(false); + }); + + it('should retrieve a single item', async () => { + const beforeTime = Date.now(); + + const note = await Note.save({ title: 'toto' }); + await ItemChange.waitForAllSaved(); + + const response = await api.route(RequestMethod.GET, 'events/1'); + + expect(response.item_type).toBe(ModelType.Note); + expect(response.type).toBe(1); + expect(response.item_id).toBe(note.id); + expect(response.created_time).toBeGreaterThanOrEqual(beforeTime); + + await expectThrow(async () => api.route(RequestMethod.GET, 'events/1234')); + }); + +}); diff --git a/packages/lib/services/rest/routes/events.ts b/packages/lib/services/rest/routes/events.ts new file mode 100644 index 0000000000..70d04fda56 --- /dev/null +++ b/packages/lib/services/rest/routes/events.ts @@ -0,0 +1,39 @@ +import { ModelType } from '../../../BaseModel'; +import { Request, RequestMethod } from '../Api'; +import { ErrorBadRequest, ErrorNotFound } from '../utils/errors'; +import ItemChange, { ChangeSinceIdOptions } from '../../../models/ItemChange'; +import requestFields from '../utils/requestFields'; + +export default async function(request: Request, id: string = null, _link: string = null) { + if (request.method === RequestMethod.GET) { + const options: ChangeSinceIdOptions = { + limit: 100, + fields: requestFields(request, ModelType.ItemChange, ['id', 'item_type', 'item_id', 'type', 'created_time']), + }; + + if (!id) { + if (!('cursor' in request.query)) { + return { + items: [], + has_more: false, + cursor: (await ItemChange.lastChangeId()).toString(), + }; + } else { + const cursor = Number(request.query.cursor); + if (isNaN(cursor)) throw new ErrorBadRequest(`Invalid cursor: ${request.query.cursor}`); + + const changes = await ItemChange.changesSinceId(cursor, options); + + return { + items: changes, + has_more: changes.length >= options.limit, + cursor: (changes.length ? changes[changes.length - 1].id : cursor).toString(), + }; + } + } else { + const change = await ItemChange.load(id, { fields: options.fields }); + if (!change) throw new ErrorNotFound(); + return change; + } + } +} diff --git a/packages/lib/services/rest/utils/requestFields.ts b/packages/lib/services/rest/utils/requestFields.ts index 6b830f3014..3810525ba6 100644 --- a/packages/lib/services/rest/utils/requestFields.ts +++ b/packages/lib/services/rest/utils/requestFields.ts @@ -11,13 +11,18 @@ function defaultFieldsByModelType(modelType: number): string[] { return output; } -export default function(request: Request, modelType: number) { +export default function(request: Request, modelType: number, defaultFields: string[] = null) { + const getDefaults = () => { + if (defaultFields) return defaultFields; + return defaultFieldsByModelType(modelType); + }; + const query = request.query; - if (!query || !query.fields) return defaultFieldsByModelType(modelType); + if (!query || !query.fields) return getDefaults(); if (Array.isArray(query.fields)) return query.fields.slice(); const fields = query.fields .split(',') .map((f: string) => f.trim()) .filter((f: string) => !!f); - return fields.length ? fields : defaultFieldsByModelType(modelType); + return fields.length ? fields : getDefaults(); }