fix: change keyfile watcher to poll instead of inotify on FAT32 (#1820)

## Summary

- Fixed GraphQL registration state not updating when license keys are
installed/upgraded
- Root cause: /boot/config is on FAT32 which doesn't support inotify -
the file watcher was silently failing

  ## Changes

  - Enable polling for key file watcher (required for FAT32 filesystem)
- Add retry logic to reload var.ini after key changes to handle emhttpd
update timing variation

  ## Test plan

  - Unit tests for retry logic (will run in CI)
- Manual test on Unraid: install/upgrade license key, verify GraphQL
returns updated state within ~8 seconds

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **Tests**
* Added a comprehensive test suite covering retry behavior, exponential
backoff timing, and various registration-change scenarios.

* **Refactor**
* Switched registration key monitoring to a polling-based watcher with
an exponential-backoff retry for config reloads; added event logging and
improved retry/stopping behavior to make state updates more reliable and
observable.

<sub>✏️ Tip: You can customize this high-level summary in your review
settings.</sub>
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

api/src/__test__/store/watch/registration-watch.test.ts

@@ -0,0 +1,151 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { StateFileKey } from '@app/store/types.js';
+import { RegistrationType } from '@app/unraid-api/graph/resolvers/registration/registration.model.js';
+
+// Mock the store module
+vi.mock('@app/store/index.js', () => ({
+    store: {
+        dispatch: vi.fn(),
+    },
+    getters: {
+        emhttp: vi.fn(),
+    },
+}));
+
+// Mock the emhttp module
+vi.mock('@app/store/modules/emhttp.js', () => ({
+    loadSingleStateFile: vi.fn((key) => ({ type: 'emhttp/load-single-state-file', payload: key })),
+}));
+
+// Mock the registration module
+vi.mock('@app/store/modules/registration.js', () => ({
+    loadRegistrationKey: vi.fn(() => ({ type: 'registration/load-registration-key' })),
+}));
+
+// Mock the logger
+vi.mock('@app/core/log.js', () => ({
+    keyServerLogger: {
+        info: vi.fn(),
+        debug: vi.fn(),
+    },
+}));
+
+describe('reloadVarIniWithRetry', () => {
+    let store: { dispatch: ReturnType<typeof vi.fn> };
+    let getters: { emhttp: ReturnType<typeof vi.fn> };
+    let loadSingleStateFile: ReturnType<typeof vi.fn>;
+
+    beforeEach(async () => {
+        vi.useFakeTimers();
+
+        const storeModule = await import('@app/store/index.js');
+        const emhttpModule = await import('@app/store/modules/emhttp.js');
+
+        store = storeModule.store as unknown as typeof store;
+        getters = storeModule.getters as unknown as typeof getters;
+        loadSingleStateFile = emhttpModule.loadSingleStateFile as unknown as typeof loadSingleStateFile;
+
+        vi.clearAllMocks();
+    });
+
+    afterEach(() => {
+        vi.useRealTimers();
+    });
+
+    it('returns early when registration state changes on first retry', async () => {
+        // Initial state is TRIAL
+        getters.emhttp
+            .mockReturnValueOnce({ var: { regTy: RegistrationType.TRIAL } }) // First call (beforeState)
+            .mockReturnValueOnce({ var: { regTy: RegistrationType.UNLEASHED } }); // After first reload
+
+        const { reloadVarIniWithRetry } = await import('@app/store/watch/registration-watch.js');
+
+        const promise = reloadVarIniWithRetry();
+
+        // Advance past the first delay (500ms)
+        await vi.advanceTimersByTimeAsync(500);
+        await promise;
+
+        // Should only dispatch once since state changed
+        expect(store.dispatch).toHaveBeenCalledTimes(1);
+        expect(loadSingleStateFile).toHaveBeenCalledWith(StateFileKey.var);
+    });
+
+    it('retries up to maxRetries when state does not change', async () => {
+        // State never changes
+        getters.emhttp.mockReturnValue({ var: { regTy: RegistrationType.TRIAL } });
+
+        const { reloadVarIniWithRetry } = await import('@app/store/watch/registration-watch.js');
+
+        const promise = reloadVarIniWithRetry(3);
+
+        // Advance through all retries: 500ms, 1000ms, 2000ms
+        await vi.advanceTimersByTimeAsync(500);
+        await vi.advanceTimersByTimeAsync(1000);
+        await vi.advanceTimersByTimeAsync(2000);
+        await promise;
+
+        // Should dispatch 3 times (maxRetries)
+        expect(store.dispatch).toHaveBeenCalledTimes(3);
+    });
+
+    it('stops retrying when state changes on second attempt', async () => {
+        getters.emhttp
+            .mockReturnValueOnce({ var: { regTy: RegistrationType.TRIAL } }) // beforeState
+            .mockReturnValueOnce({ var: { regTy: RegistrationType.TRIAL } }) // After first reload (no change)
+            .mockReturnValueOnce({ var: { regTy: RegistrationType.UNLEASHED } }); // After second reload (changed!)
+
+        const { reloadVarIniWithRetry } = await import('@app/store/watch/registration-watch.js');
+
+        const promise = reloadVarIniWithRetry(3);
+
+        // First retry
+        await vi.advanceTimersByTimeAsync(500);
+        // Second retry
+        await vi.advanceTimersByTimeAsync(1000);
+        await promise;
+
+        // Should dispatch twice - stopped after state changed
+        expect(store.dispatch).toHaveBeenCalledTimes(2);
+    });
+
+    it('handles undefined regTy gracefully', async () => {
+        getters.emhttp.mockReturnValue({ var: {} });
+
+        const { reloadVarIniWithRetry } = await import('@app/store/watch/registration-watch.js');
+
+        const promise = reloadVarIniWithRetry(1);
+
+        await vi.advanceTimersByTimeAsync(500);
+        await promise;
+
+        // Should still dispatch even with undefined regTy
+        expect(store.dispatch).toHaveBeenCalledTimes(1);
+    });
+
+    it('uses exponential backoff delays', async () => {
+        getters.emhttp.mockReturnValue({ var: { regTy: RegistrationType.TRIAL } });
+
+        const { reloadVarIniWithRetry } = await import('@app/store/watch/registration-watch.js');
+
+        const promise = reloadVarIniWithRetry(3);
+
+        // At 0ms, no dispatch yet
+        expect(store.dispatch).toHaveBeenCalledTimes(0);
+
+        // At 500ms, first dispatch
+        await vi.advanceTimersByTimeAsync(500);
+        expect(store.dispatch).toHaveBeenCalledTimes(1);
+
+        // At 1500ms (500 + 1000), second dispatch
+        await vi.advanceTimersByTimeAsync(1000);
+        expect(store.dispatch).toHaveBeenCalledTimes(2);
+
+        // At 3500ms (500 + 1000 + 2000), third dispatch
+        await vi.advanceTimersByTimeAsync(2000);
+        expect(store.dispatch).toHaveBeenCalledTimes(3);
+
+        await promise;
+    });
+});

api/src/store/watch/registration-watch.ts

@@ -1,17 +1,51 @@
 import { watch } from 'chokidar';
 
-import { CHOKIDAR_USEPOLLING } from '@app/environment.js';
-import { store } from '@app/store/index.js';
+import { keyServerLogger } from '@app/core/log.js';
+import { getters, store } from '@app/store/index.js';
+import { loadSingleStateFile } from '@app/store/modules/emhttp.js';
 import { loadRegistrationKey } from '@app/store/modules/registration.js';
+import { StateFileKey } from '@app/store/types.js';
+
+/**
+ * Reloads var.ini with retry logic to handle timing issues with emhttpd.
+ * When a key file changes, emhttpd needs time to process it and update var.ini.
+ * This function retries loading var.ini until the registration state changes
+ * or max retries are exhausted.
+ */
+export const reloadVarIniWithRetry = async (maxRetries = 3): Promise<void> => {
+    const beforeState = getters.emhttp().var?.regTy;
+
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+        const delay = 500 * Math.pow(2, attempt); // 500ms, 1s, 2s
+        await new Promise((resolve) => setTimeout(resolve, delay));
+
+        await store.dispatch(loadSingleStateFile(StateFileKey.var));
+
+        const afterState = getters.emhttp().var?.regTy;
+        if (beforeState !== afterState) {
+            keyServerLogger.info('Registration state updated: %s -> %s', beforeState, afterState);
+            return;
+        }
+        keyServerLogger.debug('Retry %d: var.ini regTy still %s', attempt + 1, afterState);
+    }
+    keyServerLogger.debug('var.ini regTy unchanged after %d retries (may be expected)', maxRetries);
+};
 
 export const setupRegistrationKeyWatch = () => {
+    // IMPORTANT: /boot/config is on FAT32 flash drive which does NOT support inotify
+    // Must use polling to detect file changes on FAT32 filesystems
     watch('/boot/config', {
         persistent: true,
         ignoreInitial: true,
         ignored: (path: string) => !path.endsWith('.key'),
-        usePolling: CHOKIDAR_USEPOLLING === true,
-    }).on('all', async () => {
-        // Load updated key into store
+        usePolling: true, // Required for FAT32 - inotify doesn't work
+        interval: 5000, // Poll every 5 seconds (balance between responsiveness and CPU usage)
+    }).on('all', async (event, path) => {
+        keyServerLogger.info('Key file %s: %s', event, path);
+
         await store.dispatch(loadRegistrationKey());
+
+        // Reload var.ini to get updated registration metadata from emhttpd
+        await reloadVarIniWithRetry();
     });
 };