mirror of
https://github.com/DRYTRIX/TimeTracker.git
synced 2026-01-06 03:30:25 -06:00
feat: Add comprehensive audit trail/history tracking system
Implement a complete audit logging system to track all changes made to tracked entities, providing full compliance and accountability capabilities. Features: - Automatic tracking of create, update, and delete operations on 25+ models - Detailed field-level change tracking with old/new value comparison - User attribution with IP address, user agent, and request path logging - Web UI for viewing and filtering audit logs with pagination - REST API endpoints for programmatic access - Entity-specific history views - Comprehensive test coverage (unit, model, route, and smoke tests) Core Components: - AuditLog model with JSON-encoded value storage and decoding helpers - SQLAlchemy event listeners for automatic change detection - Audit utility module with defensive programming for table existence checks - Blueprint routes for audit log viewing and API access - Jinja2 templates for audit log list, detail, and entity history views - Database migration (044) creating audit_logs table with proper indexes Technical Implementation: - Uses SQLAlchemy 'after_flush' event listener to capture changes - Tracks 25+ models including Projects, Tasks, TimeEntries, Invoices, Clients, Users, etc. - Excludes sensitive fields (passwords) and system fields (id, timestamps) - Implements lazy import pattern to avoid circular dependencies - Graceful error handling to prevent audit logging from breaking core functionality - Transaction-safe logging that integrates with main application transactions Fixes: - Resolved login errors caused by premature transaction commits - Fixed circular import issues with lazy model loading - Added table existence checks to prevent errors before migrations - Improved error handling with debug-level logging for non-critical failures UI/UX: - Added "Audit Logs" link to admin dropdown menu - Organized admin menu into logical sections for better usability - Filterable audit log views by entity type, user, action, and date range - Color-coded action badges and side-by-side old/new value display - Pagination support for large audit log datasets Documentation: - Added comprehensive feature documentation - Included troubleshooting guide and data examples - Created diagnostic scripts for verifying audit log setup Testing: - Unit tests for AuditLog model and value encoding/decoding - Route tests for all audit log endpoints - Integration tests for audit logging functionality - Smoke tests for end-to-end audit trail verification This implementation provides a robust foundation for compliance tracking and change accountability without impacting application performance or requiring code changes in existing routes/models.
This commit is contained in:
Dries Peeters
77
scripts/check_audit_logs.py
Normal file
77
scripts/check_audit_logs.py
Normal file
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python
|
||||
"""Script to check and verify audit_logs table setup"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Add the parent directory to the path so we can import app
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
from app import create_app, db
|
||||
from app.models.audit_log import AuditLog
|
||||
from sqlalchemy import inspect as sqlalchemy_inspect
|
||||
|
||||
def check_audit_table():
|
||||
"""Check if audit_logs table exists and show status"""
|
||||
app = create_app()
|
||||
|
||||
with app.app_context():
|
||||
print("=" * 60)
|
||||
print("Audit Logs Table Check")
|
||||
print("=" * 60)
|
||||
|
||||
# Check if table exists
|
||||
try:
|
||||
inspector = sqlalchemy_inspect(db.engine)
|
||||
tables = inspector.get_table_names()
|
||||
|
||||
if 'audit_logs' in tables:
|
||||
print("✓ audit_logs table EXISTS")
|
||||
|
||||
# Check table structure
|
||||
columns = inspector.get_columns('audit_logs')
|
||||
print(f"\nTable has {len(columns)} columns:")
|
||||
for col in columns:
|
||||
print(f" - {col['name']} ({col['type']})")
|
||||
|
||||
# Check indexes
|
||||
indexes = inspector.get_indexes('audit_logs')
|
||||
print(f"\nTable has {len(indexes)} indexes:")
|
||||
for idx in indexes:
|
||||
print(f" - {idx['name']}: {', '.join(idx['column_names'])}")
|
||||
|
||||
# Count existing audit logs
|
||||
try:
|
||||
count = AuditLog.query.count()
|
||||
print(f"\n✓ Current audit log entries: {count}")
|
||||
|
||||
if count > 0:
|
||||
# Show recent entries
|
||||
recent = AuditLog.query.order_by(AuditLog.created_at.desc()).limit(5).all()
|
||||
print("\nRecent audit log entries:")
|
||||
for log in recent:
|
||||
print(f" - {log.created_at}: {log.action} {log.entity_type}#{log.entity_id} by user#{log.user_id}")
|
||||
except Exception as e:
|
||||
print(f"\n⚠ Could not query audit logs: {e}")
|
||||
print(" This might indicate a schema mismatch.")
|
||||
|
||||
else:
|
||||
print("✗ audit_logs table DOES NOT EXIST")
|
||||
print("\nTo create the table, run:")
|
||||
print(" flask db upgrade")
|
||||
print("\nOr manually apply migration:")
|
||||
print(" migrations/versions/044_add_audit_logs_table.py")
|
||||
|
||||
except Exception as e:
|
||||
print(f"✗ Error checking table: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
return True
|
||||
|
||||
if __name__ == '__main__':
|
||||
success = check_audit_table()
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
63
scripts/test_audit_routes.py
Normal file
63
scripts/test_audit_routes.py
Normal file
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env python
|
||||
"""Test script to verify audit log routes are registered"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Add the parent directory to the path so we can import app
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
try:
|
||||
from app import create_app
|
||||
|
||||
app = create_app()
|
||||
|
||||
print("=" * 60)
|
||||
print("Checking Audit Log Routes")
|
||||
print("=" * 60)
|
||||
|
||||
# Get all routes
|
||||
audit_routes = []
|
||||
for rule in app.url_map.iter_rules():
|
||||
if 'audit' in rule.rule.lower():
|
||||
audit_routes.append({
|
||||
'rule': rule.rule,
|
||||
'endpoint': rule.endpoint,
|
||||
'methods': list(rule.methods)
|
||||
})
|
||||
|
||||
if audit_routes:
|
||||
print(f"\n✓ Found {len(audit_routes)} audit log route(s):\n")
|
||||
for route in audit_routes:
|
||||
print(f" Route: {route['rule']}")
|
||||
print(f" Endpoint: {route['endpoint']}")
|
||||
print(f" Methods: {', '.join(route['methods'])}")
|
||||
print()
|
||||
else:
|
||||
print("\n✗ No audit log routes found!")
|
||||
print("\nChecking for import errors...")
|
||||
|
||||
# Try to import the blueprint
|
||||
try:
|
||||
from app.routes.audit_logs import audit_logs_bp
|
||||
print("✓ Blueprint imported successfully")
|
||||
print(f" Blueprint name: {audit_logs_bp.name}")
|
||||
print(f" Blueprint routes: {len(audit_logs_bp.deferred_functions)}")
|
||||
except Exception as e:
|
||||
print(f"✗ Error importing blueprint: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("All routes containing 'audit':")
|
||||
print("=" * 60)
|
||||
for rule in app.url_map.iter_rules():
|
||||
if 'audit' in rule.rule.lower():
|
||||
print(f" {rule.rule} -> {rule.endpoint} ({', '.join(rule.methods)})")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
127
scripts/verify_audit_setup.py
Normal file
127
scripts/verify_audit_setup.py
Normal file
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python
|
||||
"""Verify audit logs setup - check routes, table, and imports"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Add the parent directory to the path
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
print("=" * 70)
|
||||
print("Audit Logs Setup Verification")
|
||||
print("=" * 70)
|
||||
|
||||
# Test 1: Check if modules can be imported
|
||||
print("\n1. Testing imports...")
|
||||
try:
|
||||
from app.models.audit_log import AuditLog
|
||||
print(" ✓ AuditLog model imported successfully")
|
||||
except Exception as e:
|
||||
print(f" ✗ Failed to import AuditLog: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
from app.utils.audit import check_audit_table_exists, reset_audit_table_cache
|
||||
print(" ✓ Audit utility imported successfully")
|
||||
except Exception as e:
|
||||
print(f" ✗ Failed to import audit utility: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
from app.routes.audit_logs import audit_logs_bp
|
||||
print(" ✓ Audit logs blueprint imported successfully")
|
||||
print(f" Blueprint name: {audit_logs_bp.name}")
|
||||
except Exception as e:
|
||||
print(f" ✗ Failed to import audit_logs blueprint: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
# Test 2: Check routes in blueprint
|
||||
print("\n2. Checking blueprint routes...")
|
||||
routes = []
|
||||
for rule in audit_logs_bp.url_map.iter_rules() if hasattr(audit_logs_bp, 'url_map') else []:
|
||||
routes.append(rule.rule)
|
||||
|
||||
# Check deferred functions (routes not yet registered)
|
||||
if hasattr(audit_logs_bp, 'deferred_functions'):
|
||||
print(f" Found {len(audit_logs_bp.deferred_functions)} deferred route functions")
|
||||
for func in audit_logs_bp.deferred_functions:
|
||||
if hasattr(func, '__name__'):
|
||||
print(f" - {func.__name__}")
|
||||
|
||||
# Test 3: Create app and check registered routes
|
||||
print("\n3. Creating app and checking registered routes...")
|
||||
try:
|
||||
from app import create_app
|
||||
app = create_app()
|
||||
|
||||
# Find all audit-related routes
|
||||
audit_routes = []
|
||||
for rule in app.url_map.iter_rules():
|
||||
if 'audit' in rule.rule.lower():
|
||||
audit_routes.append({
|
||||
'rule': rule.rule,
|
||||
'endpoint': rule.endpoint,
|
||||
'methods': sorted([m for m in rule.methods if m not in ['HEAD', 'OPTIONS']])
|
||||
})
|
||||
|
||||
if audit_routes:
|
||||
print(f" ✓ Found {len(audit_routes)} registered audit log route(s):")
|
||||
for route in audit_routes:
|
||||
print(f" {route['rule']} -> {route['endpoint']} [{', '.join(route['methods'])}]")
|
||||
else:
|
||||
print(" ✗ No audit log routes found in app!")
|
||||
print(" This means the blueprint was not registered properly.")
|
||||
print("\n Checking app initialization...")
|
||||
|
||||
# Check if blueprint is in the app
|
||||
blueprint_names = [bp.name for bp in app.blueprints.values()]
|
||||
if 'audit_logs' in blueprint_names:
|
||||
print(" ✓ Blueprint is registered in app")
|
||||
else:
|
||||
print(" ✗ Blueprint 'audit_logs' NOT found in app blueprints")
|
||||
print(f" Available blueprints: {', '.join(sorted(blueprint_names))}")
|
||||
|
||||
# Test 4: Check database table
|
||||
print("\n4. Checking database table...")
|
||||
with app.app_context():
|
||||
reset_audit_table_cache()
|
||||
table_exists = check_audit_table_exists(force_check=True)
|
||||
|
||||
if table_exists:
|
||||
print(" ✓ audit_logs table exists")
|
||||
try:
|
||||
count = AuditLog.query.count()
|
||||
print(f" Current log count: {count}")
|
||||
except Exception as e:
|
||||
print(f" ⚠ Could not query table: {e}")
|
||||
else:
|
||||
print(" ✗ audit_logs table does NOT exist")
|
||||
print(" Run migration: flask db upgrade")
|
||||
|
||||
# Show available tables
|
||||
try:
|
||||
from sqlalchemy import inspect as sqlalchemy_inspect
|
||||
inspector = sqlalchemy_inspect(app.extensions['sqlalchemy'].db.engine)
|
||||
tables = inspector.get_table_names()
|
||||
print(f"\n Available tables ({len(tables)}):")
|
||||
for table in sorted(tables)[:20]: # Show first 20
|
||||
print(f" - {table}")
|
||||
if len(tables) > 20:
|
||||
print(f" ... and {len(tables) - 20} more")
|
||||
except Exception as e:
|
||||
print(f" Could not list tables: {e}")
|
||||
|
||||
except Exception as e:
|
||||
print(f" ✗ Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("Verification Complete")
|
||||
print("=" * 70)
|
||||
print("\nIf routes are missing, restart your Flask application.")
|
||||
print("If table is missing, run: flask db upgrade")
|
||||
|
||||
Reference in New Issue
Block a user