mirror of
https://github.com/DRYTRIX/TimeTracker.git
synced 2026-02-16 10:38:45 -06:00
Fix AUTH_METHOD=none and add comprehensive schema verification
- Fix AUTH_METHOD=none: Read from Flask app config instead of Config class - Add comprehensive schema verification: Verify all SQLAlchemy models against database and auto-fix missing columns - Improve startup logging: Unified format with timestamps and log levels - Enhanced migration flow: Automatic schema verification after migrations Fixes authentication issue where password field showed even with AUTH_METHOD=none. Ensures all database columns from models exist, preventing missing column errors. Improves startup logging for better debugging and monitoring.
This commit is contained in:
Dries Peeters
260
scripts/fix_missing_columns.py
Normal file
260
scripts/fix_missing_columns.py
Normal file
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Script to manually add missing columns to the users table.
|
||||
This is a workaround for cases where migrations show as applied but columns are missing.
|
||||
|
||||
Usage:
|
||||
python scripts/fix_missing_columns.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
# Get database URL from environment
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql+psycopg2://timetracker:timetracker@localhost:5432/timetracker")
|
||||
|
||||
|
||||
def has_column(engine, table_name, column_name):
|
||||
"""Check if a column exists in a table"""
|
||||
inspector = inspect(engine)
|
||||
try:
|
||||
columns = [col['name'] for col in inspector.get_columns(table_name)]
|
||||
return column_name in columns
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def add_column_if_missing(engine, table_name, column_name, column_type, nullable=True, default=None):
|
||||
"""Add a column to a table if it doesn't exist"""
|
||||
if has_column(engine, table_name, column_name):
|
||||
print(f"✓ Column '{column_name}' already exists in '{table_name}'")
|
||||
return False
|
||||
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
# Build ALTER TABLE statement
|
||||
alter_sql = f"ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type}"
|
||||
if not nullable:
|
||||
alter_sql += " NOT NULL"
|
||||
if default is not None:
|
||||
alter_sql += f" DEFAULT {default}"
|
||||
|
||||
conn.execute(text(alter_sql))
|
||||
conn.commit()
|
||||
print(f"✓ Added column '{column_name}' to '{table_name}'")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"✗ Failed to add column '{column_name}' to '{table_name}': {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to add missing columns"""
|
||||
print("=" * 60)
|
||||
print("TimeTracker - Fix Missing Database Columns")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
try:
|
||||
engine = create_engine(DATABASE_URL)
|
||||
|
||||
# Test connection
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text("SELECT 1"))
|
||||
print("✓ Database connection successful")
|
||||
print()
|
||||
|
||||
# Check if users table exists
|
||||
inspector = inspect(engine)
|
||||
if 'users' not in inspector.get_table_names():
|
||||
print("✗ 'users' table does not exist. Please run migrations first.")
|
||||
return 1
|
||||
|
||||
print("Checking and adding missing columns to 'users' table...")
|
||||
print()
|
||||
|
||||
# List of columns that should exist based on the User model
|
||||
# These are the columns that are commonly missing after migration issues
|
||||
columns_to_add = [
|
||||
{
|
||||
'name': 'password_hash',
|
||||
'type': 'VARCHAR(255)',
|
||||
'nullable': True,
|
||||
'default': None
|
||||
},
|
||||
{
|
||||
'name': 'password_change_required',
|
||||
'type': 'BOOLEAN',
|
||||
'nullable': False,
|
||||
'default': 'false'
|
||||
},
|
||||
{
|
||||
'name': 'email',
|
||||
'type': 'VARCHAR(200)',
|
||||
'nullable': True,
|
||||
'default': None
|
||||
},
|
||||
{
|
||||
'name': 'full_name',
|
||||
'type': 'VARCHAR(200)',
|
||||
'nullable': True,
|
||||
'default': None
|
||||
},
|
||||
{
|
||||
'name': 'theme_preference',
|
||||
'type': 'VARCHAR(10)',
|
||||
'nullable': True,
|
||||
'default': None
|
||||
},
|
||||
{
|
||||
'name': 'preferred_language',
|
||||
'type': 'VARCHAR(8)',
|
||||
'nullable': True,
|
||||
'default': None
|
||||
},
|
||||
{
|
||||
'name': 'oidc_sub',
|
||||
'type': 'VARCHAR(255)',
|
||||
'nullable': True,
|
||||
'default': None
|
||||
},
|
||||
{
|
||||
'name': 'oidc_issuer',
|
||||
'type': 'VARCHAR(255)',
|
||||
'nullable': True,
|
||||
'default': None
|
||||
},
|
||||
{
|
||||
'name': 'avatar_filename',
|
||||
'type': 'VARCHAR(255)',
|
||||
'nullable': True,
|
||||
'default': None
|
||||
},
|
||||
{
|
||||
'name': 'email_notifications',
|
||||
'type': 'BOOLEAN',
|
||||
'nullable': False,
|
||||
'default': 'true'
|
||||
},
|
||||
{
|
||||
'name': 'notification_overdue_invoices',
|
||||
'type': 'BOOLEAN',
|
||||
'nullable': False,
|
||||
'default': 'true'
|
||||
},
|
||||
{
|
||||
'name': 'notification_task_assigned',
|
||||
'type': 'BOOLEAN',
|
||||
'nullable': False,
|
||||
'default': 'true'
|
||||
},
|
||||
{
|
||||
'name': 'notification_task_comments',
|
||||
'type': 'BOOLEAN',
|
||||
'nullable': False,
|
||||
'default': 'true'
|
||||
},
|
||||
{
|
||||
'name': 'notification_weekly_summary',
|
||||
'type': 'BOOLEAN',
|
||||
'nullable': False,
|
||||
'default': 'false'
|
||||
},
|
||||
{
|
||||
'name': 'timezone',
|
||||
'type': 'VARCHAR(50)',
|
||||
'nullable': True,
|
||||
'default': None
|
||||
},
|
||||
{
|
||||
'name': 'date_format',
|
||||
'type': 'VARCHAR(20)',
|
||||
'nullable': False,
|
||||
'default': "'YYYY-MM-DD'"
|
||||
},
|
||||
{
|
||||
'name': 'time_format',
|
||||
'type': 'VARCHAR(10)',
|
||||
'nullable': False,
|
||||
'default': "'24h'"
|
||||
},
|
||||
{
|
||||
'name': 'week_start_day',
|
||||
'type': 'INTEGER',
|
||||
'nullable': False,
|
||||
'default': '1'
|
||||
},
|
||||
{
|
||||
'name': 'time_rounding_enabled',
|
||||
'type': 'BOOLEAN',
|
||||
'nullable': False,
|
||||
'default': 'true'
|
||||
},
|
||||
{
|
||||
'name': 'time_rounding_minutes',
|
||||
'type': 'INTEGER',
|
||||
'nullable': False,
|
||||
'default': '1'
|
||||
},
|
||||
{
|
||||
'name': 'time_rounding_method',
|
||||
'type': 'VARCHAR(10)',
|
||||
'nullable': False,
|
||||
'default': "'nearest'"
|
||||
},
|
||||
{
|
||||
'name': 'standard_hours_per_day',
|
||||
'type': 'FLOAT',
|
||||
'nullable': False,
|
||||
'default': '8.0'
|
||||
},
|
||||
{
|
||||
'name': 'client_portal_enabled',
|
||||
'type': 'BOOLEAN',
|
||||
'nullable': False,
|
||||
'default': 'false'
|
||||
},
|
||||
{
|
||||
'name': 'client_id',
|
||||
'type': 'INTEGER',
|
||||
'nullable': True,
|
||||
'default': None
|
||||
},
|
||||
]
|
||||
|
||||
added_count = 0
|
||||
for col in columns_to_add:
|
||||
if add_column_if_missing(
|
||||
engine,
|
||||
'users',
|
||||
col['name'],
|
||||
col['type'],
|
||||
col['nullable'],
|
||||
col['default']
|
||||
):
|
||||
added_count += 1
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
if added_count > 0:
|
||||
print(f"✓ Successfully added {added_count} missing column(s)")
|
||||
else:
|
||||
print("✓ All columns already exist")
|
||||
print("=" * 60)
|
||||
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
print(f"✗ Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
248
scripts/verify_and_fix_schema.py
Normal file
248
scripts/verify_and_fix_schema.py
Normal file
@@ -0,0 +1,248 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Comprehensive schema verification and fix script.
|
||||
This script checks all SQLAlchemy models against the actual database schema
|
||||
and adds any missing columns based on the model definitions.
|
||||
|
||||
Usage:
|
||||
python scripts/verify_and_fix_schema.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from sqlalchemy import create_engine, inspect, text, MetaData
|
||||
from sqlalchemy.exc import OperationalError
|
||||
from sqlalchemy.schema import CreateTable
|
||||
from sqlalchemy.dialects import postgresql, sqlite
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
|
||||
def get_sqlalchemy_type(column_type, dialect):
|
||||
"""Convert SQLAlchemy column type to SQL string for the given dialect"""
|
||||
if dialect == 'postgresql':
|
||||
return str(column_type.compile(dialect=postgresql.dialect()))
|
||||
else:
|
||||
return str(column_type.compile(dialect=sqlite.dialect()))
|
||||
|
||||
|
||||
def get_column_default(column, dialect):
|
||||
"""Get the default value for a column as SQL string"""
|
||||
if column.default is None:
|
||||
return None
|
||||
|
||||
# Handle server defaults (like server_default=text("CURRENT_TIMESTAMP"))
|
||||
if hasattr(column, 'server_default') and column.server_default is not None:
|
||||
if hasattr(column.server_default, 'arg'):
|
||||
default_text = str(column.server_default.arg)
|
||||
# Remove quotes if it's a function call
|
||||
if default_text.startswith("'") and default_text.endswith("'"):
|
||||
default_text = default_text[1:-1]
|
||||
return default_text
|
||||
|
||||
# Handle Python defaults
|
||||
if hasattr(column.default, 'arg'):
|
||||
default_arg = column.default.arg
|
||||
if isinstance(default_arg, str):
|
||||
# Escape single quotes in strings
|
||||
escaped = default_arg.replace("'", "''")
|
||||
return f"'{escaped}'"
|
||||
elif isinstance(default_arg, (int, float)):
|
||||
return str(default_arg)
|
||||
elif isinstance(default_arg, bool):
|
||||
return 'true' if default_arg else 'false'
|
||||
elif callable(default_arg):
|
||||
# For callable defaults like datetime.utcnow, skip default
|
||||
# The database will handle NULL values
|
||||
return None
|
||||
elif hasattr(column.default, 'text'):
|
||||
return column.default.text
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def has_column(inspector, table_name, column_name):
|
||||
"""Check if a column exists in a table"""
|
||||
try:
|
||||
columns = [col['name'] for col in inspector.get_columns(table_name)]
|
||||
return column_name in columns
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def add_column_sql(table_name, column, dialect):
|
||||
"""Generate SQL to add a column"""
|
||||
col_type = get_sqlalchemy_type(column.type, dialect)
|
||||
nullable = "NULL" if column.nullable else "NOT NULL"
|
||||
default = get_column_default(column, dialect)
|
||||
|
||||
# Build SQL statement
|
||||
sql_parts = [f"ALTER TABLE {table_name} ADD COLUMN {column.name} {col_type}"]
|
||||
|
||||
# Add default if specified
|
||||
if default is not None:
|
||||
sql_parts.append(f"DEFAULT {default}")
|
||||
|
||||
# Add nullable constraint
|
||||
sql_parts.append(nullable)
|
||||
|
||||
return " ".join(sql_parts)
|
||||
|
||||
|
||||
def verify_and_fix_table(engine, inspector, model_class, dialect):
|
||||
"""Verify and fix columns for a single table"""
|
||||
table_name = model_class.__tablename__
|
||||
|
||||
# Check if table exists
|
||||
if table_name not in inspector.get_table_names():
|
||||
print(f"⚠ Table '{table_name}' does not exist (will be created by migrations)")
|
||||
return 0
|
||||
|
||||
# Get expected columns from model
|
||||
expected_columns = {}
|
||||
for column in model_class.__table__.columns:
|
||||
expected_columns[column.name] = column
|
||||
|
||||
# Get actual columns from database
|
||||
try:
|
||||
actual_columns = {col['name']: col for col in inspector.get_columns(table_name)}
|
||||
except Exception as e:
|
||||
print(f"✗ Error inspecting table '{table_name}': {e}")
|
||||
return 0
|
||||
|
||||
# Find missing columns
|
||||
missing_columns = []
|
||||
for col_name, col_def in expected_columns.items():
|
||||
if col_name not in actual_columns:
|
||||
missing_columns.append((col_name, col_def))
|
||||
|
||||
if not missing_columns:
|
||||
return 0
|
||||
|
||||
# Add missing columns
|
||||
added_count = 0
|
||||
with engine.begin() as conn: # Use begin() for automatic transaction management
|
||||
for col_name, col_def in missing_columns:
|
||||
try:
|
||||
sql = add_column_sql(table_name, col_def, dialect)
|
||||
# Execute with explicit transaction
|
||||
conn.execute(text(sql))
|
||||
print(f" ✓ Added column '{col_name}' to '{table_name}'")
|
||||
added_count += 1
|
||||
except Exception as e:
|
||||
# Log error but continue with other columns
|
||||
error_msg = str(e)
|
||||
# Don't fail on "column already exists" errors (race condition)
|
||||
if "already exists" not in error_msg.lower() and "duplicate" not in error_msg.lower():
|
||||
print(f" ✗ Failed to add column '{col_name}' to '{table_name}': {error_msg}")
|
||||
else:
|
||||
print(f" ⚠ Column '{col_name}' already exists in '{table_name}' (skipping)")
|
||||
|
||||
return added_count
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to verify and fix database schema"""
|
||||
print("=" * 70)
|
||||
print("TimeTracker - Comprehensive Schema Verification and Fix")
|
||||
print("=" * 70)
|
||||
print()
|
||||
|
||||
# Get database URL from environment
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql+psycopg2://timetracker:timetracker@localhost:5432/timetracker")
|
||||
|
||||
try:
|
||||
engine = create_engine(DATABASE_URL, pool_pre_ping=True)
|
||||
|
||||
# Test connection
|
||||
with engine.connect() as conn:
|
||||
conn.execute(text("SELECT 1"))
|
||||
print("✓ Database connection successful")
|
||||
|
||||
# Detect database dialect
|
||||
dialect = engine.dialect.name
|
||||
print(f"✓ Database dialect: {dialect}")
|
||||
print()
|
||||
|
||||
# Create inspector
|
||||
inspector = inspect(engine)
|
||||
|
||||
# Import all models
|
||||
print("Loading SQLAlchemy models...")
|
||||
try:
|
||||
from app import create_app
|
||||
app = create_app()
|
||||
with app.app_context():
|
||||
# Import all models dynamically from app.models
|
||||
from app.models import __all__ as model_names
|
||||
import app.models as models_module
|
||||
|
||||
# Get all model classes
|
||||
models = []
|
||||
for name in model_names:
|
||||
try:
|
||||
model_class = getattr(models_module, name)
|
||||
if hasattr(model_class, '__tablename__'):
|
||||
models.append(model_class)
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
# Also get any models that might not be in __all__
|
||||
# This ensures we catch everything
|
||||
for attr_name in dir(models_module):
|
||||
if not attr_name.startswith('_'):
|
||||
attr = getattr(models_module, attr_name)
|
||||
if (hasattr(attr, '__tablename__') and
|
||||
hasattr(attr, '__table__') and
|
||||
attr not in models):
|
||||
models.append(attr)
|
||||
|
||||
print(f"✓ Loaded {len(models)} model classes")
|
||||
print()
|
||||
print("Verifying database schema...")
|
||||
print()
|
||||
|
||||
total_added = 0
|
||||
tables_checked = 0
|
||||
|
||||
for model in models:
|
||||
if hasattr(model, '__tablename__'):
|
||||
tables_checked += 1
|
||||
added = verify_and_fix_table(engine, inspector, model, dialect)
|
||||
total_added += added
|
||||
if added > 0:
|
||||
print(f" → Fixed {added} column(s) in '{model.__tablename__}'")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print(f"✓ Schema verification complete")
|
||||
print(f" - Tables checked: {tables_checked}")
|
||||
print(f" - Columns added: {total_added}")
|
||||
print("=" * 70)
|
||||
|
||||
return 0 if total_added == 0 else 0 # Return 0 even if columns were added (success)
|
||||
|
||||
except ImportError as e:
|
||||
print(f"✗ Error importing models: {e}")
|
||||
print(" This script must be run from the application root directory")
|
||||
return 1
|
||||
except Exception as e:
|
||||
print(f"✗ Error during schema verification: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
except OperationalError as e:
|
||||
print(f"✗ Database connection error: {e}")
|
||||
print(" Please check your DATABASE_URL environment variable")
|
||||
return 1
|
||||
except Exception as e:
|
||||
print(f"✗ Unexpected error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user