From e8bf79b3253ccfae85cc380abb78753ecd9fd679 Mon Sep 17 00:00:00 2001 From: Dries Peeters Date: Sat, 3 Jan 2026 20:27:44 +0100 Subject: [PATCH] build: precompile translations at Docker build time - Add pybabel compile step in Dockerfile for faster startup - Prevents runtime translation compilation overhead - Gracefully handles missing Babel during build --- Dockerfile | 4 + docker/entrypoint_fixed.sh | 118 +++++++--- docker/start-fixed.py | 440 ++++++++++++++++++++++++++++++++++--- 3 files changed, 500 insertions(+), 62 deletions(-) diff --git a/Dockerfile b/Dockerfile index f6ad05e..611dcee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -93,6 +93,10 @@ RUN mkdir -p \ # Copy the startup script COPY --chown=timetracker:timetracker docker/start-fixed.py /app/start.py +# Precompile translations at build time for faster startup (no runtime pybabel calls). +# If Babel isn't available for some reason, don't fail the image build. +RUN pybabel compile -d /app/translations >/dev/null 2>&1 || true + # Fix line endings and set permissions in a single layer RUN find /app/docker -name "*.sh" -o -name "*.py" | xargs dos2unix 2>/dev/null || true \ && dos2unix /app/start.py /scripts/generate-certs.sh 2>/dev/null || true \ diff --git a/docker/entrypoint_fixed.sh b/docker/entrypoint_fixed.sh index eeb40bb..3ffaaa7 100644 --- a/docker/entrypoint_fixed.sh +++ b/docker/entrypoint_fixed.sh @@ -1,19 +1,29 @@ #!/bin/bash -# TimeTracker Docker Entrypoint with Automatic Migration Detection -# This script automatically detects database state and chooses the correct migration strategy - -# Don't exit on errors - let the script continue and show what's happening -# set -e +# Minimal TimeTracker entrypoint. +# +# Startup responsibilities (DB wait/migrations/seeding) are handled by /app/start.py. +# Keeping this entrypoint tiny avoids duplicated work and reduces cold-start time. +set -Eeuo pipefail echo "=== TimeTracker Docker Container Starting ===" echo "Timestamp: $(date)" echo "Container ID: $(hostname)" echo "Python version: $(python --version 2>/dev/null || echo 'Python not available')" -echo "Flask version: $(flask --version 2>/dev/null || echo 'Flask CLI not available')" echo "Current directory: $(pwd)" echo "User: $(whoami)" echo +# Best-effort: ensure writable dirs exist (container runs as non-root). +mkdir -p /data /app/logs 2>/dev/null || true +if [[ ! -w "/data" ]]; then + echo "[WARN] /data is not writable for $(whoami); persistence/uploads may fail." +fi + +exec "$@" + +# Everything below is disabled legacy logic kept only for reference. +: <<'DISABLED_LEGACY_ENTRYPOINT' + # Function to log messages with timestamp log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" @@ -1138,75 +1148,128 @@ verify_database_integrity() { # Test basic database operations if [[ "$db_url" == postgresql* ]]; then log "Checking PostgreSQL database integrity..." - if python -c " + if python3 - "$db_url" << PYTHON_SCRIPT import psycopg2 +import sys +from urllib.parse import urlparse try: # Parse connection string to remove +psycopg2 if present - conn_str = '$db_url'.replace('+psycopg2://', '://') - conn = psycopg2.connect(conn_str) + db_url = sys.argv[1] if len(sys.argv) > 1 else '' + if not db_url: + print('Error: No database URL provided') + sys.exit(1) + + # Parse URL properly + clean_url = db_url.replace('+psycopg2://', '://') + parsed = urlparse(clean_url) + + # Extract connection parameters explicitly to avoid Unix socket fallback + host = parsed.hostname or 'db' + port = parsed.port or 5432 + database = parsed.path.lstrip('/') or 'timetracker' + user = parsed.username or 'timetracker' + password = parsed.password or 'timetracker' + + # Use explicit connection parameters to force TCP/IP connection + conn = psycopg2.connect( + host=host, + port=port, + database=database, + user=user, + password=password, + connect_timeout=5 + ) cursor = conn.cursor() # Check for key tables that should exist after migration - cursor.execute(\"\"\" + cursor.execute(""" SELECT table_name FROM information_schema.tables WHERE table_name IN ('clients', 'users', 'projects', 'tasks', 'time_entries', 'settings', 'invoices', 'invoice_items') AND table_schema = 'public' ORDER BY table_name - \"\"\") + """) key_tables = [row[0] for row in cursor.fetchall()] # Also check if alembic_version table exists - cursor.execute(\"\"\" + cursor.execute(""" SELECT table_name FROM information_schema.tables WHERE table_name = 'alembic_version' AND table_schema = 'public' - \"\"\") + """) alembic_table = cursor.fetchone() conn.close() print(f'Found tables: {key_tables}') - print(f'Alembic version table: {alembic_table[0] if alembic_table else \"missing\"}') + print(f'Alembic version table: {alembic_table[0] if alembic_table else "missing"}') # Require at least the core tables and alembic_version if len(key_tables) >= 5 and alembic_table: - exit(0) + sys.exit(0) else: print(f'Expected at least 5 core tables, found {len(key_tables)}') print(f'Expected alembic_version table: {bool(alembic_table)}') - exit(1) + sys.exit(1) except Exception as e: print(f'Error during integrity check: {e}') - exit(1) -"; then + import traceback + traceback.print_exc() + sys.exit(1) +PYTHON_SCRIPT +then log "✓ Database integrity verified (PostgreSQL via Python)" return 0 else log "✗ Database integrity check failed (PostgreSQL)" log "Error details:" - python -c " + python3 - "$db_url" << PYTHON_SCRIPT 2>&1 | head -20 import psycopg2 +import sys +from urllib.parse import urlparse try: - conn_str = '$db_url'.replace('+psycopg2://', '://') - conn = psycopg2.connect(conn_str) + db_url = sys.argv[1] if len(sys.argv) > 1 else '' + if not db_url: + print('Error: No database URL provided') + sys.exit(1) + + # Parse URL properly + clean_url = db_url.replace('+psycopg2://', '://') + parsed = urlparse(clean_url) + + # Extract connection parameters explicitly to avoid Unix socket fallback + host = parsed.hostname or 'db' + port = parsed.port or 5432 + database = parsed.path.lstrip('/') or 'timetracker' + user = parsed.username or 'timetracker' + password = parsed.password or 'timetracker' + + # Use explicit connection parameters to force TCP/IP connection + conn = psycopg2.connect( + host=host, + port=port, + database=database, + user=user, + password=password, + connect_timeout=5 + ) cursor = conn.cursor() - cursor.execute(\"\"\" + cursor.execute(""" SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name - \"\"\") + """) all_tables = [row[0] for row in cursor.fetchall()] - cursor.execute(\"\"\" + cursor.execute(""" SELECT table_name FROM information_schema.tables WHERE table_name = 'alembic_version' AND table_schema = 'public' - \"\"\) + """) alembic_table = cursor.fetchone() conn.close() @@ -1215,7 +1278,9 @@ try: print(f'Alembic version table exists: {bool(alembic_table)}') except Exception as e: print(f'Error getting table list: {e}') -" 2>&1 | head -20 + import traceback + traceback.print_exc() +PYTHON_SCRIPT if [[ $attempt -lt $max_attempts ]]; then log "Waiting 3 seconds before retry..." @@ -1378,3 +1443,4 @@ main() { # Run main function with all arguments main "$@" +DISABLED_LEGACY_ENTRYPOINT diff --git a/docker/start-fixed.py b/docker/start-fixed.py index 343cb64..fa9959f 100644 --- a/docker/start-fixed.py +++ b/docker/start-fixed.py @@ -12,6 +12,9 @@ import traceback import psycopg2 from urllib.parse import urlparse +def _truthy(v: str) -> bool: + return str(v or "").strip().lower() in ("1", "true", "yes", "y", "on") + def wait_for_database(): """Wait for database to be ready with proper connection testing""" # Logging is handled by main() @@ -97,21 +100,395 @@ def wait_for_database(): return False -def run_script(script_path, description): - """Run a Python script with proper error handling""" +def detect_corrupted_database_state(app): + """Detect if database is in a corrupted/inconsistent state. + + Returns: (is_corrupted: bool, reason: str) + """ try: - result = subprocess.run( - [sys.executable, script_path], - check=True, - capture_output=False, # Let the script output directly - text=True - ) - return True - except subprocess.CalledProcessError as e: - log(f"{description} failed with exit code {e.returncode}", "ERROR") - return False + from app import db + from sqlalchemy import text + + with app.app_context(): + # Check PostgreSQL + if os.getenv("DATABASE_URL", "").startswith("postgresql"): + # Get all tables + all_tables_result = db.session.execute( + text("SELECT table_name FROM information_schema.tables WHERE table_schema='public' ORDER BY table_name") + ) + all_tables = [row[0] for row in all_tables_result] + + # Check for alembic_version + has_alembic_version = 'alembic_version' in all_tables + + # Check for core tables + core_tables = ['users', 'projects', 'time_entries', 'settings', 'clients'] + has_core_tables = any(t in all_tables for t in core_tables) + + # Database is corrupted if: + # 1. Has tables but no alembic_version (migrations were never applied) + # 2. Has tables but no core tables (partial/corrupted state) + # 3. Has alembic_version but no core tables (migrations failed) + + if len(all_tables) > 0 and not has_alembic_version and not has_core_tables: + # Has tables but they're not our tables - likely test/manual tables + return True, f"Database has {len(all_tables)} table(s) but no alembic_version or core tables. Tables: {all_tables}" + + if has_alembic_version and not has_core_tables: + return True, "alembic_version exists but core tables are missing - migrations may have failed" + + if len(all_tables) > 0 and has_core_tables and not has_alembic_version: + return True, "Core tables exist but alembic_version is missing - database state is inconsistent" + + # SQLite + else: + all_tables_result = db.session.execute( + text("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name") + ) + all_tables = [row[0] for row in all_tables_result] + + has_alembic_version = 'alembic_version' in all_tables + core_tables = ['users', 'projects', 'time_entries', 'settings', 'clients'] + has_core_tables = any(t in all_tables for t in core_tables) + + if len(all_tables) > 0 and not has_alembic_version and not has_core_tables: + return True, f"Database has {len(all_tables)} table(s) but no alembic_version or core tables" + + if has_alembic_version and not has_core_tables: + return True, "alembic_version exists but core tables are missing" + + if len(all_tables) > 0 and has_core_tables and not has_alembic_version: + return True, "Core tables exist but alembic_version is missing" + + return False, "" except Exception as e: - log(f"Unexpected error running {description}: {e}", "ERROR") + # Can't determine - assume not corrupted + return False, f"Could not check database state: {e}" + + +def cleanup_corrupted_database_state(app): + """Attempt to clean up corrupted database state by removing unexpected tables. + + This is only safe when: + - Database has tables but NO alembic_version (migrations never ran) + - Database has tables but NO core tables (corrupted/partial state) + - User can disable with TT_SKIP_DB_CLEANUP env var + + Only removes tables when it's safe (no alembic_version = migrations haven't run yet). + """ + if os.getenv("TT_SKIP_DB_CLEANUP", "").strip().lower() in ("1", "true", "yes"): + log("Database cleanup skipped (TT_SKIP_DB_CLEANUP is set)", "INFO") + return False + + try: + from app import db + from sqlalchemy import text + + with app.app_context(): + # Only cleanup if PostgreSQL (SQLite cleanup is more risky) + if not os.getenv("DATABASE_URL", "").startswith("postgresql"): + return False + + # Get all tables + all_tables_result = db.session.execute( + text("SELECT table_name FROM information_schema.tables WHERE table_schema='public' ORDER BY table_name") + ) + all_tables = [row[0] for row in all_tables_result] + + # Check if alembic_version exists + has_alembic_version = 'alembic_version' in all_tables + + # Only cleanup if alembic_version does NOT exist (migrations haven't run) + # If alembic_version exists, migrations have run and we shouldn't drop tables + if has_alembic_version: + log("alembic_version table exists - skipping cleanup (migrations may have run)", "INFO") + return False + + # Check for core tables + core_tables = ['users', 'projects', 'time_entries', 'settings', 'clients'] + has_core_tables = any(t in all_tables for t in core_tables) + + # Only cleanup if we have tables but no core tables (corrupted state) + if not all_tables: + return False # Empty database, nothing to clean + + if has_core_tables: + log("Core tables exist - skipping cleanup", "INFO") + return False + + # We have tables but no alembic_version and no core tables + # These are likely test/manual tables that prevent migrations + log(f"Attempting to clean up {len(all_tables)} unexpected table(s): {all_tables}", "INFO") + log("These appear to be test/manual tables that prevent migrations from running.", "INFO") + + # Drop all unexpected tables + cleaned = False + for table in all_tables: + try: + log(f"Dropping unexpected table: {table}", "INFO") + db.session.execute(text(f'DROP TABLE IF EXISTS "{table}" CASCADE')) + db.session.commit() + log(f"✓ Dropped table: {table}", "SUCCESS") + cleaned = True + except Exception as e: + log(f"Failed to drop table {table}: {e}", "WARNING") + db.session.rollback() + + return cleaned + except Exception as e: + log(f"Database cleanup failed: {e}", "WARNING") + import traceback + traceback.print_exc() + return False + + +def run_migrations(): + """Apply Alembic migrations once (fast path).""" + log("Applying database migrations (Flask-Migrate)...", "INFO") + + # Prevent app from starting background jobs / DB-dependent init during migrations + os.environ["TT_BOOTSTRAP_MODE"] = "migrate" + os.environ.setdefault("FLASK_APP", "app") + os.environ.setdefault("FLASK_ENV", os.getenv("FLASK_ENV", "production") or "production") + os.chdir("/app") + + try: + from flask_migrate import upgrade + from app import create_app + + app = create_app() + # Log the DB URL we're about to use (mask password) + try: + raw = os.environ.get("DATABASE_URL", "") + masked = raw + if "://" in raw and "@" in raw: + import re as _re + + masked = _re.sub(r"//([^:]+):[^@]+@", r"//\\1:***@", raw) + log(f"DATABASE_URL (env): {masked}", "INFO") + except Exception: + pass + + with app.app_context(): + # Check for corrupted database state BEFORE migrations + is_corrupted, reason = detect_corrupted_database_state(app) + if is_corrupted: + log(f"⚠ Detected corrupted database state: {reason}", "WARNING") + log("Attempting automatic cleanup...", "INFO") + + if cleanup_corrupted_database_state(app): + log("✓ Database cleanup completed", "SUCCESS") + log("Retrying migrations after cleanup...", "INFO") + else: + log("Database cleanup was skipped or failed", "WARNING") + log("Migrations will still be attempted, but may fail.", "WARNING") + + # Sanity: show which DB we're connected to before migrating + try: + from app import db as _db + from sqlalchemy import text as _text + + cur_db = _db.session.execute(_text("select current_database()")).scalar() + table_count = _db.session.execute( + _text("select count(1) from information_schema.tables where table_schema='public'") + ).scalar() + log(f"Pre-migration DB: {cur_db} (public tables={table_count})", "INFO") + except Exception as e: + log(f"Pre-migration DB probe failed: {e}", "WARNING") + + # Use heads to handle branched histories safely + upgrade(revision="heads") + + # CRITICAL: Verify migrations actually created tables (detect transaction rollback issues) + try: + from app import db as _db + from sqlalchemy import text as _text + + cur_db = _db.session.execute(_text("select current_database()")).scalar() + table_count = _db.session.execute( + _text("select count(1) from information_schema.tables where table_schema='public'") + ).scalar() + log(f"Post-migration DB: {cur_db} (public tables={table_count})", "INFO") + + # Check if alembic_version table exists (migrations actually ran) + alembic_exists = _db.session.execute( + _text("SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema='public' AND table_name='alembic_version')") + ).scalar() + + if not alembic_exists: + log("✗ WARNING: alembic_version table missing after migrations!", "ERROR") + log("Migrations reported success but alembic_version table was not created.", "ERROR") + log("This indicates migrations did not actually run or were rolled back.", "ERROR") + log("The database may be in an inconsistent state.", "ERROR") + log("", "ERROR") + log("RECOVERY OPTIONS:", "ERROR") + log("1. Reset database: docker compose down -v && docker compose up -d", "ERROR") + log("2. Or set TT_SKIP_DB_CLEANUP=false and restart to try automatic cleanup", "ERROR") + return None + + # Check if core tables exist + core_tables_check = _db.session.execute( + _text(""" + SELECT COUNT(*) + FROM information_schema.tables + WHERE table_schema='public' + AND table_name IN ('users', 'projects', 'time_entries', 'settings', 'clients') + """) + ).scalar() + + if core_tables_check < 5: + log(f"✗ WARNING: Only {core_tables_check}/5 core tables exist after migrations!", "ERROR") + log("Migrations reported success but core tables are missing.", "ERROR") + log("This indicates migrations did not complete successfully.", "ERROR") + log("", "ERROR") + log("RECOVERY OPTIONS:", "ERROR") + log("1. Reset database: docker compose down -v && docker compose up -d", "ERROR") + log("2. Check migration logs for errors", "ERROR") + return None + + except Exception as e: + log(f"Post-migration verification failed: {e}", "ERROR") + import traceback + traceback.print_exc() + return None + + log("Migrations applied and verified", "SUCCESS") + return app + except Exception as e: + log(f"Migration failed: {e}", "ERROR") + traceback.print_exc() + return None + finally: + # Important: don't leak migrate bootstrap mode into gunicorn runtime + os.environ.pop("TT_BOOTSTRAP_MODE", None) + + +def verify_core_tables(app): + """Verify core application tables exist after migrations (fail-fast).""" + log("Verifying core database tables exist...", "INFO") + try: + from app import db + from sqlalchemy import text + + with app.app_context(): + # Core tables required for the app to function + core_tables = ['users', 'projects', 'time_entries', 'settings', 'clients'] + + # Check PostgreSQL + if os.getenv("DATABASE_URL", "").startswith("postgresql"): + # First, list ALL tables for debugging + try: + all_tables_query = text("SELECT table_name FROM information_schema.tables WHERE table_schema='public' ORDER BY table_name") + all_tables_result = db.session.execute(all_tables_query) + all_tables = [row[0] for row in all_tables_result] + log(f"All tables in database: {all_tables}", "INFO") + except Exception as e: + log(f"Could not list all tables: {e}", "WARNING") + + # Use IN clause with proper parameter binding for PostgreSQL + placeholders = ",".join([f":table_{i}" for i in range(len(core_tables))]) + query = text(f""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name IN ({placeholders}) + """) + params = {f"table_{i}": table for i, table in enumerate(core_tables)} + result = db.session.execute(query, params) + found_tables = [row[0] for row in result] + missing = set(core_tables) - set(found_tables) + + if missing: + log(f"✗ Core tables missing: {sorted(missing)}", "ERROR") + log(f"Found core tables: {sorted(found_tables)}", "ERROR") + log("Database migrations may have failed silently.", "ERROR") + log("Try: docker compose down -v && docker compose up -d", "ERROR") + return False + + # Also verify alembic_version exists (migrations were applied) + alembic_check = db.session.execute( + text("SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema='public' AND table_name='alembic_version')") + ).scalar() + if not alembic_check: + log("✗ alembic_version table missing - migrations may not have been applied", "ERROR") + return False + + log(f"✓ Core tables verified: {sorted(found_tables)}", "SUCCESS") + return True + + # SQLite check (simpler) + else: + # Build IN clause with placeholders for SQLite + placeholders = ",".join(["?" for _ in core_tables]) + query = text(f"SELECT name FROM sqlite_master WHERE type='table' AND name IN ({placeholders})") + result = db.session.execute(query, core_tables) + found_tables = [row[0] for row in result] + missing = set(core_tables) - set(found_tables) + + if missing: + log(f"✗ Core tables missing: {sorted(missing)}", "ERROR") + return False + + # Check alembic_version + alembic_check = db.session.execute( + text("SELECT name FROM sqlite_master WHERE type='table' AND name='alembic_version'") + ).fetchone() + if not alembic_check: + log("✗ alembic_version table missing - migrations may not have been applied", "ERROR") + return False + + log(f"✓ Core tables verified: {sorted(found_tables)}", "SUCCESS") + return True + + except Exception as e: + log(f"✗ Core table verification failed: {e}", "ERROR") + traceback.print_exc() + return False + + +def ensure_default_data(app): + """Ensure Settings row + admin users exist (idempotent, no create_all).""" + log("Ensuring default data exists...", "INFO") + try: + from app import db + from app.models import Settings, User + with app.app_context(): + # Settings + try: + Settings.get_settings() + except Exception: + # Fallback: create row if model supports it + if not Settings.query.first(): + db.session.add(Settings()) + db.session.commit() + + # Admin users + admin_usernames = [u.strip().lower() for u in os.getenv("ADMIN_USERNAMES", "admin").split(",") if u.strip()] + for username in admin_usernames[:5]: # safety cap + existing = User.query.filter_by(username=username).first() + if not existing: + u = User(username=username, role="admin") + try: + u.is_active = True + except Exception: + pass + db.session.add(u) + else: + changed = False + if getattr(existing, "role", None) != "admin": + existing.role = "admin" + changed = True + if hasattr(existing, "is_active") and not getattr(existing, "is_active", True): + existing.is_active = True + changed = True + if changed: + db.session.add(existing) + db.session.commit() + + log("Default data ensured", "SUCCESS") + return True + except Exception as e: + log(f"Default data initialization failed (continuing): {e}", "WARNING") return False def display_network_info(): @@ -161,30 +538,21 @@ def main(): log("Database is not available, exiting...", "ERROR") sys.exit(1) - # Run enhanced database initialization and migration - log("Running database initialization...", "INFO") - if not run_script('/app/docker/init-database-enhanced.py', 'Database initialization'): - log("Database initialization failed, exiting...", "ERROR") + # Migrations (single source of truth) + migration_app = run_migrations() + if not migration_app: + log("Migrations failed, exiting...", "ERROR") sys.exit(1) - - log("Database initialization completed", "SUCCESS") - - # Ensure default settings and admin user exist (idempotent) - # Note: Database initialization is already handled by the migration system above - # The flask init_db command is optional and may not be available in all environments - try: - result = subprocess.run( - ['flask', 'init_db'], - check=False, # Don't fail if command doesn't exist - capture_output=True, - text=True, - timeout=30 - ) - if result.returncode != 0 and "No such command" not in (result.stderr or ""): - log("flask init_db returned non-zero exit code (continuing)", "WARNING") - except (FileNotFoundError, subprocess.TimeoutExpired, Exception): - # All errors are non-fatal - database is already initialized - pass + + # Fail-fast: verify core tables exist (don't start app with broken DB) + if not verify_core_tables(migration_app): + log("Core database tables are missing. Startup aborted.", "ERROR") + log("If this is a fresh install, migrations may have failed.", "ERROR") + log("If this persists, check migration logs and consider resetting the database.", "ERROR") + sys.exit(1) + + # Seed minimal default rows (fast, idempotent) + ensure_default_data(migration_app) log("=" * 60, "INFO") log("Starting application server", "INFO")