diff --git a/Dockerfile b/Dockerfile index c2f5ccf..a01d2df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,18 +44,18 @@ RUN mkdir -p /app/app/static/uploads/logos /app/static/uploads/logos && \ chmod -R 755 /app/static/uploads # Copy the startup script and ensure it's executable -COPY docker/start-new.sh /app/start.sh +COPY docker/start-fixed.py /app/start.py # Make startup scripts executable -RUN chmod +x /app/start.sh /app/docker/init-database.py /app/docker/init-database-sql.py /app/docker/test-db.py /app/docker/test-routing.py +RUN chmod +x /app/start.py /app/docker/init-database.py /app/docker/init-database-sql.py /app/docker/test-db.py /app/docker/test-routing.py # Create non-root user RUN useradd -m -u 1000 timetracker && \ chown -R timetracker:timetracker /app /data /app/logs /app/app/static/uploads /app/static/uploads # Verify startup script exists and is accessible -RUN ls -la /app/start.sh && \ - head -1 /app/start.sh +RUN ls -la /app/start.py && \ + head -1 /app/start.py USER timetracker @@ -67,4 +67,4 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ CMD curl -f http://localhost:8080/_health || exit 1 # Run the application -CMD ["/app/start.sh"] +CMD ["python", "/app/start.py"] diff --git a/docker/init-database-sql.py b/docker/init-database-sql.py index 04b9074..e111468 100644 --- a/docker/init-database-sql.py +++ b/docker/init-database-sql.py @@ -112,6 +112,25 @@ def create_tables_sql(engine): created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); + -- Create tasks table + CREATE TABLE IF NOT EXISTS tasks ( + id SERIAL PRIMARY KEY, + project_id INTEGER REFERENCES projects(id) ON DELETE CASCADE NOT NULL, + name VARCHAR(200) NOT NULL, + description TEXT, + status VARCHAR(20) DEFAULT 'pending' NOT NULL, + priority VARCHAR(20) DEFAULT 'medium' NOT NULL, + assigned_to INTEGER REFERENCES users(id), + created_by INTEGER REFERENCES users(id) NOT NULL, + due_date DATE, + estimated_hours NUMERIC(5,2), + actual_hours NUMERIC(5,2), + started_at TIMESTAMP, + completed_at TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL + ); + -- Create settings table CREATE TABLE IF NOT EXISTS settings ( id SERIAL PRIMARY KEY, @@ -220,6 +239,11 @@ def create_triggers(engine): CREATE TRIGGER update_settings_updated_at BEFORE UPDATE ON settings FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); """)) + conn.execute(text(""" + DROP TRIGGER IF EXISTS update_tasks_updated_at ON tasks; + CREATE TRIGGER update_tasks_updated_at BEFORE UPDATE ON tasks FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); + """)) + conn.commit() print("✓ Triggers created successfully") @@ -272,7 +296,7 @@ def verify_tables(engine): try: inspector = inspect(engine) existing_tables = inspector.get_table_names() - required_tables = ['users', 'projects', 'time_entries', 'settings', 'invoices', 'invoice_items'] + required_tables = ['users', 'projects', 'time_entries', 'tasks', 'settings', 'invoices', 'invoice_items'] missing_tables = [table for table in required_tables if table not in existing_tables] diff --git a/docker/init-database.py b/docker/init-database.py index 03f9916..1a28edc 100644 --- a/docker/init-database.py +++ b/docker/init-database.py @@ -67,8 +67,8 @@ def check_database_initialization(engine): # Check if tasks table exists if 'tasks' not in existing_tables: - print("✗ tasks table missing") - return False + print("⚠ tasks table missing - will be created by SQL script") + # Don't return False here, let the SQL script handle it else: print("✓ tasks table exists") @@ -106,9 +106,8 @@ def ensure_correct_schema(engine): # Define required columns for each table required_columns = { 'time_entries': ['id', 'user_id', 'project_id', 'task_id', 'start_time', 'end_time', - 'duration_seconds', 'notes', 'tags', 'source', 'billable', 'created_at', 'updated_at'], - 'tasks': ['id', 'project_id', 'name', 'description', 'status', 'priority', 'assigned_to', - 'created_by', 'due_date', 'estimated_hours', 'actual_hours', 'created_at', 'updated_at'] + 'duration_seconds', 'notes', 'tags', 'source', 'billable', 'created_at', 'updated_at'] + # Note: tasks table is created by SQL script, not checked here } needs_recreation = False diff --git a/docker/start-fixed.py b/docker/start-fixed.py new file mode 100644 index 0000000..d31afdd --- /dev/null +++ b/docker/start-fixed.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +""" +Improved Python startup script for TimeTracker +This script ensures proper database initialization order and handles errors gracefully +""" + +import os +import sys +import time +import subprocess +import traceback + +def wait_for_database(): + """Wait for database to be ready""" + print("Waiting for database to be ready...") + time.sleep(5) # Simple wait for now + +def run_script(script_path, description): + """Run a Python script with proper error handling""" + print(f"Running {description}...") + try: + result = subprocess.run( + [sys.executable, script_path], + check=True, + capture_output=True, + text=True + ) + print(f"✓ {description} completed successfully") + if result.stdout: + print(f"Output: {result.stdout}") + return True + except subprocess.CalledProcessError as e: + print(f"✗ {description} failed with exit code {e.returncode}") + if e.stdout: + print(f"stdout: {e.stdout}") + if e.stderr: + print(f"stderr: {e.stderr}") + return False + except Exception as e: + print(f"✗ Unexpected error running {description}: {e}") + traceback.print_exc() + return False + +def main(): + print("=== Starting TimeTracker (Improved Python Mode) ===") + + # Set environment + os.environ['FLASK_APP'] = 'app' + os.chdir('/app') + + # Wait for database + wait_for_database() + + # Step 1: Run SQL database initialization first (creates basic tables including tasks) + if not run_script('/app/docker/init-database-sql.py', 'SQL database initialization'): + print("SQL database initialization failed, exiting...") + sys.exit(1) + + # Step 2: Run main database initialization (handles Flask-specific setup) + if not run_script('/app/docker/init-database.py', 'main database initialization'): + print("Main database initialization failed, exiting...") + sys.exit(1) + + print("✓ All database initialization completed successfully") + + print("Starting application...") + # Start gunicorn + os.execv('/usr/local/bin/gunicorn', [ + 'gunicorn', + '--bind', '0.0.0.0:8080', + '--worker-class', 'eventlet', + '--workers', '1', + '--timeout', '120', + 'app:create_app()' + ]) + +if __name__ == '__main__': + main() diff --git a/docker/start-fixed.sh b/docker/start-fixed.sh index cb5a3d0..5dd4f92 100644 --- a/docker/start-fixed.sh +++ b/docker/start-fixed.sh @@ -3,7 +3,7 @@ set -e cd /app export FLASK_APP=app -echo "=== Starting TimeTracker ===" +echo "=== Starting TimeTracker (Fixed Shell Mode) ===" echo "Waiting for database to be ready..." # Wait for Postgres to be ready @@ -33,152 +33,33 @@ else: print("No PostgreSQL database configured, skipping connection check") PY -echo "=== FIXING DATABASE SCHEMA ===" +echo "=== RUNNING DATABASE INITIALIZATION ===" -# Step 1: Create tasks table if it doesn't exist -echo "Step 1: Ensuring tasks table exists..." -python - <<"PY" -import os -import sys -from sqlalchemy import create_engine, text, inspect - -url = os.getenv("DATABASE_URL", "") -if url.startswith("postgresql"): - try: - engine = create_engine(url, pool_pre_ping=True) - inspector = inspect(engine) - - if 'tasks' not in inspector.get_table_names(): - print("Creating tasks table...") - create_tasks_sql = """ - CREATE TABLE tasks ( - id SERIAL PRIMARY KEY, - project_id INTEGER REFERENCES projects(id) ON DELETE CASCADE NOT NULL, - name VARCHAR(200) NOT NULL, - description TEXT, - status VARCHAR(20) DEFAULT 'pending' NOT NULL, - priority VARCHAR(20) DEFAULT 'medium' NOT NULL, - assigned_to INTEGER REFERENCES users(id), - created_by INTEGER REFERENCES users(id) NOT NULL, - due_date DATE, - estimated_hours NUMERIC(5,2), - actual_hours NUMERIC(5,2), - started_at TIMESTAMP, - completed_at TIMESTAMP, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL - ); - """ - with engine.connect() as conn: - conn.execute(text(create_tasks_sql)) - conn.commit() - print("✓ Tasks table created successfully") - else: - print("✓ Tasks table already exists") - - except Exception as e: - print(f"Error creating tasks table: {e}") - sys.exit(1) -else: - print("No PostgreSQL database configured") - sys.exit(0) -PY - -# Step 2: Add task_id column to time_entries if it doesn't exist -echo "Step 2: Ensuring task_id column exists in time_entries..." -python - <<"PY" -import os -import sys -from sqlalchemy import create_engine, text, inspect - -url = os.getenv("DATABASE_URL", "") -if url.startswith("postgresql"): - try: - engine = create_engine(url, pool_pre_ping=True) - inspector = inspect(engine) - - if 'time_entries' in inspector.get_table_names(): - columns = inspector.get_columns("time_entries") - column_names = [col['name'] for col in columns] - print(f"Current columns in time_entries: {column_names}") - - if 'task_id' not in column_names: - print("Adding task_id column...") - with engine.connect() as conn: - conn.execute(text("ALTER TABLE time_entries ADD COLUMN task_id INTEGER;")) - conn.commit() - print("✓ task_id column added successfully") - else: - print("✓ task_id column already exists") - else: - print("⚠ Warning: time_entries table does not exist") - - except Exception as e: - print(f"Error adding task_id column: {e}") - sys.exit(1) -else: - print("No PostgreSQL database configured") - sys.exit(0) -PY - -# Step 2.5: Add missing columns to tasks table if it exists -echo "Step 2.5: Ensuring tasks table has all required columns..." -python - <<"PY" -import os -import sys -from sqlalchemy import create_engine, text, inspect - -url = os.getenv("DATABASE_URL", "") -if url.startswith("postgresql"): - try: - engine = create_engine(url, pool_pre_ping=True) - inspector = inspect(engine) - - if 'tasks' in inspector.get_table_names(): - columns = inspector.get_columns("tasks") - column_names = [col['name'] for col in columns] - print(f"Current columns in tasks: {column_names}") - - # Check for missing columns - missing_columns = [] - required_columns = ['started_at', 'completed_at'] - - for col in required_columns: - if col not in column_names: - missing_columns.append(col) - - if missing_columns: - print(f"Adding missing columns to tasks table: {missing_columns}") - with engine.connect() as conn: - for col in missing_columns: - if col == 'started_at': - conn.execute(text("ALTER TABLE tasks ADD COLUMN started_at TIMESTAMP;")) - elif col == 'completed_at': - conn.execute(text("ALTER TABLE tasks ADD COLUMN completed_at TIMESTAMP;")) - conn.commit() - print("✓ Missing columns added to tasks table successfully") - else: - print("✓ tasks table has all required columns") - else: - print("⚠ Warning: tasks table does not exist") - - except Exception as e: - print(f"Error adding missing columns to tasks: {e}") - sys.exit(1) -else: - print("No PostgreSQL database configured") - sys.exit(0) -PY - -# Step 3: Run the main database initialization -echo "Step 3: Running main database initialization..." -python /app/docker/init-database.py -if [ $? -ne 0 ]; then - echo "Database initialization failed. Exiting." +# Step 1: Run SQL database initialization first (creates basic tables including tasks) +echo "Step 1: Running SQL database initialization..." +if python /app/docker/init-database-sql.py; then + echo "✓ SQL database initialization completed" +else + echo "✗ SQL database initialization failed" exit 1 fi -echo "=== DATABASE SCHEMA FIXED SUCCESSFULLY ===" +# Step 2: Run main database initialization (handles Flask-specific setup) +echo "Step 2: Running main database initialization..." +if python /app/docker/init-database.py; then + echo "✓ Main database initialization completed" +else + echo "✗ Main database initialization failed" + exit 1 +fi + +echo "✓ All database initialization completed successfully" echo "Starting application..." -exec gunicorn --bind 0.0.0.0:8080 --worker-class eventlet --workers 1 --timeout 120 "app:create_app()" +# Start gunicorn +exec gunicorn \ + --bind 0.0.0.0:8080 \ + --worker-class eventlet \ + --workers 1 \ + --timeout 120 \ + app:create_app() diff --git a/docker/start.py b/docker/start.py index 15b331c..46cd325 100644 --- a/docker/start.py +++ b/docker/start.py @@ -19,15 +19,7 @@ def main(): print("Waiting for database to be ready...") time.sleep(5) # Simple wait - print("Running database initialization...") - try: - subprocess.run([sys.executable, '/app/docker/init-database.py'], check=True) - print("Database initialization completed") - except subprocess.CalledProcessError as e: - print(f"Database initialization failed: {e}") - sys.exit(1) - - print("Running SQL database initialization (for invoice tables)...") + print("Running SQL database initialization (for basic tables)...") try: subprocess.run([sys.executable, '/app/docker/init-database-sql.py'], check=True) print("SQL database initialization completed") @@ -35,6 +27,14 @@ def main(): print(f"SQL database initialization failed: {e}") sys.exit(1) + print("Running main database initialization...") + try: + subprocess.run([sys.executable, '/app/docker/init-database.py'], check=True) + print("Database initialization completed") + except subprocess.CalledProcessError as e: + print(f"Database initialization failed: {e}") + sys.exit(1) + print("Starting application...") # Start gunicorn os.execv('/usr/local/bin/gunicorn', [ diff --git a/docker/test-database-complete.py b/docker/test-database-complete.py new file mode 100644 index 0000000..8bcea19 --- /dev/null +++ b/docker/test-database-complete.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +""" +Comprehensive database testing script for TimeTracker +This script verifies that all required tables exist and have the correct schema +""" + +import os +import sys +import time +from sqlalchemy import create_engine, text, inspect + +def wait_for_database(url, max_attempts=30, delay=2): + """Wait for database to be ready""" + print(f"Waiting for database to be ready...") + + for attempt in range(max_attempts): + try: + engine = create_engine(url, pool_pre_ping=True) + with engine.connect() as conn: + conn.execute(text("SELECT 1")) + print("Database connection established successfully") + return engine + except Exception as e: + print(f"Waiting for database... (attempt {attempt+1}/{max_attempts}): {e}") + if attempt < max_attempts - 1: + time.sleep(delay) + else: + print("Database not ready after waiting, exiting...") + sys.exit(1) + + return None + +def verify_table_exists(engine, table_name, description=""): + """Verify that a specific table exists""" + try: + inspector = inspect(engine) + existing_tables = inspector.get_table_names() + + if table_name in existing_tables: + print(f"✓ {table_name} table exists {description}") + return True + else: + print(f"✗ {table_name} table missing {description}") + return False + except Exception as e: + print(f"✗ Error checking {table_name} table: {e}") + return False + +def verify_table_schema(engine, table_name, required_columns): + """Verify that a table has the required columns""" + try: + inspector = inspect(engine) + if table_name not in inspector.get_table_names(): + print(f"✗ Cannot check schema for {table_name} - table doesn't exist") + return False + + existing_columns = [col['name'] for col in inspector.get_columns(table_name)] + missing_columns = [col for col in required_columns if col not in existing_columns] + + if missing_columns: + print(f"✗ {table_name} table missing columns: {missing_columns}") + print(f" Available columns: {existing_columns}") + return False + else: + print(f"✓ {table_name} table has correct schema") + return True + except Exception as e: + print(f"✗ Error checking schema for {table_name}: {e}") + return False + +def main(): + """Main function""" + url = os.getenv("DATABASE_URL", "") + + if not url.startswith("postgresql"): + print("No PostgreSQL database configured, skipping verification") + return + + print(f"Database URL: {url}") + + # Wait for database to be ready + engine = wait_for_database(url) + + print("\n=== VERIFYING DATABASE SCHEMA ===") + + # Define all required tables and their required columns + required_tables = { + 'users': ['id', 'username', 'role', 'created_at', 'last_login', 'is_active', 'updated_at'], + 'projects': ['id', 'name', 'client', 'description', 'billable', 'hourly_rate', 'billing_ref', 'status', 'created_at', 'updated_at'], + 'time_entries': ['id', 'user_id', 'project_id', 'task_id', 'start_time', 'end_time', 'duration_seconds', 'notes', 'tags', 'source', 'billable', 'created_at', 'updated_at'], + 'tasks': ['id', 'project_id', 'name', 'description', 'status', 'priority', 'assigned_to', 'created_by', 'due_date', 'estimated_hours', 'actual_hours', 'started_at', 'completed_at', 'created_at', 'updated_at'], + 'settings': ['id', 'timezone', 'currency', 'rounding_minutes', 'single_active_timer', 'allow_self_register', 'idle_timeout_minutes', 'backup_retention_days', 'backup_time', 'export_delimiter', 'company_name', 'company_address', 'company_email', 'company_phone', 'company_website', 'company_logo_filename', 'company_tax_id', 'company_bank_info', 'invoice_prefix', 'invoice_start_number', 'invoice_terms', 'invoice_notes', 'created_at', 'updated_at'], + 'invoices': ['id', 'invoice_number', 'project_id', 'client_name', 'client_email', 'client_address', 'issue_date', 'due_date', 'status', 'subtotal', 'tax_rate', 'tax_amount', 'total_amount', 'notes', 'terms', 'created_by', 'created_at', 'updated_at'], + 'invoice_items': ['id', 'invoice_id', 'description', 'quantity', 'unit_price', 'total_amount', 'time_entry_ids', 'created_at'] + } + + all_tables_exist = True + all_schemas_correct = True + + # Check if all tables exist + print("\n--- Checking Table Existence ---") + for table_name in required_tables.keys(): + if not verify_table_exists(engine, table_name): + all_tables_exist = False + + # Check schema for existing tables + if all_tables_exist: + print("\n--- Checking Table Schemas ---") + for table_name, required_columns in required_tables.items(): + if not verify_table_schema(engine, table_name, required_columns): + all_schemas_correct = False + + # Summary + print("\n=== VERIFICATION SUMMARY ===") + if all_tables_exist and all_schemas_correct: + print("✓ All tables exist and have correct schema") + print("✓ Database is properly initialized") + sys.exit(0) + else: + if not all_tables_exist: + print("✗ Some tables are missing") + if not all_schemas_correct: + print("✗ Some tables have incorrect schema") + print("✗ Database verification failed") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/docs/DATABASE_STARTUP_FIX_README.md b/docs/DATABASE_STARTUP_FIX_README.md new file mode 100644 index 0000000..a75ca4e --- /dev/null +++ b/docs/DATABASE_STARTUP_FIX_README.md @@ -0,0 +1,161 @@ +# Database Startup Fix + +## Problem Description + +The TimeTracker application was experiencing startup failures due to incorrect database initialization order. The main issue was: + +1. **Dependency Order Problem**: The `tasks` table has a foreign key reference to `projects(id)`, but the startup scripts were trying to create the `tasks` table before the `projects` table existed. + +2. **Script Execution Order**: The startup sequence was running `init-database.py` first (which tried to create tables using Flask models), then `init-database-sql.py` (which created basic tables), but the `tasks` table was missing from the SQL script. + +3. **Error Message**: + ``` + Error creating tasks table: (psycopg2.errors.UndefinedTable) relation "projects" does not exist + ``` + +## Root Cause + +The `tasks` table creation was embedded in the shell script (`start-new.sh`) but was failing because: +- It referenced `projects(id)` before the `projects` table was created +- The table creation logic was scattered across multiple scripts +- No proper dependency management between table creation steps + +## Solution Implemented + +### 1. Fixed Database Initialization Order + +**Before**: +- `init-database.py` runs first → fails to create tasks table +- `init-database-sql.py` runs second → creates basic tables but missing tasks + +**After**: +- `init-database-sql.py` runs first → creates all basic tables including tasks +- `init-database.py` runs second → handles Flask-specific setup + +### 2. Added Tasks Table to SQL Script + +Updated `docker/init-database-sql.py` to include the `tasks` table creation: + +```sql +-- Create tasks table +CREATE TABLE IF NOT EXISTS tasks ( + id SERIAL PRIMARY KEY, + project_id INTEGER REFERENCES projects(id) ON DELETE CASCADE NOT NULL, + name VARCHAR(200) NOT NULL, + description TEXT, + status VARCHAR(20) DEFAULT 'pending' NOT NULL, + priority VARCHAR(20) DEFAULT 'medium' NOT NULL, + assigned_to INTEGER REFERENCES users(id), + created_by INTEGER REFERENCES users(id) NOT NULL, + due_date DATE, + estimated_hours NUMERIC(5,2), + actual_hours NUMERIC(5,2), + started_at TIMESTAMP, + completed_at TIMESTAMP, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL +); +``` + +### 3. Updated Table Verification + +- Added `tasks` to the required tables list in `init-database-sql.py` +- Added trigger for automatic `updated_at` column updates +- Updated main init script to not fail if tasks table is missing initially + +### 4. Improved Startup Scripts + +Created multiple startup options: + +- **`docker/start-fixed.py`**: Python-based startup with proper error handling +- **`docker/start-fixed.sh`**: Shell script version with correct execution order +- **`docker/test-database-complete.py`**: Comprehensive database verification script + +### 5. Updated Dockerfile + +- Changed from `start-new.sh` to `start-fixed.py` +- Updated CMD to use Python script +- Maintained all existing functionality + +## Files Modified + +1. **`docker/init-database-sql.py`** + - Added tasks table creation + - Added tasks to required tables list + - Added trigger for tasks table + +2. **`docker/init-database.py`** + - Modified to not fail if tasks table missing initially + - Updated schema checking to skip tasks table validation + +3. **`docker/start.py`** + - Swapped execution order of initialization scripts + +4. **`Dockerfile`** + - Updated to use improved startup script + +5. **New Files Created**: + - `docker/start-fixed.py` - Improved Python startup script + - `docker/start-fixed.sh` - Fixed shell startup script + - `docker/test-database-complete.py` - Database verification script + +## How to Use + +### Option 1: Use Python Startup Script (Recommended) +```bash +# In Dockerfile or docker-compose +CMD ["python", "/app/start.py"] +``` + +### Option 2: Use Shell Startup Script +```bash +# In Dockerfile or docker-compose +CMD ["/app/start-fixed.sh"] +``` + +### Option 3: Test Database Setup +```bash +# Run verification script +python docker/test-database-complete.py +``` + +## Verification + +After the fix, the startup sequence should show: + +``` +=== Starting TimeTracker === +Waiting for database to be ready... +Database connection established successfully +=== RUNNING DATABASE INITIALIZATION === +Step 1: Running SQL database initialization... +✓ SQL database initialization completed +Step 2: Running main database initialization... +✓ Main database initialization completed +✓ All database initialization completed successfully +Starting application... +``` + +## Benefits + +1. **Reliable Startup**: Tables are created in the correct dependency order +2. **Better Error Handling**: Clear error messages and proper exit codes +3. **Maintainable Code**: Centralized table creation logic +4. **Flexible Options**: Multiple startup script options for different needs +5. **Comprehensive Testing**: Database verification script for troubleshooting + +## Troubleshooting + +If you still encounter issues: + +1. **Check Database Logs**: Look for specific error messages +2. **Run Verification Script**: Use `test-database-complete.py` to check table status +3. **Verify Environment**: Ensure `DATABASE_URL` is properly set +4. **Check Permissions**: Ensure database user has CREATE TABLE permissions + +## Future Improvements + +1. **Migration System**: Implement proper database migrations instead of table recreation +2. **Dependency Graph**: Create explicit dependency management for table creation +3. **Rollback Support**: Add ability to rollback failed initialization +4. **Health Checks**: Implement database health checks during startup