TimeTracker/app/utils/pdf_generator_reportlab.py

"""
ReportLab PDF Template Renderer

Converts ReportLab template JSON (generated by visual editor) into PDF documents
using ReportLab's programmatic PDF generation.
"""

import io
import json
import os
import tempfile
from datetime import datetime
from types import SimpleNamespace
from typing import Any, Dict, List, Optional, Union

from reportlab.lib.pagesizes import A3, A4, A5, LEGAL, LETTER, TABLOID
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
from reportlab.lib.units import cm, inch, mm

# ReportLab doesn't export 'pt' from units
# In ReportLab, the default unit is already points (1 point = 1 unit = 1/72 inch)
pt = 1  # 1 point = 1 unit in ReportLab
from flask import current_app
from flask_babel import gettext as _
from reportlab.lib import colors
from reportlab.lib.enums import TA_CENTER, TA_JUSTIFY, TA_LEFT, TA_RIGHT
from reportlab.lib.utils import ImageReader
from reportlab.pdfgen import canvas
from reportlab.platypus import (
    Flowable,
    Image,
    KeepTogether,
    PageBreak,
    Paragraph,
    SimpleDocTemplate,
    Spacer,
    Table,
    TableStyle,
)

from app.models import Settings
from app.utils.pdf_template_schema import (
    PAGE_SIZE_DIMENSIONS_MM,
    ElementType,
    TextAlign,
    get_page_dimensions_points,
    validate_template_json,
)


class AbsolutePositionedFlowable(Flowable):
    """Flowable that stores element info for canvas drawing (elements drawn in page callbacks)"""

    def __init__(self, element_data):
        Flowable.__init__(self)
        self.element_data = element_data  # Store element data for drawing in page callback

    def draw(self):
        """This won't be called - elements are drawn in page callbacks"""
        pass

    def wrap(self, availWidth, availHeight):
        """Return zero size so it doesn't affect flow"""
        return (0, 0)


def _normalize_color(color: Any) -> Optional[str]:
    """
    Convert color to hex format for ReportLab.
    Handles named colors, hex colors, and returns default if invalid.

    Args:
        color: Color value (string like 'white', 'black', '#ffffff', etc.)

    Returns:
        Hex color string (e.g., '#ffffff')
    """
    if not color or color == "transparent":
        return None

    # If already a hex color, return as-is
    if isinstance(color, str) and color.startswith("#"):
        return color

    # Map common named colors to hex
    color_map = {
        "white": "#ffffff",
        "black": "#000000",
        "red": "#ff0000",
        "green": "#00ff00",
        "blue": "#0000ff",
        "yellow": "#ffff00",
        "cyan": "#00ffff",
        "magenta": "#ff00ff",
        "gray": "#808080",
        "grey": "#808080",
        "orange": "#ffa500",
        "purple": "#800080",
        "pink": "#ffc0cb",
        "brown": "#a52a2a",
        "silver": "#c0c0c0",
        "gold": "#ffd700",
    }

    # Convert to lowercase for case-insensitive matching
    color_lower = str(color).lower().strip()

    if color_lower in color_map:
        return color_map[color_lower]

    # Try to use ReportLab's built-in colors
    try:
        if hasattr(colors, color_lower.capitalize()):
            color_obj = getattr(colors, color_lower.capitalize())
            # Convert ReportLab color to hex
            if hasattr(color_obj, "hexval"):
                return color_obj.hexval()
            # Some colors might be tuples
            if isinstance(color_obj, tuple) and len(color_obj) == 3:
                r, g, b = color_obj
                return f"#{int(r*255):02x}{int(g*255):02x}{int(b*255):02x}"
    except Exception:
        pass

    # Default to black if we can't convert
    return "#000000"


class ReportLabTemplateRenderer:
    """Render ReportLab template JSON into PDF"""

    def __init__(self, template_json: Dict[str, Any], data_context: Dict[str, Any], page_size: str = "A4"):
        """
        Initialize the renderer.

        Args:
            template_json: Template definition dictionary
            data_context: Data to fill template (invoice/quote, settings, etc.)
            page_size: Override page size from template
        """
        self.template = template_json
        self.context = data_context
        template_page_size = template_json.get("page", {}).get("size", "A4")
        self.page_size = page_size or template_page_size

        # Log initialization
        try:
            from flask import current_app

            element_count = len(template_json.get("elements", []))
            current_app.logger.info(
                f"[PDF_EXPORT] ReportLab renderer initialized - PageSize: '{self.page_size}', Template PageSize: '{template_page_size}', Elements: {element_count}"
            )
        except Exception:
            pass  # Logging not available or not in request context

        # Initialize styles
        self.styles = getSampleStyleSheet()
        self._setup_custom_styles()

        # Page size mapping for ReportLab
        self.PAGE_SIZE_MAP = {
            "A4": A4,
            "A5": A5,
            "A3": A3,
            "Letter": LETTER,
            "Legal": LEGAL,
            "Tabloid": TABLOID,
        }

    def _setup_custom_styles(self):
        """Setup custom paragraph styles"""
        # Add default styles if not already present
        # Use unique names to avoid conflicts with default stylesheet

        # Try to add custom styles, catching any errors if they already exist
        # ReportLab will raise an error if a style with the same name already exists
        try:
            # Check if style exists by trying to access it
            _ = self.styles["CustomHeading1"]
            # If we get here, style already exists, skip adding
        except (KeyError, AttributeError):
            # Style doesn't exist, try to add it
            try:
                self.styles.add(
                    ParagraphStyle(
                        name="CustomHeading1",
                        parent=self.styles["Heading1"],
                        fontSize=24,
                        spaceAfter=12,
                        textColor=colors.HexColor("#007bff"),
                    )
                )
            except (ValueError, KeyError, AttributeError, Exception):
                # Failed to add style - might already exist or other error
                # Just continue without this custom style
                pass

        try:
            # Check if style exists by trying to access it
            _ = self.styles["NormalText"]
            # If we get here, style already exists, skip adding
        except (KeyError, AttributeError):
            # Style doesn't exist, try to add it
            try:
                self.styles.add(
                    ParagraphStyle(
                        name="NormalText",
                        parent=self.styles["Normal"],
                        fontSize=10,
                        spaceAfter=6,
                    )
                )
            except (ValueError, KeyError, AttributeError, Exception):
                # Failed to add style - might already exist or other error
                # Just continue without this custom style
                pass

    def render_to_bytes(self) -> bytes:
        """
        Render template to PDF bytes.

        Returns:
            PDF content as bytes
        """
        from flask import current_app

        try:
            current_app.logger.info(f"[PDF_EXPORT] ReportLab render_to_bytes started - PageSize: '{self.page_size}'")
        except Exception:
            pass  # Logging not available or not in request context

        # Create temporary file
        with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as tmp_file:
            tmp_path = tmp_file.name

        try:
            # Get page size and dimensions
            page_size_obj = self.PAGE_SIZE_MAP.get(self.page_size, A4)
            page_dimensions = get_page_dimensions_points(self.page_size)

            try:
                current_app.logger.info(
                    f"[PDF_EXPORT] ReportLab page size configured - PageSize: '{self.page_size}', Dimensions: {page_dimensions['width']}x{page_dimensions['height']}pt"
                )
            except Exception:
                pass

            # Get margins (default to 20mm if not specified)
            page_config = self.template.get("page", {})
            margins = page_config.get("margin", {"top": 20, "right": 20, "bottom": 20, "left": 20})

            try:
                current_app.logger.info(
                    f"[PDF_EXPORT] ReportLab margins configured - PageSize: '{self.page_size}', Margins: top={margins.get('top', 20)}mm, right={margins.get('right', 20)}mm, bottom={margins.get('bottom', 20)}mm, left={margins.get('left', 20)}mm"
                )
            except Exception:
                pass

            # Create document
            doc = SimpleDocTemplate(
                tmp_path,
                pagesize=page_size_obj,
                rightMargin=margins.get("right", 20) * mm,
                leftMargin=margins.get("left", 20) * mm,
                topMargin=margins.get("top", 20) * mm,
                bottomMargin=margins.get("bottom", 20) * mm,
            )

            # Build story (flowables)
            story = self._build_story()
            story_length = len(story)

            try:
                current_app.logger.info(
                    f"[PDF_EXPORT] ReportLab story built - PageSize: '{self.page_size}', Story length: {story_length} flowables"
                )
            except Exception:
                pass

            # Store reference to renderer in doc for access in callbacks
            self._doc = doc
            # Store page dimensions for boundary checking
            self.page_width = page_dimensions["width"]
            self.page_height = page_dimensions["height"]

            # Build PDF with custom page callbacks for absolute positioning
            try:
                current_app.logger.info(
                    f"[PDF_EXPORT] ReportLab building PDF - PageSize: '{self.page_size}', Starting doc.build()"
                )
            except Exception:
                pass

            doc.build(story, onFirstPage=self._on_page, onLaterPages=self._on_page)

            # Read PDF bytes
            with open(tmp_path, "rb") as f:
                pdf_bytes = f.read()

            pdf_size_bytes = len(pdf_bytes)
            try:
                current_app.logger.info(
                    f"[PDF_EXPORT] ReportLab PDF build completed successfully - PageSize: '{self.page_size}', PDFSize: {pdf_size_bytes} bytes"
                )
            except Exception:
                pass

            return pdf_bytes

        except Exception as e:
            try:
                current_app.logger.error(
                    f"[PDF_EXPORT] ReportLab render_to_bytes failed - PageSize: '{self.page_size}', Error: {str(e)}",
                    exc_info=True,
                )
            except Exception:
                pass
            raise

        finally:
            # Clean up temporary file
            if os.path.exists(tmp_path):
                try:
                    os.unlink(tmp_path)
                except Exception:
                    pass

    def _build_story(self) -> List[Flowable]:
        """Build the PDF story (list of flowables) from template elements"""
        from flask import current_app

        # Store elements for drawing in page callbacks (absolute positioning)
        self.absolute_elements = []
        story = []

        elements = self.template.get("elements", [])
        element_count = len(elements)

        try:
            current_app.logger.info(
                f"[PDF_EXPORT] ReportLab building story - PageSize: '{self.page_size}', Elements: {element_count}"
            )
        except Exception:
            pass

        # Sort elements by y position (top to bottom) for proper rendering order
        sorted_elements = sorted(elements, key=lambda e: e.get("y", 0))

        element_types: dict = {}
        for element in sorted_elements:
            if element.get("hidden"):
                continue
            element_type = element.get("type")
            element_types[element_type] = element_types.get(element_type, 0) + 1

            # Process element (existing code continues here)

            if element_type == ElementType.SPACER:
                height = element.get("height", 20)
                story.append(Spacer(1, height * pt))
            elif element_type == ElementType.TABLE:
                # Tables are rendered as flowables (they flow naturally)
                # Add spacer to position table vertically based on y coordinate
                table_y = element.get("y", 0)
                page_config = self.template.get("page", {})
                margins = page_config.get("margin", {"top": 20, "right": 20, "bottom": 20, "left": 20})
                margin_top = margins.get("top", 20) * mm
                margin_left = margins.get("left", 20) * mm
                if table_y > 0:
                    # Tables are flowables, so they start from the top margin
                    # The y position in the template is from top of page (0,0 = top-left)
                    # We need to position the table relative to the top margin
                    # table_y is absolute from top of page, so we need to account for the margin
                    # The spacer should position the table at the correct y position
                    spacer_height = max(0, table_y - margin_top)
                    if spacer_height > 0:
                        story.append(Spacer(1, spacer_height))

                table_flowable = self._render_table(element)
                if table_flowable:
                    col_widths = self._scaled_table_col_widths(element)
                    total_table_w = sum(col_widths) if col_widths else 0.0
                    table_x = element.get("x") or 0
                    try:
                        table_x_f = float(table_x)
                    except (TypeError, ValueError):
                        table_x_f = 0.0
                    try:
                        margin_left_f = float(margin_left)
                    except (TypeError, ValueError):
                        margin_left_f = 0.0
                    left_offset = max(0.0, table_x_f - margin_left_f)
                    if left_offset > 0.01 and total_table_w > 0.01:
                        spacer_para = Paragraph("", self._get_style({"style": {}}, "NormalText"))
                        outer = Table([[spacer_para, table_flowable]], colWidths=[left_offset, total_table_w])
                        outer.setStyle(
                            TableStyle(
                                [
                                    ("LEFTPADDING", (0, 0), (-1, -1), 0),
                                    ("RIGHTPADDING", (0, 0), (-1, -1), 0),
                                    ("TOPPADDING", (0, 0), (-1, -1), 0),
                                    ("BOTTOMPADDING", (0, 0), (-1, -1), 0),
                                    ("VALIGN", (0, 0), (-1, -1), "TOP"),
                                ]
                            )
                        )
                        outer.hAlign = "LEFT"
                        story.append(outer)
                    else:
                        story.append(table_flowable)
            else:
                # For absolute positioning, store element data to draw in page callback
                element_data = {"element": element, "renderer": self}
                positioned = AbsolutePositionedFlowable(element_data)
                self.absolute_elements.append(element_data)
                story.append(positioned)

        try:
            current_app.logger.info(
                f"[PDF_EXPORT] ReportLab story built - PageSize: '{self.page_size}', Flowables: {len(story)}, Absolute elements: {len(self.absolute_elements)}, Element types: {element_types}"
            )
        except Exception:
            pass

        return story

    def _render_element(self, element: Dict[str, Any]) -> Optional[Flowable]:
        """Render a single element to a ReportLab flowable"""
        element_type = element.get("type")

        if element_type == ElementType.TEXT:
            return self._render_text(element)
        elif element_type == ElementType.IMAGE:
            return self._render_image(element)
        elif element_type == ElementType.RECTANGLE:
            return self._render_rectangle(element)
        elif element_type == ElementType.CIRCLE:
            return self._render_circle(element)
        elif element_type == ElementType.LINE:
            return self._render_line(element)
        elif element_type == ElementType.TABLE:
            return self._render_table(element)

        return None

    def _render_text(self, element: Dict[str, Any]) -> Paragraph:
        """Render a text element"""
        text = element.get("text", "")

        # Process template variables (Jinja2-style)
        text = self._process_template_variables(text)

        # Convert newlines to <br/> tags for Paragraph (which expects HTML-like markup)
        # Handle different line ending formats (\n, \r\n, \r)
        text = text.replace("\r\n", "<br/>").replace("\r", "<br/>").replace("\n", "<br/>")

        # Get style
        style_name = element.get("style_name", "NormalText")
        style = self._get_style(element, style_name)

        # Create paragraph
        return Paragraph(text, style)

    def _render_image(self, element: Dict[str, Any]) -> Optional[Image]:
        """Render an image element"""
        source = element.get("source", "")

        # Process template variables for source
        source = self._process_template_variables(source)

        # Issue #432: Explicit skip for decorative images with empty source (never add flowable or draw)
        if element.get("decorative", False) and (not source or not source.strip()):
            if current_app:
                current_app.logger.warning("Skipping decorative image flowable with empty source")
            return None
        if not source or not source.strip():
            return None

        # Handle base64 data URI with validated decode (Issue #432)
        if source.startswith("data:image"):
            import base64

            parts = source.split(",", 1)
            if len(parts) < 2 or not (parts[1] and parts[1].strip()):
                if current_app:
                    current_app.logger.warning("Skipping image: data URI has no base64 payload")
                return None
            try:
                img_data = base64.b64decode(parts[1])
                img_reader = ImageReader(io.BytesIO(img_data))
                width = element.get("width", 100)
                height = element.get("height", 100)
                return Image(img_reader, width=width, height=height)
            except Exception as e:
                if current_app:
                    current_app.logger.error(f"Error decoding base64 image: {e}")
                else:
                    print(f"Error decoding base64 image: {e}")
                return None

        # Handle template image URLs (convert to file path or base64)
        # Issue #537: Also handle full URLs; use robust path resolution
        if "/uploads/template_images/" in source:
            try:
                from app.utils.template_filters import get_image_base64

                filename = self._extract_template_image_filename(source)
                if not filename:
                    if current_app:
                        current_app.logger.warning(f"Could not extract filename from template image URL: {source}")
                    return None
                relative_path = f"app/static/uploads/template_images/{filename}"
                base64_data = get_image_base64(relative_path)
                if base64_data:
                    import base64 as b64

                    header, data = base64_data.split(",", 1)
                    img_data = b64.b64decode(data)
                    img_reader = ImageReader(io.BytesIO(img_data))
                    width = element.get("width", 100)
                    height = element.get("height", 100)
                    return Image(img_reader, width=width, height=height)
                # Fallback: try direct file path (Issue #537: robust path resolution)
                file_path = self._resolve_template_image_path(filename)
                if file_path and os.path.exists(file_path):
                    width = element.get("width", 100)
                    height = element.get("height", 100)
                    return Image(file_path, width=width, height=height)
                if current_app:
                    current_app.logger.warning(
                        f"Template image not found: {source} (tried get_image_base64 and _resolve_template_image_path)"
                    )
            except Exception as e:
                if current_app:
                    current_app.logger.error(f"Error loading template image {source}: {e}")
                else:
                    print(f"Error loading template image {source}: {e}")
                return None

        # Handle file path
        if os.path.exists(source):
            try:
                width = element.get("width", 100)  # Already in points from generateCode
                height = element.get("height", 100)  # Already in points from generateCode
                # Image() constructor preserves transparency automatically for PNG images
                return Image(source, width=width, height=height)
            except Exception as e:
                if current_app:
                    current_app.logger.error(f"Error loading image {source}: {e}")
                else:
                    print(f"Error loading image {source}: {e}")
                return None

        return None

    def _render_rectangle(self, element: Dict[str, Any]) -> Flowable:
        """Render a rectangle element"""

        class RectangleFlowable(Flowable):
            def __init__(self, width, height, fill, stroke, stroke_width):
                self.width = width
                self.height = height
                self.fill = fill
                self.stroke = stroke
                self.stroke_width = stroke_width
                Flowable.__init__(self)

            def draw(self):
                self.canv.saveState()
                if self.fill:
                    fill_hex = _normalize_color(self.fill)
                    if fill_hex:
                        self.canv.setFillColor(colors.HexColor(fill_hex))
                if self.stroke:
                    stroke_hex = _normalize_color(self.stroke)
                    if stroke_hex:
                        self.canv.setStrokeColor(colors.HexColor(stroke_hex))
                self.canv.setLineWidth(self.stroke_width)
                self.canv.rect(0, 0, self.width, self.height, fill=bool(self.fill), stroke=bool(self.stroke))
                self.canv.restoreState()

            def wrap(self, availWidth, availHeight):
                return (self.width, self.height)

        width = element.get("width", 100) * pt
        height = element.get("height", 100) * pt
        fill = element.get("style", {}).get("fill", element.get("fill"))
        stroke = element.get("style", {}).get("stroke", element.get("stroke"))
        stroke_width = element.get("style", {}).get("strokeWidth", element.get("strokeWidth", 1))

        return RectangleFlowable(width, height, fill, stroke, stroke_width or 0)

    def _render_circle(self, element: Dict[str, Any]) -> Flowable:
        """Render a circle element"""

        class CircleFlowable(Flowable):
            def __init__(self, radius, fill, stroke, stroke_width):
                self.radius = radius
                self.fill = fill
                self.stroke = stroke
                self.stroke_width = stroke_width
                Flowable.__init__(self)

            def draw(self):
                self.canv.saveState()
                if self.fill:
                    fill_hex = _normalize_color(self.fill)
                    if fill_hex:
                        self.canv.setFillColor(colors.HexColor(fill_hex))
                if self.stroke:
                    stroke_hex = _normalize_color(self.stroke)
                    if stroke_hex:
                        self.canv.setStrokeColor(colors.HexColor(stroke_hex))
                self.canv.setLineWidth(self.stroke_width)
                self.canv.circle(self.radius, self.radius, self.radius, fill=bool(self.fill), stroke=bool(self.stroke))
                self.canv.restoreState()

            def wrap(self, availWidth, availHeight):
                size = self.radius * 2
                return (size, size)

        radius = (element.get("width", 100) / 2) * pt
        fill = element.get("style", {}).get("fill", element.get("fill"))
        stroke = element.get("style", {}).get("stroke", element.get("stroke"))
        stroke_width = element.get("style", {}).get("strokeWidth", element.get("strokeWidth", 1))

        return CircleFlowable(radius, fill, stroke, stroke_width or 0)

    def _render_line(self, element: Dict[str, Any]) -> Flowable:
        """Render a line element"""

        class LineFlowable(Flowable):
            def __init__(self, width, stroke, stroke_width):
                self.width = width
                self.stroke = stroke
                self.stroke_width = stroke_width
                Flowable.__init__(self)

            def draw(self):
                self.canv.saveState()
                if self.stroke:
                    stroke_hex = _normalize_color(self.stroke)
                    if stroke_hex:
                        self.canv.setStrokeColor(colors.HexColor(stroke_hex))
                self.canv.setLineWidth(self.stroke_width)
                self.canv.line(0, 0, self.width, 0)
                self.canv.restoreState()

            def wrap(self, availWidth, availHeight):
                return (self.width, self.stroke_width)

        width = element.get("width", 100) * pt
        stroke = element.get("style", {}).get("stroke", element.get("stroke", "#000000"))
        stroke_width = element.get("style", {}).get("strokeWidth", element.get("strokeWidth", 1))

        return LineFlowable(width, stroke, stroke_width or 1)

    def _normalize_extra_good_for_items_row(self, good: Any) -> SimpleNamespace:
        """Convert an ExtraGood to a row-like object with description, quantity, unit_price, total_amount.
        Description is built from name + optional description, SKU, category (same shape as items table)."""
        description_parts = [getattr(good, "name", "") or ""]
        if getattr(good, "description", None):
            description_parts.append(f"\n{good.description}")
        if getattr(good, "sku", None):
            description_parts.append(f"\n{_('SKU')}: {good.sku}")
        if getattr(good, "category", None):
            cat = good.category.title() if isinstance(good.category, str) else str(good.category)
            description_parts.append(f"\n{_('Category')}: {cat}")
        description = "\n".join(description_parts)
        return SimpleNamespace(
            description=description,
            quantity=getattr(good, "quantity", 0),
            unit_price=getattr(good, "unit_price", 0),
            total_amount=getattr(good, "total_amount", 0),
        )

    def _normalize_expense_for_items_row(self, expense: Any) -> SimpleNamespace:
        """Convert an Expense to a row-like object with description, quantity=1, unit_price, total_amount.
        Same shape as items table for backward compatibility when template uses invoice.items."""
        desc_parts = [getattr(expense, "title", str(expense)) or ""]
        if getattr(expense, "description", None):
            desc_parts.append(str(expense.description))
        description = "\n".join(desc_parts)
        amt = getattr(expense, "total_amount", None) or getattr(expense, "amount", 0)
        return SimpleNamespace(
            description=description,
            quantity=1,
            unit_price=amt,
            total_amount=amt,
        )

    def _scaled_table_col_widths(self, element: Dict[str, Any]) -> List[float]:
        """Column widths in points, scaled so their sum matches element.width when set."""
        columns = element.get("columns") or []
        if not columns:
            return []
        raw: List[float] = []
        for c in columns:
            try:
                raw.append(float(c.get("width", 100) or 100))
            except (TypeError, ValueError):
                raw.append(100.0)
        s = sum(raw)
        if s <= 0:
            raw = [100.0] * len(columns)
            s = sum(raw)
        target_f: Optional[float] = None
        tw = element.get("width")
        if tw is not None:
            try:
                twf = float(tw)
                if twf > 0:
                    target_f = twf
            except (TypeError, ValueError):
                target_f = None
        if target_f is not None and abs(s - target_f) > 0.01:
            scale = target_f / s
            raw = [w * scale for w in raw]
        min_w = 10.0
        raw = [max(min_w, w) for w in raw]
        s2 = sum(raw)
        if target_f is not None and s2 > target_f + 0.01:
            scale2 = target_f / s2
            raw = [w * scale2 for w in raw]
        return raw

    def _render_table(self, element: Dict[str, Any]) -> Table:
        """Render a table element"""
        columns = element.get("columns", [])

        # Build header row - process template variables in headers
        headers = []
        for col in columns:
            header_text = col.get("header", "")
            header_text = self._process_template_variables(header_text)
            # Convert newlines to <br/> tags for Paragraph
            header_text = header_text.replace("\r\n", "<br/>").replace("\r", "<br/>").replace("\n", "<br/>")
            # Use Paragraph for headers to handle line breaks
            header_style = self._get_style({"style": {}}, "NormalText")
            headers.append(Paragraph(header_text, header_style))
        table_data = [headers]

        # Get data source
        data_source = element.get("data", "")
        if data_source:
            # Process template variable to get actual data
            data = self._resolve_data_source(data_source)
            # When this table is the invoice items table, include extra_goods and expenses so they appear in the PDF
            var_name = data_source.replace("{{", "").replace("}}", "").strip()
            if var_name == "invoice.items":
                invoice = self.context.get("invoice")
                if invoice is not None:
                    extra_goods = getattr(invoice, "extra_goods", None)
                    if extra_goods is not None:
                        if hasattr(extra_goods, "all"):
                            extra_goods = list(extra_goods.all())
                        elif hasattr(extra_goods, "__iter__") and not isinstance(extra_goods, (str, bytes)):
                            extra_goods = list(extra_goods)
                        else:
                            extra_goods = []
                        data = list(data) + [self._normalize_extra_good_for_items_row(g) for g in extra_goods]
                    expenses = getattr(invoice, "expenses", None)
                    if expenses is not None:
                        if hasattr(expenses, "all"):
                            expenses = list(expenses.all())
                        elif hasattr(expenses, "__iter__") and not isinstance(expenses, (str, bytes)):
                            expenses = list(expenses)
                        else:
                            expenses = []
                        data = list(data) + [self._normalize_expense_for_items_row(e) for e in expenses]
            row_template = element.get("row_template", {})

            if isinstance(data, list) and data:
                for item in data:
                    row = []
                    for col in columns:
                        field = col.get("field", "")
                        # Process row template with item context
                        value = self._process_row_template(field, row_template.get(field, ""), item)
                        cell_text = str(value) if value is not None else ""
                        # Convert newlines to <br/> tags for Paragraph
                        cell_text = cell_text.replace("\r\n", "<br/>").replace("\r", "<br/>").replace("\n", "<br/>")
                        # Use Paragraph for cells to handle line breaks
                        cell_style = self._get_style({"style": {}}, "NormalText")
                        row.append(Paragraph(cell_text, cell_style))
                    table_data.append(row)
            else:
                # No data - add empty row message
                num_cols = len(columns)
                empty_row = [Paragraph("No data", self._get_style({"style": {}}, "NormalText"))]
                empty_row.extend([Paragraph("", self._get_style({"style": {}}, "NormalText"))] * (num_cols - 1))
                table_data.append(empty_row)

        # Column widths in points (scaled to element.width when set)
        col_widths = self._scaled_table_col_widths(element)

        # Create table with proper column widths
        table = Table(table_data, colWidths=col_widths, repeatRows=1)

        # Apply table style
        table_style = self._get_table_style(element, columns)
        table.setStyle(table_style)

        # Wrap table in KeepTogether to prevent breaking across pages
        return KeepTogether(table)

    def _get_table_style(self, element: Dict[str, Any], columns: List[Dict[str, Any]]) -> TableStyle:
        """Get table style from element configuration"""
        style_config = element.get("style", {})

        # Get default colors from style config or use defaults
        header_bg = colors.HexColor(style_config.get("headerBackground", "#f8f9fa"))
        header_text_color = colors.HexColor(style_config.get("headerTextColor", "#000000"))
        row_bg = colors.HexColor(style_config.get("rowBackground", "#ffffff"))
        row_text_color = colors.HexColor(style_config.get("rowTextColor", "#000000"))

        try:
            grid_w = float(style_config.get("borderWidth", 0.5))
        except (TypeError, ValueError):
            grid_w = 0.5
        if grid_w <= 0:
            grid_w = 0.5
        grid_color_hex = _normalize_color(style_config.get("borderColor", "#e2e8f0")) or "#e2e8f0"
        grid_color = colors.HexColor(grid_color_hex)

        commands = [
            # Header row styling
            ("BACKGROUND", (0, 0), (-1, 0), header_bg),
            ("TEXTCOLOR", (0, 0), (-1, 0), header_text_color),
            ("FONTNAME", (0, 0), (-1, 0), "Helvetica-Bold"),
            ("FONTSIZE", (0, 0), (-1, 0), 12),
            ("BOTTOMPADDING", (0, 0), (-1, 0), 12),
            ("TOPPADDING", (0, 0), (-1, 0), 12),
            # Data rows styling
            ("BACKGROUND", (0, 1), (-1, -1), row_bg),
            ("TEXTCOLOR", (0, 1), (-1, -1), row_text_color),
            ("FONTNAME", (0, 1), (-1, -1), "Helvetica"),
            ("FONTSIZE", (0, 1), (-1, -1), 10),
            ("ROWBACKGROUNDS", (0, 1), (-1, -1), [colors.HexColor("#ffffff"), colors.HexColor("#f9fafb")]),
            # Grid and borders
            ("GRID", (0, 0), (-1, -1), grid_w, grid_color),
        ]

        # Apply column alignments (both header and data rows)
        for idx, col in enumerate(columns):
            align = col.get("align", "left").lower()
            if align == "right":
                commands.append(("ALIGN", (idx, 0), (idx, -1), "RIGHT"))  # Header + data
            elif align == "center":
                commands.append(("ALIGN", (idx, 0), (idx, -1), "CENTER"))  # Header + data
            else:  # left
                commands.append(("ALIGN", (idx, 0), (idx, -1), "LEFT"))  # Header + data

        return TableStyle(commands)

    def _get_style(self, element: Dict[str, Any], default_style_name: str = "NormalText") -> ParagraphStyle:
        """Get paragraph style from element configuration"""
        style_config = element.get("style", {})

        # Get base style
        base_style = self.styles.get(default_style_name, self.styles["Normal"])

        # Create custom style
        custom_style = ParagraphStyle(
            name=f"Custom_{id(element)}",
            parent=base_style,
            fontSize=style_config.get("size", base_style.fontSize),
            textColor=colors.HexColor(style_config.get("color", "#000000")),
            fontName=style_config.get("font", base_style.fontName),
            alignment=self._get_alignment(style_config.get("align", "left")),
            spaceAfter=style_config.get("spaceAfter", base_style.spaceAfter),
        )

        return custom_style

    def _get_alignment(self, align: str) -> int:
        """Convert alignment string to ReportLab constant"""
        align_map = {
            "left": TA_LEFT,
            "center": TA_CENTER,
            "right": TA_RIGHT,
            "justify": TA_JUSTIFY,
        }
        return align_map.get(align.lower(), TA_LEFT)

    def _process_template_variables(self, text: str) -> str:
        """Process Jinja2-style template variables in text"""
        from app.utils.safe_template_render import render_sandboxed_string

        try:
            # Render with context
            rendered = render_sandboxed_string(text, autoescape=False, **self.context)
            return rendered
        except Exception as e:
            if current_app:
                current_app.logger.error(f"Error processing template variables: {e}")
            else:
                print(f"Error processing template variables: {e}")
            return text

    def _resolve_data_source(self, data_source: str) -> List[Any]:
        """Resolve data source template variable to actual data"""
        # Remove template syntax
        var_name = data_source.replace("{{", "").replace("}}", "").strip()

        # Resolve from context (which is a dict)
        try:
            parts = var_name.split(".")
            value: Any = self.context

            # Navigate through nested attributes/dict keys
            for part in parts:
                if isinstance(value, dict):
                    value = value.get(part)
                else:
                    value = getattr(value, part, None)

                if value is None:
                    break

            # Convert to list if it's a query object or SQLAlchemy result
            if value is None:
                return []

            if hasattr(value, "all"):  # SQLAlchemy query
                return list(value.all())
            elif hasattr(value, "__iter__") and not isinstance(value, (str, bytes)):
                return list(value)
            elif value is not None:
                return [value]

            return []
        except Exception as e:
            if current_app:
                current_app.logger.error(f"Error resolving data source {data_source}: {e}")
            else:
                print(f"Error resolving data source {data_source}: {e}")
            import traceback

            if current_app:
                current_app.logger.debug(traceback.format_exc())
            else:
                print(traceback.format_exc())
            return []

    def _process_row_template(self, field: str, template: str, item: Any) -> str:
        """Process row template with item context"""
        if not template:
            # Fallback to direct attribute access
            return str(getattr(item, field, ""))

        # Process template with item in context
        from app.utils.safe_template_render import render_sandboxed_string

        try:
            return render_sandboxed_string(template, autoescape=False, item=item, **self.context)
        except Exception as e:
            if current_app:
                current_app.logger.error(f"Error processing row template: {e}")
            else:
                print(f"Error processing row template: {e}")
            return str(getattr(item, field, ""))

    def _on_page(self, canv: canvas.Canvas, doc: SimpleDocTemplate):
        """Handle page rendering - draw absolutely positioned elements and page number"""
        # Get page dimensions
        page_width = getattr(self, "page_width", doc.pagesize[0])
        page_height = getattr(self, "page_height", doc.pagesize[1])

        # Get margins
        margins = self.template.get("page", {}).get("margin", {"top": 20, "right": 20, "bottom": 20, "left": 20})
        margin_top = margins.get("top", 20) * mm
        margin_left = margins.get("left", 20) * mm

        for element_data in getattr(self, "absolute_elements", []):
            element = element_data["element"]
            if element.get("hidden"):
                continue
            element_type = element.get("type")

            # Get position (already in points from generateCode); coerce to float (JSON may give int/str)
            try:
                x = float(element.get("x", 0) or 0)
                y = float(element.get("y", 0) or 0)
                width = float(element.get("width", 0) or 0)
                height = float(element.get("height", 0) or 0)
            except (TypeError, ValueError):
                x = y = width = height = 0

            # Validate element position to page boundaries
            # Elements are positioned relative to page (0,0 = top-left of page)
            # IMAGE: skip only when rect does not intersect page (so overflowing images still draw)
            # Others: skip when top-left is outside page
            if element_type == ElementType.IMAGE:
                if x + width <= 0 or x >= page_width or y >= page_height or y + height <= 0:
                    continue
            else:
                if x < 0 or y < 0 or x >= page_width or y >= page_height:
                    continue

            # For elements with explicit dimensions, constrain them to page boundaries
            # Issue #537: IMAGE elements - don't constrain; draw at full size and clip in _draw_image_on_canvas
            # draw at full size and let PDF clip; user expects overflowing images to still show
            is_image = element_type == ElementType.IMAGE
            if width > 0 and height > 0 and not is_image:
                # Constrain dimensions if they would extend beyond page
                if x + width > page_width:
                    width = max(0, page_width - x)
                if y + height > page_height:
                    height = max(0, page_height - y)

                # Skip if constrained to zero size
                if width <= 0 or height <= 0:
                    continue

            # Update element with constrained dimensions for drawing
            element_copy = element.copy()
            element_copy["x"] = x
            element_copy["y"] = y
            element_copy["width"] = width
            element_copy["height"] = height

            # Elements are positioned relative to page (0,0 = top-left of page)
            # ReportLab's SimpleDocTemplate margins define the content area, but when drawing
            # directly on canvas in _on_page(), we use absolute page coordinates
            # So we should NOT add margins - elements are already at correct page positions
            actual_x = x
            # y is from top, so invert it for ReportLab (which uses bottom-left origin)
            actual_y = page_height - y

            try:
                if element_type == ElementType.TEXT:
                    self._draw_text_on_canvas(canv, element_copy, actual_x, actual_y, page_height)
                elif element_type == ElementType.IMAGE:
                    self._draw_image_on_canvas(canv, element_copy, actual_x, actual_y, page_height)
                elif element_type == ElementType.RECTANGLE:
                    self._draw_rectangle_on_canvas(canv, element_copy, actual_x, actual_y, page_height)
                elif element_type == ElementType.CIRCLE:
                    self._draw_circle_on_canvas(canv, element_copy, actual_x, actual_y, page_height)
                elif element_type == ElementType.LINE:
                    self._draw_line_on_canvas(canv, element_copy, actual_x, actual_y, page_height)
                # Tables are handled as flowables, not drawn on canvas
            except Exception as e:
                if current_app:
                    current_app.logger.error(f"Error drawing element {element_type} on canvas: {e}")
                else:
                    print(f"Error drawing element {element_type} on canvas: {e}")

        # Add page number (within page boundaries)
        page_num = canv.getPageNumber()
        text = f"Page {page_num}"
        canv.saveState()
        canv.setFont("Helvetica", 9)
        canv.setFillColor(colors.HexColor("#666666"))
        # Ensure page number is within page boundaries
        page_num_x = min(doc.leftMargin + doc.width, page_width - 10)
        page_num_y = max(doc.bottomMargin - 0.5 * cm, 10)
        canv.drawRightString(page_num_x, page_num_y, text)
        canv.restoreState()

    def _draw_text_on_canvas(
        self, canv: canvas.Canvas, element: Dict[str, Any], x: float, y: float, page_height: float
    ):
        """Draw text element on canvas"""
        text = element.get("text", "")
        text = self._process_template_variables(text)

        style = element.get("style", {})
        font = style.get("font", "Helvetica")
        size = style.get("size", 10)
        color = style.get("color", "#000000")
        align = style.get("align", "left")

        # Get page width for boundary checking
        page_width = getattr(self, "page_width", 595)  # Default A4 width

        canv.saveState()
        canv.setFont(font, size)
        color_hex = _normalize_color(color)
        if color_hex:
            canv.setFillColor(colors.HexColor(color_hex))

        # Constrain text width to page boundaries
        width = element.get("width", 400)
        max_x = min(x + width, page_width)
        constrained_width = max_x - x

        # Split text by newlines to handle line breaks properly
        # Also handle \r\n (Windows) and \r (old Mac) line endings
        lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")

        # Calculate line height (font size * 1.2 for spacing)
        line_height = size * 1.2

        valign = style.get("verticalAlign", "top")
        box_h = float(element.get("height") or 0)
        total_text_height = line_height * max(1, len(lines))
        current_y = y
        if box_h > 0:
            if valign == "middle":
                current_y = y - (box_h - total_text_height) / 2.0
            elif valign == "bottom":
                current_y = y - box_h + total_text_height

        if align == "justify" and constrained_width > 0:
            joined = "<br/>".join(lines) if lines else text
            ps = ParagraphStyle(
                name="_pdf_editor_justify",
                fontName=font,
                fontSize=size,
                leading=line_height,
                alignment=TA_JUSTIFY,
                textColor=colors.black,
            )
            if color_hex:
                ps.textColor = colors.HexColor(color_hex)
            para = Paragraph(joined, ps)
            w, h = para.wrap(constrained_width, page_height)
            para.drawOn(canv, x, current_y - h + size * 0.15)
            canv.restoreState()
            return

        # Draw each line separately
        for line in lines:
            # Skip empty lines but still advance y position
            if not line.strip() and len(lines) > 1:
                current_y -= line_height
                continue

            # Handle text alignment with boundary constraints for each line
            if align == "right":
                draw_x = min(x + constrained_width, page_width - 5)
                canv.drawRightString(draw_x, current_y, line)
            elif align == "center":
                draw_x = x + constrained_width / 2
                canv.drawCentredString(draw_x, current_y, line)
            else:
                draw_x = min(x, page_width - 5)
                canv.drawString(draw_x, current_y, line)

            # Move to next line (decrease y since ReportLab uses bottom-left origin)
            current_y -= line_height

        canv.restoreState()

    def _resolve_template_image_path(self, filename: str) -> Optional[str]:
        """
        Resolve template image filename to absolute file path.
        Issue #537: Tries multiple paths to handle different deployment layouts.

        Returns:
            Absolute file path if file exists, None otherwise.
        """
        if not filename or not isinstance(filename, str):
            return None
        filename = filename.strip()
        if not filename:
            return None
        try:
            # Same path logic as upload route (app/routes/admin.py upload_template_image)
            root = getattr(current_app, "root_path", None) if current_app else None
            if not root:
                return None
            upload_folder = "app/static/uploads/template_images"
            candidates = [
                os.path.join(root, "..", upload_folder, filename),
                os.path.join(root, "static", "uploads", "template_images", filename),
                os.path.join(os.path.dirname(root), upload_folder, filename),
            ]
            for path in candidates:
                resolved = os.path.abspath(path)
                if os.path.exists(resolved) and os.path.isfile(resolved):
                    return resolved
        except Exception as e:
            if current_app:
                current_app.logger.warning(f"Error resolving template image path for '{filename}': {e}")
        return None

    def _extract_template_image_filename(self, source: str) -> Optional[str]:
        """
        Extract filename from template image URL/path.
        Handles: /uploads/template_images/xxx, https://host/uploads/template_images/xxx
        """
        if not source or not isinstance(source, str):
            return None
        source = source.strip()
        marker = "/uploads/template_images/"
        idx = source.find(marker)
        if idx >= 0:
            return source[idx + len(marker) :].strip()
        return None

    def _draw_image_on_canvas(
        self, canv: canvas.Canvas, element: Dict[str, Any], x: float, y: float, page_height: float
    ):
        """Draw image element on canvas"""
        source = element.get("source", "")
        source = self._process_template_variables(source)

        # Issue #432: Explicit skip for decorative images with empty source (never draw)
        if element.get("decorative", False) and (not source or not source.strip()):
            if current_app:
                current_app.logger.warning(f"Skipping decorative image with empty source at position ({x}, {y})")
            return
        if not source or not source.strip():
            if current_app:
                current_app.logger.warning(f"Skipping image with empty source at position ({x}, {y})")
            return

        width = element.get("width", 100)
        height = element.get("height", 100)
        # Issue #537: Decorative images use exact dimensions (no aspect ratio preservation)
        preserve_aspect = not element.get("decorative", False)

        # Issue #537: Draw images at full size when overflowing; clip to page so visible portion shows
        if width <= 0 or height <= 0:
            return

        page_width = getattr(self, "page_width", 595)

        def _draw_image_clipped(img_source, draw_x, draw_y, draw_w, draw_h):
            """Draw image with clip to page rectangle so overflow is clipped."""
            canv.saveState()
            try:
                # Clip to page: path rect then clipPath (no stroke/fill)
                path = canv.beginPath()
                path.rect(0, 0, page_width, page_height)
                canv.clipPath(path, stroke=0, fill=0)
                canv.drawImage(
                    img_source,
                    draw_x,
                    draw_y,
                    width=draw_w,
                    height=draw_h,
                    preserveAspectRatio=preserve_aspect,
                    mask="auto",
                )
            finally:
                canv.restoreState()

        try:
            draw_y = y - height
            # Handle base64 data URI with validated decode (Issue #432)
            if source.startswith("data:image"):
                import base64

                parts = source.split(",", 1)
                if len(parts) < 2 or not (parts[1] and parts[1].strip()):
                    if current_app:
                        current_app.logger.warning("Skipping image draw: data URI has no base64 payload")
                    return
                try:
                    img_data = base64.b64decode(parts[1])
                    img_reader = ImageReader(io.BytesIO(img_data))
                    _draw_image_clipped(img_reader, x, draw_y, width, height)
                except Exception as e:
                    if current_app:
                        current_app.logger.error(f"Error decoding base64 image for canvas: {e}")
                    return
            # Handle template image URLs (convert to file path or base64)
            # Issue #537: Also handle full URLs; try file path first for reliability
            elif "/uploads/template_images/" in source:
                try:
                    filename = self._extract_template_image_filename(source)
                    if not filename:
                        if current_app:
                            current_app.logger.warning(f"Could not extract filename from template image URL: {source}")
                        raise ValueError("Invalid template image URL")
                    img_reader = None
                    # Try direct file path first (most reliable across deployments)
                    file_path = self._resolve_template_image_path(filename)
                    if file_path and os.path.exists(file_path):
                        img_reader = ImageReader(file_path)
                    if not img_reader:
                        from app.utils.template_filters import get_image_base64

                        relative_path = f"app/static/uploads/template_images/{filename}"
                        base64_data = get_image_base64(relative_path)
                        if base64_data:
                            import base64 as b64

                            header, data = base64_data.split(",", 1)
                            img_data = b64.b64decode(data)
                            img_reader = ImageReader(io.BytesIO(img_data))
                    if img_reader:
                        if current_app:
                            current_app.logger.info(
                                f"[PDF_EXPORT] Drawing template image: {filename} at ({x}, {y - height}) size {width}x{height}"
                            )
                        _draw_image_clipped(img_reader, x, draw_y, width, height)
                    elif current_app:
                        current_app.logger.warning(
                            f"Template image not found: {source} (tried file path and get_image_base64)"
                        )
                except Exception as e:
                    if current_app:
                        current_app.logger.error(f"Error loading template image {source}: {e}")
                    else:
                        print(f"Error loading template image {source}: {e}")
            elif os.path.exists(source):
                _draw_image_clipped(source, x, draw_y, width, height)
        except Exception as e:
            if current_app:
                current_app.logger.error(f"Error drawing image on canvas: {e}")
            else:
                print(f"Error drawing image on canvas: {e}")

    def _draw_rectangle_on_canvas(
        self, canv: canvas.Canvas, element: Dict[str, Any], x: float, y: float, page_height: float
    ):
        """Draw rectangle element on canvas"""
        width = element.get("width", 100)
        height = element.get("height", 100)
        style = element.get("style", {})
        fill = style.get("fill", element.get("fill"))
        stroke = style.get("stroke", element.get("stroke", "black"))
        stroke_width = style.get("strokeWidth", element.get("strokeWidth", 1)) or 0

        # Get page dimensions for boundary checking
        page_width = getattr(self, "page_width", 595)  # Default A4 width

        # Constrain rectangle to page boundaries
        # y is in ReportLab coordinates (bottom-left origin), so y - height is the bottom
        max_width = page_width - x
        max_height = y  # y is the top, so y is the max height from bottom
        width = min(width, max_width)
        height = min(height, max_height)

        # Skip if rectangle would be outside page boundaries
        if width <= 0 or height <= 0 or x >= page_width or y <= 0:
            return

        canv.saveState()
        if fill and fill != "transparent":
            fill_hex = _normalize_color(fill)
            if fill_hex:
                canv.setFillColor(colors.HexColor(fill_hex))
        if stroke and stroke_width > 0:
            stroke_hex = _normalize_color(stroke)
            if stroke_hex:
                canv.setStrokeColor(colors.HexColor(stroke_hex))
            canv.setLineWidth(stroke_width)

        canv.rect(
            x,
            y - height,
            width,
            height,
            fill=bool(fill and fill != "transparent"),
            stroke=bool(stroke and stroke_width > 0),
        )
        canv.restoreState()

    def _draw_circle_on_canvas(
        self, canv: canvas.Canvas, element: Dict[str, Any], x: float, y: float, page_height: float
    ):
        """Draw circle element on canvas"""
        width = element.get("width", 100)
        radius = width / 2
        style = element.get("style", {})
        fill = style.get("fill", element.get("fill"))
        stroke = style.get("stroke", element.get("stroke", "black"))
        stroke_width = style.get("strokeWidth", element.get("strokeWidth", 1)) or 0

        # Get page dimensions for boundary checking
        page_width = getattr(self, "page_width", 595)  # Default A4 width

        # Constrain circle to page boundaries
        # Circle center will be at (x + radius, y - radius)
        # Ensure circle doesn't extend beyond page
        max_radius_x = min(page_width - x, x)  # Distance to nearest horizontal edge
        max_radius_y = min(y, page_height - y)  # Distance to nearest vertical edge
        max_radius = min(max_radius_x, max_radius_y)
        radius = min(radius, max_radius)

        # Skip if circle would be outside page boundaries
        if radius <= 0 or x + radius > page_width or y - radius < 0:
            return

        canv.saveState()
        if fill and fill != "transparent":
            fill_hex = _normalize_color(fill)
            if fill_hex:
                canv.setFillColor(colors.HexColor(fill_hex))
        if stroke and stroke_width > 0:
            stroke_hex = _normalize_color(stroke)
            if stroke_hex:
                canv.setStrokeColor(colors.HexColor(stroke_hex))
            canv.setLineWidth(stroke_width)

        # Circle center is at (x + radius, y - radius), radius is half width
        canv.circle(
            x + radius,
            y - radius,
            radius,
            fill=bool(fill and fill != "transparent"),
            stroke=bool(stroke and stroke_width > 0),
        )
        canv.restoreState()

    def _draw_line_on_canvas(
        self, canv: canvas.Canvas, element: Dict[str, Any], x: float, y: float, page_height: float
    ):
        """Draw line element on canvas"""
        width = element.get("width", 100)
        style = element.get("style", {})
        stroke = style.get("stroke", element.get("stroke", "black"))
        stroke_width = style.get("strokeWidth", element.get("strokeWidth", 1)) or 1

        # Get page dimensions for boundary checking
        page_width = getattr(self, "page_width", 595)  # Default A4 width

        # Constrain line to page boundaries
        # Ensure line doesn't extend beyond page
        max_width = page_width - x
        width = min(width, max_width)

        # Skip if line would be outside page boundaries
        if width <= 0 or x >= page_width or y < 0 or y > page_height:
            return

        canv.saveState()
        stroke_hex = _normalize_color(stroke)
        if stroke_hex:
            canv.setStrokeColor(colors.HexColor(stroke_hex))
        canv.setLineWidth(stroke_width)
        canv.line(x, y, x + width, y)
        canv.restoreState()