Merge pull request 'feature/peeron' (#104) from feature/peeron into release/1.3

Reviewed-on: https://gitea.baerentsen.space/FrederikBaerentsen/BrickTracker/pulls/104
This commit is contained in:
FrederikBaerentsen
2025-09-26 11:40:01 +02:00
15 changed files with 1205 additions and 17 deletions
+30 -3
View File
@@ -262,9 +262,36 @@
# Default: https://rebrickable.com/instructions/{path}
# BK_REBRICKABLE_LINK_INSTRUCTIONS_PATTERN=
# Optional: User-Agent to use when querying Rebrickable outside of the Rebrick python library
# Default: 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
# BK_REBRICKABLE_USER_AGENT=
# Optional: User-Agent to use when querying Rebrickable and Peeron outside of the Rebrick python library
# Default: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36
# BK_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36
# Legacy: User-Agent for Rebrickable (use BK_USER_AGENT instead)
# BK_REBRICKABLE_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36
# Optional: Delay in milliseconds between Peeron page downloads to avoid being potentially blocked
# Default: 1000
# BK_PEERON_DOWNLOAD_DELAY=1000
# Optional: Minimum image size (width/height) for valid Peeron instruction pages
# Images smaller than this are considered error placeholders and will be rejected
# Default: 100
# BK_PEERON_MIN_IMAGE_SIZE=100
# Optional: Pattern for Peeron instruction page URLs. Will be passed to Python .format()
# Supports {set_number} and {version_number} parameters
# Default: http://peeron.com/scans/{set_number}-{version_number}
# BK_PEERON_INSTRUCTION_PATTERN=
# Optional: Pattern for Peeron thumbnail URLs. Will be passed to Python .format()
# Supports {set_number} and {version_number} parameters
# Default: http://belay.peeron.com/thumbs/{set_number}-{version_number}/
# BK_PEERON_THUMBNAIL_PATTERN=
# Optional: Pattern for Peeron scan URLs. Will be passed to Python .format()
# Supports {set_number} and {version_number} parameters
# Default: http://belay.peeron.com/scans/{set_number}-{version_number}/
# BK_PEERON_SCAN_PATTERN=
# Optional: Display Rebrickable links wherever applicable
# Default: false
+2 -1
View File
@@ -60,7 +60,8 @@ def setup_app(app: Flask) -> None:
# Setup the login manager
LoginManager(app)
# I don't know :-)
# Configure proxy header handling for reverse proxy deployments (nginx, Apache, etc.)
# This ensures proper client IP detection and HTTPS scheme recognition
app.wsgi_app = ProxyFix(
app.wsgi_app,
x_for=1,
+4 -2
View File
@@ -101,8 +101,9 @@ class BrickInstructions(object):
# Skip if we already have it
if os.path.isfile(target):
pdf_url = self.url()
return self.socket.complete(
message=f"File {self.filename} already exists, skipped"
message=f'File {self.filename} already exists, skipped - <a href="{pdf_url}" target="_blank" class="btn btn-sm btn-primary ms-2"><i class="ri-external-link-line"></i> Open PDF</a>'
)
# Fetch PDF via cloudscraper (to bypass Cloudflare)
@@ -141,8 +142,9 @@ class BrickInstructions(object):
# Done!
logger.info(f"Downloaded {self.filename}")
pdf_url = self.url()
self.socket.complete(
message=f"File {self.filename} downloaded ({self.human_size()})"
message=f'File {self.filename} downloaded ({self.human_size()}) - <a href="{pdf_url}" target="_blank" class="btn btn-sm btn-primary ms-2"><i class="ri-external-link-line"></i> Open PDF</a>'
)
except Exception as e:
+437
View File
@@ -0,0 +1,437 @@
import hashlib
import logging
import os
from pathlib import Path
import time
from typing import Any, NamedTuple, TYPE_CHECKING
from urllib.parse import urljoin
from bs4 import BeautifulSoup
import cloudscraper
from flask import current_app, url_for
import requests
from .exceptions import ErrorException
if TYPE_CHECKING:
from .socket import BrickSocket
logger = logging.getLogger(__name__)
def get_peeron_user_agent():
"""Get the User-Agent string for Peeron requests from config"""
return current_app.config.get('REBRICKABLE_USER_AGENT',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36')
def get_peeron_download_delay():
"""Get the delay in milliseconds between Peeron page downloads from config"""
return current_app.config.get('PEERON_DOWNLOAD_DELAY', 1000)
def get_min_image_size():
"""Get the minimum image size for valid Peeron instruction pages from config"""
return current_app.config.get('PEERON_MIN_IMAGE_SIZE', 100)
def get_peeron_instruction_url(set_number: str, version_number: str):
"""Get the Peeron instruction page URL using the configured pattern"""
pattern = current_app.config.get('PEERON_INSTRUCTION_PATTERN', 'http://peeron.com/scans/{set_number}-{version_number}')
return pattern.format(set_number=set_number, version_number=version_number)
def get_peeron_thumbnail_url(set_number: str, version_number: str):
"""Get the Peeron thumbnail base URL using the configured pattern"""
pattern = current_app.config.get('PEERON_THUMBNAIL_PATTERN', 'http://belay.peeron.com/thumbs/{set_number}-{version_number}/')
return pattern.format(set_number=set_number, version_number=version_number)
def get_peeron_scan_url(set_number: str, version_number: str):
"""Get the Peeron scan base URL using the configured pattern"""
pattern = current_app.config.get('PEERON_SCAN_PATTERN', 'http://belay.peeron.com/scans/{set_number}-{version_number}/')
return pattern.format(set_number=set_number, version_number=version_number)
def create_peeron_scraper():
"""Create a cloudscraper instance configured for Peeron"""
scraper = cloudscraper.create_scraper()
scraper.headers.update({
"User-Agent": get_peeron_user_agent()
})
return scraper
def get_peeron_cache_dir():
"""Get the base directory for Peeron caching"""
static_dir = Path(current_app.static_folder)
cache_dir = static_dir / 'images' / 'peeron_cache'
cache_dir.mkdir(parents=True, exist_ok=True)
return cache_dir
def get_set_cache_dir(set_number: str, version_number: str) -> tuple[Path, Path]:
"""Get cache directories for a specific set"""
base_cache_dir = get_peeron_cache_dir()
set_cache_key = f"{set_number}-{version_number}"
full_cache_dir = base_cache_dir / 'full' / set_cache_key
thumb_cache_dir = base_cache_dir / 'thumbs' / set_cache_key
full_cache_dir.mkdir(parents=True, exist_ok=True)
thumb_cache_dir.mkdir(parents=True, exist_ok=True)
return full_cache_dir, thumb_cache_dir
def cache_full_image_and_generate_thumbnail(image_url: str, page_number: str, set_number: str, version_number: str, session=None) -> tuple[str | None, str | None]:
"""
Download and cache full-size image, then generate a thumbnail preview.
Uses the full-size scan URLs from Peeron.
Returns (cached_image_path, thumbnail_url) or (None, None) if caching fails.
"""
try:
full_cache_dir, thumb_cache_dir = get_set_cache_dir(set_number, version_number)
full_filename = f"{page_number}.jpg"
thumb_filename = f"{page_number}.jpg"
full_cache_path = full_cache_dir / full_filename
thumb_cache_path = thumb_cache_dir / thumb_filename
# Return existing cached files if they exist
if full_cache_path.exists() and thumb_cache_path.exists():
set_cache_key = f"{set_number}-{version_number}"
thumbnail_url = url_for('static', filename=f'images/peeron_cache/thumbs/{set_cache_key}/{thumb_filename}')
return str(full_cache_path), thumbnail_url
# Download the full-size image using provided session or create new one
if session is None:
session = create_peeron_scraper()
response = session.get(image_url, timeout=30)
if response.status_code == 200 and len(response.content) > 0:
# Validate it's actually an image by checking minimum size
min_size = get_min_image_size()
if len(response.content) < min_size:
logger.warning(f"Image too small, skipping cache: {image_url}")
return None, None
# Write full-size image to cache
with open(full_cache_path, 'wb') as f:
f.write(response.content)
logger.debug(f"Cached full image: {image_url} -> {full_cache_path}")
# Generate thumbnail from the cached full image
try:
from PIL import Image
with Image.open(full_cache_path) as img:
# Create thumbnail (max 150px on longest side to match template)
img.thumbnail((150, 150), Image.Resampling.LANCZOS)
img.save(thumb_cache_path, 'JPEG', quality=85)
logger.debug(f"Generated thumbnail: {full_cache_path} -> {thumb_cache_path}")
set_cache_key = f"{set_number}-{version_number}"
thumbnail_url = url_for('static', filename=f'images/peeron_cache/thumbs/{set_cache_key}/{thumb_filename}')
return str(full_cache_path), thumbnail_url
except Exception as thumb_error:
logger.error(f"Failed to generate thumbnail for {page_number}: {thumb_error}")
# Clean up the full image if thumbnail generation failed
if full_cache_path.exists():
full_cache_path.unlink()
return None, None
else:
logger.warning(f"Failed to download full image: {image_url}")
return None, None
except Exception as e:
logger.error(f"Error caching full image {image_url}: {e}")
return None, None
def clear_set_cache(set_number: str, version_number: str) -> int:
"""
Clear all cached files for a specific set after PDF generation.
Returns the number of files deleted.
"""
try:
full_cache_dir, thumb_cache_dir = get_set_cache_dir(set_number, version_number)
deleted_count = 0
# Delete full images
if full_cache_dir.exists():
for cache_file in full_cache_dir.glob('*.jpg'):
try:
cache_file.unlink()
deleted_count += 1
logger.debug(f"Deleted cached full image: {cache_file}")
except OSError as e:
logger.warning(f"Failed to delete cache file {cache_file}: {e}")
# Remove directory if empty
try:
full_cache_dir.rmdir()
except OSError:
pass # Directory not empty or other error
# Delete thumbnails
if thumb_cache_dir.exists():
for cache_file in thumb_cache_dir.glob('*.jpg'):
try:
cache_file.unlink()
deleted_count += 1
logger.debug(f"Deleted cached thumbnail: {cache_file}")
except OSError as e:
logger.warning(f"Failed to delete cache file {cache_file}: {e}")
# Remove directory if empty
try:
thumb_cache_dir.rmdir()
except OSError:
pass # Directory not empty or other error
# Try to remove set directory if empty
try:
set_cache_key = f"{set_number}-{version_number}"
full_cache_dir.parent.rmdir() if full_cache_dir.parent.name == set_cache_key else None
thumb_cache_dir.parent.rmdir() if thumb_cache_dir.parent.name == set_cache_key else None
except OSError:
pass # Directory not empty or other error
logger.info(f"Set cache cleanup completed for {set_number}-{version_number}: {deleted_count} files deleted")
return deleted_count
except Exception as e:
logger.error(f"Error during set cache cleanup for {set_number}-{version_number}: {e}")
return 0
def clear_old_cache(max_age_days: int = 7) -> int:
"""
Clear old cache files across all sets.
Returns the number of files deleted.
"""
try:
base_cache_dir = get_peeron_cache_dir()
if not base_cache_dir.exists():
return 0
deleted_count = 0
max_age_seconds = max_age_days * 24 * 60 * 60
current_time = time.time()
# Clean both full and thumbs directories
for cache_type in ['full', 'thumbs']:
cache_type_dir = base_cache_dir / cache_type
if cache_type_dir.exists():
for set_dir in cache_type_dir.iterdir():
if set_dir.is_dir():
for cache_file in set_dir.glob('*.jpg'):
file_age = current_time - os.path.getmtime(cache_file)
if file_age > max_age_seconds:
try:
cache_file.unlink()
deleted_count += 1
logger.debug(f"Deleted old cache file: {cache_file}")
except OSError as e:
logger.warning(f"Failed to delete cache file {cache_file}: {e}")
# Remove empty directories
try:
if not any(set_dir.iterdir()):
set_dir.rmdir()
except OSError:
pass
logger.info(f"Old cache cleanup completed: {deleted_count} files deleted")
return deleted_count
except Exception as e:
logger.error(f"Error during old cache cleanup: {e}")
return 0
class PeeronPage(NamedTuple):
"""Represents a single instruction page from Peeron"""
page_number: str
original_image_url: str # Original Peeron full-size image URL
cached_full_image_path: str # Local full-size cached image path
cached_thumbnail_url: str # Local thumbnail URL for preview
alt_text: str
rotation: int = 0 # Rotation in degrees (0, 90, 180, 270)
# Peeron instruction scraper
class PeeronInstructions(object):
socket: 'BrickSocket | None'
set_number: str
version_number: str
pages: list[PeeronPage]
def __init__(
self,
set_number: str,
version_number: str = '1',
/,
*,
socket: 'BrickSocket | None' = None,
):
# Save the socket
self.socket = socket
# Parse set number (handle both "4011" and "4011-1" formats)
if '-' in set_number:
parts = set_number.split('-', 1)
self.set_number = parts[0]
self.version_number = parts[1] if len(parts) > 1 else '1'
else:
self.set_number = set_number
self.version_number = version_number
# Placeholder for pages
self.pages = []
# Check if instructions exist on Peeron (lightweight)
def exists(self, /) -> bool:
"""Check if the set exists on Peeron without caching thumbnails"""
try:
base_url = get_peeron_instruction_url(self.set_number, self.version_number)
scraper = create_peeron_scraper()
response = scraper.get(base_url)
if response.status_code != 200:
return False
soup = BeautifulSoup(response.text, 'html.parser')
# Check for "Browse instruction library" header (set not found)
if soup.find('h1', string="Browse instruction library"):
return False
# Look for thumbnail images to confirm instructions exist
thumbnails = soup.select('table[cellspacing="5"] a img[src^="http://belay.peeron.com/thumbs/"]')
return len(thumbnails) > 0
except Exception:
return False
# Find all available instruction pages on Peeron
def find_pages(self, /) -> list[PeeronPage]:
"""
Scrape Peeron's HTML and return a list of available instruction pages.
Similar to BrickInstructions.find_instructions() but for Peeron.
"""
base_url = get_peeron_instruction_url(self.set_number, self.version_number)
thumb_base_url = get_peeron_thumbnail_url(self.set_number, self.version_number)
scan_base_url = get_peeron_scan_url(self.set_number, self.version_number)
logger.debug(f"[find_pages] fetching HTML from {base_url!r}")
# Set up session with persistent cookies for Peeron (like working dl_peeron.py)
scraper = create_peeron_scraper()
# Download the main HTML page to establish session and cookies
try:
logger.debug(f"[find_pages] Establishing session by visiting: {base_url}")
response = scraper.get(base_url)
logger.debug(f"[find_pages] Main page visit: HTTP {response.status_code}")
if response.status_code != 200:
raise ErrorException(f'Failed to load Peeron page for {self.set_number}-{self.version_number}. HTTP {response.status_code}')
except requests.exceptions.RequestException as e:
raise ErrorException(f'Failed to connect to Peeron: {e}')
# Parse HTML to locate instruction pages
soup = BeautifulSoup(response.text, 'html.parser')
# Check for "Browse instruction library" header (set not found)
if soup.find('h1', string="Browse instruction library"):
raise ErrorException(f'Set {self.set_number}-{self.version_number} not found on Peeron')
# Locate all thumbnail images in the expected table structure
# Use the configured thumbnail pattern to build the expected URL prefix
thumb_base_url = get_peeron_thumbnail_url(self.set_number, self.version_number)
thumbnails = soup.select(f'table[cellspacing="5"] a img[src^="{thumb_base_url}"]')
if not thumbnails:
raise ErrorException(f'No instruction pages found for {self.set_number}-{self.version_number} on Peeron')
pages: list[PeeronPage] = []
total_thumbnails = len(thumbnails)
# Initialize progress if socket is available
if self.socket:
self.socket.progress_total = total_thumbnails
self.socket.progress_count = 0
self.socket.progress(message=f"Starting to cache {total_thumbnails} full images")
for idx, img in enumerate(thumbnails, 1):
thumb_url = img['src']
# Extract the page number from the thumbnail URL
page_number = thumb_url.split('/')[-2]
# Build the full-size scan URL using the page number
full_size_url = f"{scan_base_url}{page_number}/"
logger.debug(f"[find_pages] Page {page_number}: thumb={thumb_url}, full_size={full_size_url}")
# Create alt text for the page
alt_text = f"LEGO Instructions {self.set_number}-{self.version_number} Page {page_number}"
# Report progress if socket is available
if self.socket:
self.socket.progress_count = idx
self.socket.progress(message=f"Caching full image {idx} of {total_thumbnails}")
# Cache the full-size image and generate thumbnail preview using established session
cached_full_path, cached_thumb_url = cache_full_image_and_generate_thumbnail(
full_size_url, page_number, self.set_number, self.version_number, session=scraper
)
# Skip this page if caching failed
if not cached_full_path or not cached_thumb_url:
logger.warning(f"[find_pages] Skipping page {page_number} due to caching failure")
continue
page = PeeronPage(
page_number=page_number,
original_image_url=full_size_url,
cached_full_image_path=cached_full_path,
cached_thumbnail_url=cached_thumb_url,
alt_text=alt_text
)
pages.append(page)
# Cache the pages for later use
self.pages = pages
logger.debug(f"[find_pages] found {len(pages)} pages for {self.set_number}-{self.version_number}")
return pages
# Find instructions with fallback to Peeron
@staticmethod
def find_instructions_with_peeron_fallback(set: str, /) -> tuple[list[tuple[str, str]], list[PeeronPage] | None]:
"""
Enhanced version of BrickInstructions.find_instructions() that falls back to Peeron.
Returns (rebrickable_instructions, peeron_pages).
If rebrickable_instructions is empty, peeron_pages will contain Peeron data.
"""
from .instructions import BrickInstructions
# First try Rebrickable
try:
rebrickable_instructions = BrickInstructions.find_instructions(set)
return rebrickable_instructions, None
except ErrorException as e:
logger.info(f"Rebrickable failed for {set}: {e}. Trying Peeron fallback...")
# Fallback to Peeron
try:
peeron = PeeronInstructions(set)
peeron_pages = peeron.find_pages()
return [], peeron_pages
except ErrorException as peeron_error:
# Both failed, re-raise original Rebrickable error
logger.info(f"Peeron also failed for {set}: {peeron_error}")
raise e from peeron_error
+200
View File
@@ -0,0 +1,200 @@
import logging
import os
import tempfile
import time
from typing import Any, TYPE_CHECKING
import cloudscraper
from flask import current_app
from PIL import Image
from .exceptions import DownloadException, ErrorException
from .instructions import BrickInstructions
from .peeron_instructions import PeeronPage, get_min_image_size, get_peeron_download_delay, get_peeron_instruction_url, create_peeron_scraper
if TYPE_CHECKING:
from .socket import BrickSocket
logger = logging.getLogger(__name__)
# PDF generator for Peeron instruction pages
class PeeronPDF(object):
socket: 'BrickSocket'
set_number: str
version_number: str
pages: list[PeeronPage]
filename: str
def __init__(
self,
set_number: str,
version_number: str,
pages: list[PeeronPage],
/,
*,
socket: 'BrickSocket',
):
# Save the socket
self.socket = socket
# Save set information
self.set_number = set_number
self.version_number = version_number
self.pages = pages
# Generate filename following BrickTracker conventions
self.filename = f"{set_number}-{version_number}_peeron.pdf"
# Download pages and create PDF
def create_pdf(self, /) -> None:
"""
Downloads selected Peeron pages and merges them into a PDF.
Uses progress updates via socket similar to BrickInstructions.download()
"""
try:
target_path = self._get_target_path()
# Skip if we already have it
if os.path.isfile(target_path):
# Create BrickInstructions instance to get PDF URL
instructions = BrickInstructions(self.filename)
pdf_url = instructions.url()
return self.socket.complete(
message=f'File {self.filename} already exists, skipped - <a href="{pdf_url}" target="_blank" class="btn btn-sm btn-primary ms-2"><i class="ri-external-link-line"></i> Open PDF</a>'
)
# Set up progress tracking
total_pages = len(self.pages)
self.socket.update_total(total_pages)
self.socket.progress_count = 0
self.socket.progress(message=f"Starting PDF creation from {total_pages} cached pages")
# Use cached images directly - no downloads needed!
cached_files_with_rotation = []
missing_pages = []
for i, page in enumerate(self.pages):
# Check if cached file exists
if os.path.isfile(page.cached_full_image_path):
cached_files_with_rotation.append((page.cached_full_image_path, page.rotation))
# Update progress
self.socket.progress_count += 1
self.socket.progress(
message=f"Processing cached page {page.page_number} ({i + 1}/{total_pages})"
)
else:
missing_pages.append(page.page_number)
logger.warning(f"Cached image missing for page {page.page_number}: {page.cached_full_image_path}")
if not cached_files_with_rotation:
raise DownloadException(f"No cached images available for set {self.set_number}-{self.version_number}. Cache may have been cleared.")
elif len(cached_files_with_rotation) < total_pages:
# Partial success
error_msg = f"Only found {len(cached_files_with_rotation)}/{total_pages} cached images."
if missing_pages:
error_msg += f" Missing pages: {', '.join(missing_pages)}."
logger.warning(error_msg)
# Create PDF from cached images with rotation
self._create_pdf_from_images(cached_files_with_rotation, target_path)
# Success
logger.info(f"Created PDF {self.filename} with {len(cached_files_with_rotation)} pages")
# Create BrickInstructions instance to get PDF URL
instructions = BrickInstructions(self.filename)
pdf_url = instructions.url()
self.socket.complete(
message=f'PDF {self.filename} created with {len(cached_files_with_rotation)} pages - <a href="{pdf_url}" target="_blank" class="btn btn-sm btn-primary ms-2"><i class="ri-external-link-line"></i> Open PDF</a>'
)
# Clean up set cache after successful PDF creation
try:
from .peeron_instructions import clear_set_cache
deleted_count = clear_set_cache(self.set_number, self.version_number)
if deleted_count > 0:
logger.info(f"[create_pdf] Cleaned up {deleted_count} cache files for set {self.set_number}-{self.version_number}")
except Exception as e:
logger.warning(f"[create_pdf] Failed to clean set cache: {e}")
except Exception as e:
logger.error(f"Error creating PDF {self.filename}: {e}")
self.socket.fail(
message=f"Error creating PDF {self.filename}: {e}"
)
# Create PDF from downloaded images
def _create_pdf_from_images(self, image_paths_and_rotations: list[tuple[str, int]], output_path: str, /) -> None:
"""Create a PDF from a list of image files with their rotations"""
try:
# Import FPDF (should be available from requirements)
from fpdf import FPDF
except ImportError:
raise ErrorException("FPDF library not available. Install with: pip install fpdf2")
pdf = FPDF()
for i, (img_path, rotation) in enumerate(image_paths_and_rotations):
try:
# Open image and apply rotation if needed
with Image.open(img_path) as image:
# Apply rotation if specified
if rotation != 0:
# PIL rotation is counter-clockwise, so we negate for clockwise rotation
image = image.rotate(-rotation, expand=True)
width, height = image.size
# Add page with image dimensions (convert pixels to mm)
# 1 pixel = 0.264583 mm (assuming 96 DPI)
page_width = width * 0.264583
page_height = height * 0.264583
pdf.add_page(format=(page_width, page_height))
# Save rotated image to temporary file for FPDF
temp_rotated_path = None
if rotation != 0:
import tempfile
temp_fd, temp_rotated_path = tempfile.mkstemp(suffix='.jpg', prefix=f'peeron_rotated_{i}_')
try:
os.close(temp_fd) # Close file descriptor, we'll use the path
image.save(temp_rotated_path, 'JPEG', quality=95)
pdf.image(temp_rotated_path, x=0, y=0, w=page_width, h=page_height)
finally:
# Clean up rotated temp file
if temp_rotated_path and os.path.exists(temp_rotated_path):
os.remove(temp_rotated_path)
else:
pdf.image(img_path, x=0, y=0, w=page_width, h=page_height)
# Update progress
progress_msg = f"Processing page {i + 1}/{len(image_paths_and_rotations)} into PDF"
if rotation != 0:
progress_msg += f" (rotated {rotation}°)"
self.socket.progress(message=progress_msg)
except Exception as e:
logger.warning(f"Failed to add image {img_path} to PDF: {e}")
continue
# Save the PDF
pdf.output(output_path)
# Get target file path
def _get_target_path(self, /) -> str:
"""Get the full path where the PDF should be saved"""
instructions_folder = os.path.join(
current_app.static_folder, # type: ignore
current_app.config['INSTRUCTIONS_FOLDER']
)
return os.path.join(instructions_folder, self.filename)
# Create BrickInstructions instance for the generated PDF
def get_instructions(self, /) -> BrickInstructions:
"""Return a BrickInstructions instance for the generated PDF"""
return BrickInstructions(self.filename)
+82
View File
@@ -6,6 +6,8 @@ from flask_socketio import SocketIO
from .instructions import BrickInstructions
from .instructions_list import BrickInstructionsList
from .peeron_instructions import PeeronInstructions, PeeronPage
from .peeron_pdf import PeeronPDF
from .set import BrickSet
from .socket_decorator import authenticated_socket, rebrickable_socket
from .sql import close as sql_close
@@ -18,8 +20,10 @@ MESSAGES: Final[dict[str, str]] = {
'CONNECT': 'connect',
'DISCONNECT': 'disconnect',
'DOWNLOAD_INSTRUCTIONS': 'download_instructions',
'DOWNLOAD_PEERON_PAGES': 'download_peeron_pages',
'FAIL': 'fail',
'IMPORT_SET': 'import_set',
'LOAD_PEERON_PAGES': 'load_peeron_pages',
'LOAD_SET': 'load_set',
'PROGRESS': 'progress',
'SET_LOADED': 'set_loaded',
@@ -106,6 +110,84 @@ class BrickSocket(object):
BrickInstructionsList(force=True)
@self.socket.on(MESSAGES['LOAD_PEERON_PAGES'], namespace=self.namespace) # noqa: E501
def load_peeron_pages(data: dict[str, Any], /) -> None:
logger.debug('Socket: LOAD_PEERON_PAGES={data} (from: {fr})'.format(
data=data, fr=request.remote_addr))
try:
set_number = data.get('set', '')
if not set_number:
self.fail(message="Set number is required")
return
# Create Peeron instructions instance with socket for progress reporting
peeron = PeeronInstructions(set_number, socket=self)
# Find pages (this will report progress for thumbnail caching)
pages = peeron.find_pages()
# Complete the operation (JavaScript will handle redirect)
self.complete(message=f"Found {len(pages)} instruction pages on Peeron")
except Exception as e:
logger.error(f"Error in load_peeron_pages: {e}")
self.fail(message=f"Error loading Peeron pages: {e}")
@self.socket.on(MESSAGES['DOWNLOAD_PEERON_PAGES'], namespace=self.namespace) # noqa: E501
@authenticated_socket(self)
def download_peeron_pages(data: dict[str, Any], /) -> None:
logger.debug('Socket: DOWNLOAD_PEERON_PAGES={data} (from: {fr})'.format(
data=data,
fr=request.sid, # type: ignore
))
try:
# Extract data from the request
set_number = data.get('set', '')
pages_data = data.get('pages', [])
if not set_number:
raise ValueError("Set number is required")
if not pages_data:
raise ValueError("No pages selected")
# Parse set number
if '-' in set_number:
parts = set_number.split('-', 1)
set_num = parts[0]
version_num = parts[1] if len(parts) > 1 else '1'
else:
set_num = set_number
version_num = '1'
# Convert page data to PeeronPage objects
pages = []
for page_data in pages_data:
page = PeeronPage(
page_number=page_data.get('page_number', ''),
original_image_url=page_data.get('original_image_url', ''),
cached_full_image_path=page_data.get('cached_full_image_path', ''),
cached_thumbnail_url='', # Not needed for PDF generation
alt_text=page_data.get('alt_text', ''),
rotation=page_data.get('rotation', 0)
)
pages.append(page)
# Create PDF generator and start download
pdf_generator = PeeronPDF(set_num, version_num, pages, socket=self)
pdf_generator.create_pdf()
# Note: Cache cleanup is handled automatically by pdf_generator.create_pdf()
# Refresh instructions list to include new PDF
BrickInstructionsList(force=True)
except Exception as e:
logger.error(f"Error in download_peeron_pages: {e}")
self.fail(message=f"Error downloading Peeron pages: {e}")
@self.socket.on(MESSAGES['IMPORT_SET'], namespace=self.namespace)
@rebrickable_socket(self)
def import_set(data: dict[str, Any], /) -> None:
+68 -9
View File
@@ -14,6 +14,7 @@ from .exceptions import exception_handler
from ..instructions import BrickInstructions
from ..instructions_list import BrickInstructionsList
from ..parser import parse_set
from ..peeron_instructions import PeeronInstructions
from ..socket import MESSAGES
from .upload import upload_helper
@@ -24,6 +25,22 @@ instructions_page = Blueprint(
)
def _render_peeron_select_page(set: str) -> str:
"""Helper function to render the Peeron page selection interface with cached thumbnails."""
peeron = PeeronInstructions(set)
peeron_pages = peeron.find_pages() # This will use the cached thumbnails
current_app.logger.debug(f"[peeron_loaded] Found {len(peeron_pages)} pages for {set}")
return render_template(
'peeron_select.html',
download=True,
pages=peeron_pages,
set=set,
path=current_app.config['SOCKET_PATH'],
namespace=current_app.config['SOCKET_NAMESPACE'],
messages=MESSAGES
)
# Index
@instructions_page.route('/', methods=['GET'])
@exception_handler(__file__)
@@ -141,6 +158,10 @@ def download() -> str:
except Exception:
set = ''
# Check if this is a redirect after Peeron pages were loaded
if request.args.get('peeron_loaded'):
return _render_peeron_select_page(set)
return render_template(
'instructions.html',
download=True,
@@ -160,12 +181,50 @@ def do_download() -> str:
except Exception:
set = ''
return render_template(
'instructions.html',
download=True,
instructions=BrickInstructions.find_instructions(set),
set=set,
path=current_app.config['SOCKET_PATH'],
namespace=current_app.config['SOCKET_NAMESPACE'],
messages=MESSAGES
)
# Check if this is a redirect after Peeron pages were loaded
if request.args.get('peeron_loaded'):
return _render_peeron_select_page(set)
# Try Rebrickable first
try:
from .instructions import BrickInstructions
rebrickable_instructions = BrickInstructions.find_instructions(set)
# Standard Rebrickable instructions found
return render_template(
'instructions.html',
download=True,
instructions=rebrickable_instructions,
set=set,
path=current_app.config['SOCKET_PATH'],
namespace=current_app.config['SOCKET_NAMESPACE'],
messages=MESSAGES
)
except Exception:
# Rebrickable failed, check if Peeron has instructions (without caching thumbnails yet)
try:
peeron = PeeronInstructions(set)
# Just check if pages exist, don't cache thumbnails yet
if peeron.exists():
# Peeron has instructions - show loading interface
return render_template(
'peeron_select.html',
download=True,
loading_peeron=True, # Flag to show loading state
set=set,
path=current_app.config['SOCKET_PATH'],
namespace=current_app.config['SOCKET_NAMESPACE'],
messages=MESSAGES
)
else:
raise Exception("Not found on Peeron either")
except Exception:
return render_template(
'instructions.html',
download=True,
instructions=[],
set=set,
error='No instructions found on Rebrickable or Peeron',
path=current_app.config['SOCKET_PATH'],
namespace=current_app.config['SOCKET_NAMESPACE'],
messages=MESSAGES
)
+2
View File
@@ -11,3 +11,5 @@ requests
tzdata
bs4
cloudscraper
fpdf2
pillow
+206
View File
@@ -0,0 +1,206 @@
// Peeron Socket class
class BrickPeeronSocket extends BrickSocket {
constructor(id, path, namespace, messages) {
super(id, path, namespace, messages, true);
// Form elements (built based on the initial id)
this.html_button = document.getElementById(id);
this.html_files = document.getElementById(`${id}-files`);
if (this.html_button) {
this.html_button.addEventListener("click", (e) => {
this.execute();
});
}
// Setup select all button
this.setup_select_all_button();
// Setup rotation buttons
this.setup_rotation_buttons();
// Setup the socket
this.setup();
}
setup_select_all_button() {
const selectAllButton = document.getElementById('peeron-select-all');
if (selectAllButton) {
selectAllButton.addEventListener('click', () => {
const checkboxes = this.get_files();
const allChecked = checkboxes.every(cb => cb.checked);
checkboxes.forEach(cb => cb.checked = !allChecked);
// Update button text and icon
if (allChecked) {
selectAllButton.innerHTML = '<i class="ri-checkbox-multiple-line"></i> Select All';
} else {
selectAllButton.innerHTML = '<i class="ri-checkbox-blank-line"></i> Deselect All';
}
});
}
}
setup_rotation_buttons() {
document.querySelectorAll('.peeron-rotate-btn').forEach(button => {
button.addEventListener('click', (e) => {
e.preventDefault(); // Prevent label click
e.stopPropagation(); // Stop event bubbling
const targetId = button.getAttribute('data-target');
const checkboxId = button.getAttribute('data-checkbox');
const targetImg = document.getElementById(targetId);
const checkbox = document.getElementById(checkboxId);
if (targetImg && checkbox) {
let currentRotation = parseInt(button.getAttribute('data-rotation') || '0');
currentRotation = (currentRotation + 90) % 360;
// Update image rotation
targetImg.style.transform = `rotate(${currentRotation}deg)`;
button.setAttribute('data-rotation', currentRotation.toString());
// Store rotation in checkbox data for later use
checkbox.setAttribute('data-rotation', currentRotation.toString());
// Update the rotation icon to indicate current state
const icon = button.querySelector('i');
if (icon) {
// Rotate the icon to match the image rotation
icon.style.transform = `rotate(${currentRotation}deg)`;
}
}
});
});
}
// Upon receiving a complete message
complete(data) {
super.complete(data);
// Clear progress display after completion
if (this.html_progress_message) {
this.html_progress_message.classList.add("d-none");
this.html_progress_message.textContent = "";
}
if (this.html_count) {
this.html_count.classList.add("d-none");
this.html_count.textContent = "";
}
// Ensure spinner is hidden
this.spinner(false);
this.toggle(true);
}
// Execute the action
execute() {
if (!this.disabled && this.socket !== undefined && this.socket.connected) {
this.toggle(false);
this.download_peeron_pages();
}
}
// Get the list of checkboxes describing files
get_files(checked=false) {
let files = [];
if (this.html_files) {
files = [...this.html_files.querySelectorAll('input[type="checkbox"]')];
if (checked) {
files = files.filter(file => file.checked);
}
}
return files;
}
// Download Peeron pages
download_peeron_pages() {
if (this.html_files) {
const selectedFiles = this.get_files(true);
if (selectedFiles.length === 0) {
this.fail({message: "Please select at least one page to download."});
this.toggle(true);
return;
}
const pages = selectedFiles.map(checkbox => ({
page_number: checkbox.getAttribute('data-page-number'),
original_image_url: checkbox.getAttribute('data-original-image-url'),
cached_full_image_path: checkbox.getAttribute('data-cached-full-image-path'),
alt_text: checkbox.getAttribute('data-alt-text'),
rotation: parseInt(checkbox.getAttribute('data-rotation') || '0')
}));
this.clear();
this.spinner(true);
const setElement = document.querySelector('input[name="download-set"]');
const set = setElement ? setElement.value : '';
this.socket.emit(this.messages.DOWNLOAD_PEERON_PAGES, {
set: set,
pages: pages,
total: pages.length,
current: 0
});
} else {
this.fail({message: "Could not find the list of pages to download"});
}
}
// Toggle clicking on the button, or sending events
toggle(enabled) {
super.toggle(enabled);
if (this.html_files) {
this.get_files().forEach(el => el.disabled = !enabled);
}
if (this.html_button) {
this.html_button.disabled = !enabled;
}
}
}
// Simple Peeron page loader using standard socket pattern
class BrickPeeronPageLoader extends BrickSocket {
constructor(set, path, namespace, messages) {
// Use 'peeron-loader' as the ID for socket elements
super('peeron-loader', path, namespace, messages, false);
this.set = set;
this.setup();
// Auto-start loading when connected
setTimeout(() => {
if (this.socket && this.socket.connected) {
this.loadPages();
} else {
this.socket.on('connect', () => this.loadPages());
}
}, 100);
}
loadPages() {
this.socket.emit(this.messages.LOAD_PEERON_PAGES, {
set: this.set
});
}
// Override complete to redirect when done
complete(data) {
super.complete(data);
// Redirect to show the pages selection interface
const params = new URLSearchParams();
params.set('set', this.set);
params.set('peeron_loaded', '1');
window.location.href = `${window.location.pathname}?${params.toString()}`;
}
}
+3
View File
@@ -105,6 +105,9 @@
{% if request.endpoint == 'set.list' %}
<script src="{{ url_for('static', filename='scripts/sets.js') }}"></script>
{% endif %}
{% if request.endpoint == 'instructions.download' or request.endpoint == 'instructions.do_download' %}
<script src="{{ url_for('static', filename='scripts/socket/peeron.js') }}"></script>
{% endif %}
<script type="text/javascript">
document.addEventListener("DOMContentLoaded", () => {
setup_grids();
+1 -1
View File
@@ -16,7 +16,7 @@
{% if g.login.is_authenticated() %}
<p class="border-bottom pb-2 px-2 text-center">
<a class="btn btn-primary" href="{{ url_for('instructions.upload') }}"><i class="ri-upload-line"></i> Upload an instructions file</a>
<a class="btn btn-primary" href="{{ url_for('instructions.download') }}"><i class="ri-download-line"></i> Download instructions from Rebrickable</a>
<a class="btn btn-primary" href="{{ url_for('instructions.download') }}"><i class="ri-download-line"></i> Download instructions</a>
<a href="{{ url_for('admin.admin', open_instructions=true) }}" class="btn btn-light border" role="button"><i class="ri-refresh-line"></i> Refresh the instructions cache</a>
</p>
{% endif %}
+1 -1
View File
@@ -5,7 +5,7 @@
<form method="POST" action="{{ url_for('instructions.do_download') }}">
<div class="card mb-3">
<div class="card-header">
<h5 class="mb-0"><i class="ri-download-line"></i> Download instructions from Rebrickable</h5>
<h5 class="mb-0"><i class="ri-download-line"></i> Download instructions</h5>
</div>
<div class="card-body">
<div class="mb-3">
@@ -0,0 +1,15 @@
<script type="text/javascript">
document.addEventListener("DOMContentLoaded", () => {
new BrickPeeronPageLoader(
'{{ set }}',
'{{ path }}',
'{{ namespace }}',
{
COMPLETE: '{{ messages['COMPLETE'] }}',
FAIL: '{{ messages['FAIL'] }}',
LOAD_PEERON_PAGES: '{{ messages['LOAD_PEERON_PAGES'] }}',
PROGRESS: '{{ messages['PROGRESS'] }}',
}
);
});
</script>
+10
View File
@@ -0,0 +1,10 @@
<script type="text/javascript">
document.addEventListener("DOMContentLoaded", () => {
new BrickPeeronSocket('peeron-download', '{{ path }}', '{{ namespace }}', {
COMPLETE: '{{ messages['COMPLETE'] }}',
DOWNLOAD_PEERON_PAGES: 'download_peeron_pages',
FAIL: '{{ messages['FAIL'] }}',
PROGRESS: '{{ messages['PROGRESS'] }}',
});
});
</script>
+144
View File
@@ -0,0 +1,144 @@
{% extends 'base.html' %}
{% block title %} - Download instructions from Peeron{% endblock %}
{% block main %}
<div class="container">
{% if error %}<div class="alert alert-danger" role="alert"><strong>Error:</strong> {{ error }}.</div>{% endif %}
<div class="row">
<div class="col-12">
<form method="POST" action="{{ url_for('instructions.do_download') }}">
<div class="card mb-3">
<div class="card-header">
<h5 class="mb-0"><i class="ri-download-line"></i> Download instructions from Rebrickable</h5>
</div>
<div class="card-body">
<div class="mb-3">
<label for="download-set" class="form-label">Set number (only one)</label>
<input type="text" class="form-control" id="download-set" name="download-set" placeholder="107-1 or 1642-1 or ..." value="{{ set }}">
</div>
</div>
<div class="card-footer text-end">
<button type="submit" class="btn btn-primary"><i class="ri-search-line"></i> Search</button>
</div>
</div>
</form>
{% if loading_peeron %}
<div class="alert alert-info" role="alert">
<i class="ri-information-line"></i> <strong>Found on Peeron:</strong> {{ set }} was not available on Rebrickable, loading instruction pages from Peeron...
</div>
<!-- Socket elements for peeron-loader -->
<div id="peeron-loader-fail" class="alert alert-danger d-none" role="alert"></div>
<div id="peeron-loader-complete" class="alert alert-success d-none" role="alert"></div>
<div class="mb-3">
<p>
Progress <span id="peeron-loader-count"></span>
<span id="peeron-loader-spinner" class="d-none">
<span class="spinner-border spinner-border-sm" aria-hidden="true"></span>
<span class="visually-hidden" role="status">Loading...</span>
</span>
</p>
<div id="peeron-loader-progress" class="progress" role="progressbar" aria-label="Loading Peeron pages" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">
<div id="peeron-loader-progress-bar" class="progress-bar" style="width: 0%"></div>
</div>
<p id="peeron-loader-progress-message" class="text-center d-none"></p>
</div>
{% endif %}
{% if loading_peeron %}
<!-- Include socket for automatic loading -->
{% with set=set, path=path, namespace=namespace, messages=messages %}
{% include 'instructions/peeron_loader_socket.html' %}
{% endwith %}
{% endif %}
{% if pages %}
<div id="peeron-loading-alert" class="alert alert-info" role="alert">
<i class="ri-information-line"></i> <strong>Instructions found on Peeron:</strong> {{ set }} was not available on Rebrickable, but {{ pages|length }} instruction pages were found on Peeron.
<div id="peeron-cache-progress" class="mt-2 d-none">
<div class="progress" role="progressbar" aria-label="Caching thumbnails" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">
<div id="peeron-cache-progress-bar" class="progress-bar" style="width: 0%"></div>
</div>
<small id="peeron-cache-message" class="text-muted">Caching thumbnails...</small>
</div>
</div>
<div class="card mb-3">
<div class="card-header">
<h5 class="mb-0"><i class="ri-checkbox-line"></i> Select instructions to download</h5>
</div>
<div class="card-body">
<div class="mb-3">
<div id="peeron-download-fail" class="alert alert-danger d-none" role="alert"></div>
<div id="peeron-download-complete"></div>
<div class="d-flex justify-content-between align-items-center border-bottom mb-3">
<h5 class="mb-0">Available Instructions</h5>
<button id="peeron-select-all" type="button" class="btn btn-sm btn-outline-secondary">
<i class="ri-checkbox-multiple-line"></i> Select All
</button>
</div>
<div id="peeron-download-files" class="row g-2">
{% for page in pages %}
<div class="col-12 col-md-6 col-lg-4">
<div class="card border-0 shadow-sm">
<div class="card-body p-2">
<div class="form-check">
<input class="form-check-input" type="checkbox" id="peeron-page-{{ loop.index }}"
data-page-number="{{ page.page_number }}"
data-original-image-url="{{ page.original_image_url }}"
data-cached-full-image-path="{{ page.cached_full_image_path }}"
data-alt-text="{{ page.alt_text }}"
data-rotation="0"
autocomplete="off">
<label class="form-check-label w-100" for="peeron-page-{{ loop.index }}">
<div class="text-center position-relative">
<div class="position-relative d-inline-block">
<img id="peeron-img-{{ loop.index }}" src="{{ page.cached_thumbnail_url }}" alt="{{ page.alt_text }}"
class="img-fluid mb-2 border rounded peeron-thumbnail" style="max-height: 150px; transform: rotate(0deg); transition: transform 0.3s ease;"
data-index="{{ loop.index }}" data-total="{{ pages|length }}">
<button type="button" class="btn btn-sm btn-light position-absolute top-0 end-0 p-1 me-1 mt-1 peeron-rotate-btn"
data-target="peeron-img-{{ loop.index }}" data-checkbox="peeron-page-{{ loop.index }}" data-rotation="0"
title="Rotate page" style="font-size: 0.7rem; line-height: 1;">
<i class="ri-refresh-line"></i>
</button>
</div>
<div class="small fw-bold">Page {{ page.page_number }}</div>
</div>
</label>
</div>
</div>
</div>
</div>
{% endfor %}
</div>
</div>
<hr>
<div class="mb-3">
<p>
Progress <span id="peeron-download-count"></span>
<span id="peeron-download-spinner" class="d-none">
<span class="spinner-border spinner-border-sm" aria-hidden="true"></span>
<span class="visually-hidden" role="status">Loading...</span>
</span>
</p>
<div id="peeron-download-progress" class="progress" role="progressbar" aria-label="Download Peeron instructions progress" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">
<div id="peeron-download-progress-bar" class="progress-bar" style="width: 0%"></div>
</div>
<p id="peeron-download-progress-message" class="text-center d-none"></p>
</div>
</div>
<div class="card-footer text-end">
<span id="peeron-download-status-icon" class="me-1"></span><span id="peeron-download-status" class="me-1"></span><button id="peeron-download" type="button" class="btn btn-primary"><i class="ri-download-line"></i> Download selected files</button>
</div>
</div>
{% if not loading_peeron %}
<!-- Include normal socket for downloading -->
{% include 'instructions/peeron_socket.html' %}
{% endif %}
{% endif %}
</div>
</div>
</div>
{% endblock %}