config.template.yaml

# Configuration template for Curator
# Copy this to local/config/config.yaml and update with your settings

# ==============================================================================
# Search Providers
# ==============================================================================
# Providers used to search for and download periodical issues
# You can configure multiple providers (Newsnab indexers, RSS feeds)

search_providers:
  # ==============================================================================
  # Internet Archive Provider (FREE - No API key required!)
  # ==============================================================================
  # Archive.org has millions of free magazines, periodicals, newspapers, and comics.
  # Results from Internet Archive are prioritized when available (priority: 10).
  # Downloads are direct (no NZB client needed).

  - type: internet_archive
    name: Internet Archive
    enabled: true
    priority: 1 # Lower = higher priority (searched first, preferred in dedup)

    # Collections to search (defaults to common periodical collections)
    # Available collections: magazines, periodicals, americana, newspaper, comics, pulpmagazinearchive
    collections:
      - magazines
      - periodicals
      - americana
      - comics

    # Preferred file formats for download (in order of preference)
    # PDF is most common for scanned magazines; EPUB for modern digital publications
    # ZIP/GZIP for collection archives containing multiple files
    file_formats:
      - PDF
      - EPUB
      - ZIP
      - GZIP

    # Max results per search (default: 500)
    # Higher values ensure comprehensive coverage of large collections.
    max_results: 500

    # Rate limiting (archive.org is generally permissive)
    request_delay: 1.0 # Seconds between requests
    max_requests_per_minute: 15

    # Optional authentication for restricted items (uncomment to enable)
    # Some items on archive.org require login to download.
    # Create a free account at https://archive.org/account/signup
    # username: your-email@example.com
    # password: your-archive-org-password

  # ==============================================================================
  # Newsnab Provider (e.g., NZBHydra2, NZBGeek, etc.)
  # ==============================================================================
  - type: newsnab
    name: Newsnab Provider
    api_url: https://your-newsnab-server.com/api
    api_key: your-api-key-here
    enabled: true
    priority: 50 # Searched after Internet Archive

    # Newsnab categories to search
    # 7000 = Other Misc, 7010 = Magazines, 7020 = Ebooks, 7030 = Comics
    # 6000 = Adult, 8000 = Other (varies by indexer)
    categories: '7000,7010,7020,7030'

    # Maximum results per API search query (default: 250)
    # Many indexers default to 25-50 results without an explicit limit.
    # Increase this to find older releases and collection items.
    search_limit: 250

    # Rate limiting settings (to avoid provider bans)
    # NOTE: With provider cache enabled (see cache section below), these limits become
    # much less important since searches use local cache instead of hitting providers.
    # Cache sync makes only ~4-8 API calls/hour (RSS mode), far below typical limits.
    # You can safely increase these limits when cache is enabled.
    max_requests_per_hour: 100 # Self-imposed limit (default: 100 requests/hour)
    request_delay_seconds: 1.0 # Delay between requests (default: 1 second)

  # RSS Provider (optional - for fast new release discovery)
  # Uncomment and configure to enable RSS-based searching
  # - type: rss
  #   name: RSS Feed Provider
  #   feed_url: https://your-indexer.com/rss
  #   enabled: true
  #   priority: 50
  #
  #   # RSS caching settings (dramatically speeds up searches)
  #   cache_ttl: 3600 # Cache TTL in seconds (1 hour)
  #   enable_cache: true # Enable/disable caching

  # Torznab Provider (optional - for torrent-based discovery via Prowlarr/Jackett)
  # Requires a qBittorrent download client configured in download_clients below.
  # Uncomment and configure to enable torrent support via any Torznab-compatible indexer.
  # - type: torznab
  #   name: Prowlarr
  #   api_url: http://localhost:9696/1/api   # Prowlarr indexer URL (append /api to the indexer path)
  #   api_key: your-prowlarr-api-key
  #   enabled: true
  #   priority: 50
  #   categories: '7010,7020,7030'   # 7010=Magazines, 7020=Ebooks, 7030=Comics
  #   search_limit: 100

# ==============================================================================
# Download Clients
# ==============================================================================
# Downloads are automatically routed based on provider type:
# - Newsnab/RSS providers → NZB client (SABnzbd or NZBGet)
# - Internet Archive providers → Internet Archive built-in HTTP client
# - Torznab providers → qBittorrent client
#
# All clients are configured as a unified list. Multiple clients of any type
# are supported. The first enabled client of the appropriate type is used.

download_clients:
  # SABnzbd — NZB client for Newsnab and RSS provider downloads
  - type: sabnzbd
    name: SABnzbd
    enabled: true
    api_url: http://localhost:8080
    api_key: your-sabnzbd-api-key
    default_category: books
    # remote_path: /downloads/  # Path prefix as seen by the download client.
    # Set this when the client runs in a different container with different mount points.
    # Client paths starting with this prefix are remapped to storage.download_dir.
    # Example: SABnzbd saves to /downloads/Books/file.pdf, Curator sees /app/local/downloads/Books/file.pdf
    #   remote_path: /downloads/

  # Internet Archive HTTP client (used automatically for IA provider downloads)
  # This is a built-in HTTP download client — no external software needed.
  - type: internet_archive
    name: Internet Archive
    enabled: true
    downloads_dir: ./local/downloads # Files download here before import
    max_concurrent: 3 # Max simultaneous downloads from archive.org
    file_formats: # Preferred formats (in order)
      - PDF
      - EPUB

  # qBittorrent — torrent client for Torznab provider downloads
  # Uncomment and configure to enable torrent support
  # - type: qbittorrent
  #   name: qBittorrent
  #   enabled: true
  #   api_url: http://localhost:8090
  #   username: admin
  #   password: adminadmin
  #   default_category: curator   # Torrent category/label used to identify curator downloads

storage:
  db_path: ./local/config/periodicals.db
  download_dir: ./local/downloads
  library_dir: ./local/data
  cache_dir: ./local/cache

# ==============================================================================
# NZB Content Cache Configuration
# ==============================================================================
# Caches NZB file content fetched from providers to avoid repeated hits
# on download retries and resubmissions.
#
# How it works:
# - When a download is submitted, the NZB file is fetched once and cached
# - Subsequent downloads/retries use the cached NZB content directly
# - NZB content is sent directly to SABnzbd/NZBGet (no provider hit)
# - Prowlarr proxy URLs are stored as-is (Prowlarr handles resolution)
#
# Benefits:
# - Eliminates rate limit errors from NZB providers on retries
# - Download retries never hit the provider again

cache:
  # Enable/disable NZB content caching
  enabled: true

  # NZB fetch rate limiting
  # Maximum number of NZB file fetches per hour
  # Applies when downloading NZB content for caching
  # Set to 0 to disable rate limiting
  max_nzb_fetches_per_hour: 50

import:
  # Organization pattern for file structure
  # Available patterns (can also use custom pattern with {tags}):
  #   '{category}/{title}/{year}/' - Default: organized by year (requires date)
  #   '{category}/{title}/Vol{volume}/' - Volume-based (for series with volumes but no dates)
  #   '{category}/{title}/' - Flat structure (all issues in title folder)
  #   '{category}/{title}/Vol{volume}/{year}/' - Hybrid volume-year structure
  #
  # Available tags: {category}, {title}, {year}, {month}, {day}, {language}, {volume}, {issue}
  #
  # Smart auto-selection: If pattern is not specified or uses default, the system will
  # automatically select the best pattern based on available metadata:
  #   - Has date → uses year-based pattern
  #   - Has volume but no date → uses volume-based pattern
  #   - Has issue but no date → uses flat pattern
  #   - No metadata → uses flat pattern with warning
  organization_pattern: '{category}/{title}/{year}/'
  auto_track_imports: true
  enable_text_scan: true # Enable direct text extraction from PDF/EPUB during import (fast)
  enable_ocr: true # Enable OCR for image-based text extraction (slower, queued)
  category_prefix: _

  # Auto-cleanup settings
  # Controls automatic removal of empty folders and folders without importable files
  # WARNING: If disabled, folders may accumulate over time. Manual cleanup may be required.
  auto_cleanup:
    # Enable auto-cleanup of downloads folder
    # When enabled, automatically removes folders without supported files (.pdf, .epub, .cbz, .cbr)
    # Disable this if downloads are being removed before they are processed/imported
    enable_downloads: false

    # Enable auto-cleanup of library folder
    # When enabled, automatically removes empty folders after file reorganization
    # Generally safe to keep enabled as it only removes truly empty folders
    enable_library: true

metadata:
  # Metadata aggregation settings - controls how metadata from different sources is combined
  # When multiple sources provide conflicting metadata (e.g. OCR says 2024, filename says 2023),
  # this configuration determines which source wins based on priority order and confidence scores.

  # Source priority order for metadata aggregation (first = highest priority)
  # Available sources:
  #   - ocr: Optical Character Recognition from cover images (has per-field confidence scores)
  #   - text_scan: Direct text extraction from native PDFs/EPUBs (always 100% confidence)
  #   - filename: Metadata parsed from the filename (always 100% confidence)
  # Default: OCR first (most accurate for scanned magazines), then text_scan, then filename
  source_priority:
    - ocr
    - text_scan
    - filename

  # Minimum confidence thresholds (0-100) for each source
  # Source must meet or exceed threshold for its value to be used
  # If a source's confidence is below threshold, the next source in priority order is tried
  confidence_thresholds:
    ocr: 70 # OCR must be 70%+ confident (has per-field confidence scores)
    text_scan: 50 # Text scan usually high quality (no confidence scores, treated as 100%)
    filename: 0 # Always accept filename parsing (no confidence scores, treated as 100%)

  # Per-field confidence overrides (optional)
  # Allows fine-tuning thresholds for specific metadata fields
  # If not specified, uses the general source threshold from confidence_thresholds above
  field_overrides:
    year:
      ocr: 80 # Years are critical - require higher confidence from OCR
    month:
      ocr: 60 # Month names are easier to OCR - lower threshold is acceptable
    issue_number:
      ocr: 75 # Issue numbers are moderately important
    volume:
      ocr: 75 # Volume numbers are moderately important
    # special_edition: No override, uses general ocr threshold (70%)

  # Notes:
  # - If this metadata section is not present, system uses OCR-first defaults shown above
  # - Each metadata field (year, month, issue_number, etc.) is evaluated independently
  # - Different fields can come from different sources based on which source passes first
  # - To revert to old behavior (filename always wins), use:
  #     source_priority: [filename, text_scan, ocr]
  #     confidence_thresholds: {filename: 0, text_scan: 0, ocr: 0}

matching:
  # Fuzzy matching threshold (0-100) for duplicate detection
  # Higher = stricter matching, Lower = more lenient
  fuzzy_threshold: 80

  # Days threshold for considering publications as duplicates
  # Publications with same title within this many days are considered duplicates
  duplicate_date_threshold_days: 5

pdf:
  # DPI settings for cover extraction (higher = better quality, slower)
  cover_dpi_low: 60 # For thumbnails/previews
  cover_dpi_high: 200 # For high-quality covers

  # JPEG quality (1-100) for cover images
  cover_quality_low: 50 # For thumbnails
  cover_quality_high: 85 # For high-quality covers

downloads:
  # Maximum number of retry attempts for failed downloads
  # Note: Currently not configurable via YAML - hardcoded in core/constants/app.py
  max_retries: 1

  # Maximum number of concurrent downloads allowed system-wide
  # This is the total limit across all tracked periodicals
  # With provider cache enabled, API rate limits are no longer a concern
  # (cache sync uses minimal API calls), so this can be safely increased
  max_concurrent: 30

# OCR (Optical Character Recognition) Configuration
# Controls image preprocessing for OCR text extraction
ocr:
  # Image resize width for OCR processing (pixels)
  # Smaller = faster processing, larger = better accuracy
  # Default: 1200 pixels balances speed and quality
  resize_width: 1200

  # Contrast enhancement factor
  # Higher values increase contrast for better text recognition
  # Range: 1.0-2.0, Default: 1.5
  contrast_enhance: 1.5

  # Denoise strength (h parameter for fastNlMeansDenoising)
  # Higher values remove more noise but may blur text
  # Range: 5-15, Default: 10
  denoise_h: 10

  # Sharpening kernel size (must be odd number)
  # Larger values create stronger sharpening effect
  # Common values: 3, 5, 7. Default: 5
  sharpen_kernel: 5

  # Tesseract Page Segmentation Mode (PSM)
  # Controls how Tesseract segments text on the page
  # Common values:
  #   6 = Uniform block of text (good for organized layouts)
  #   11 = Sparse text with no OSD (best for scattered magazine cover elements)
  # Default: 11 (sparse text mode)
  tesseract_psm: 11

  # Tesseract OCR Engine Mode (OEM)
  # Controls which OCR engine to use
  # Values:
  #   0 = Legacy engine only
  #   1 = LSTM neural network only (recommended - faster and more accurate)
  #   2 = Legacy + LSTM
  #   3 = Default (uses whichever is available)
  # Default: 1 (LSTM only)
  tesseract_oem: 1

tasks:
  # Task interval settings (in seconds)
  # How often each background task runs

  # Task enable/disable flags
  # Set to false to disable a task entirely. Disabled tasks will not run
  # on their schedule but can still be triggered manually from the UI.
  feed_sync_enabled: true
  auto_download_enabled: true
  download_monitor_enabled: true
  cleanup_covers_enabled: true
  ocr_processor_enabled: true
  folder_cleanup_enabled: true
  auto_metadata_enabled: true

  # ==============================================================================
  # Feed Sync: Cache-first auto-download system
  # Polls each provider's RSS feed on a lightweight schedule (one HTTP GET per
  # provider per interval). Entries are cached locally and matched against ALL
  # tracked periodicals with zero additional API calls.
  # ==============================================================================

  # How often to poll provider RSS feeds (in seconds)
  # This is lightweight (one GET per provider), so it can run frequently.
  # Recommended: 900 (15 min) - catches new uploads quickly without hammering APIs
  feed_sync_interval: 900 # 15 minutes

  # Maximum feed entries to match per auto-download run
  # Controls how many cached entries are processed at once.
  # Higher = faster catch-up after downtime, but more DB work per cycle.
  # Recommended: 100-200
  feed_sync_match_batch_size: 200

  # Days to retain feed entries before expiring them
  # Entries older than this are deleted. Bounds the working set.
  feed_entry_retention_days: 7

  # Auto-download: Search for and download new tracked issues
  # IMPORTANT: With the cache-first system, most discovery happens via feed sync
  # (zero API calls). This task now primarily:
  # 1. Matches cached feed entries against tracked periodicals (local, instant)
  # 2. Falls back to per-periodical API search for adaptive scheduler items
  # 3. Processes the download queue
  #
  # Recommended: 1800-3600 (30 min - 1 hour)
  auto_download_interval: 1800 # 30 minutes (recommended with new system)

  # Delay (in seconds) between per-periodical API searches during auto-download.
  # Spreads out API calls to avoid hitting provider rate limits when tracking
  # many periodicals. Set to 0 to disable pacing.
  # Recommended: 3-10 seconds depending on number of tracked periodicals
  inter_search_delay: 5

  # Download monitor: Check download status and import completed files
  download_monitor_interval: 30 # 30 seconds

  # Cover cleanup: Remove orphaned covers and generate missing ones
  cleanup_covers_interval: 86400 # 24 hours

  # OCR cover generator: Generate high-res cover PNGs for OCR processing
  # Note: This task runs as part of the import process, interval not currently used
  # ocr_cover_generator_interval: 300 # 5 minutes

  # OCR processor: Run OCR text extraction on queued covers
  ocr_processor_interval: 10 # 10 seconds

  # OCR worker settings
  ocr_max_workers: 1 # Number of parallel OCR processes
  ocr_batch_size: 5 # Max jobs to process per batch

  # Folder cleanup: Remove empty folders and folders without importable files
  # See import.auto_cleanup section for enable/disable settings
  folder_cleanup_interval: 86400 # 24 hours

  # ==============================================================================
  # Issue Discovery & Tracking System
  # Controls adaptive search scheduling and issue discovery behavior
  # ==============================================================================

  # Maximum number of periodicals to search per auto_download_task run
  # With provider cache enabled, this can be much higher without overwhelming providers
  # Higher = faster discovery of new issues across your tracked periodicals
  # Recommended: 5-10 with cache enabled (was 1-2 without cache)
  max_periodicals_per_search: 10

  # Search interval settings (in hours)
  # The system adapts search frequency based on discovery success:
  # - Finds new issues → rapid interval (searches more often)
  # - No new issues for 1-2 searches → normal interval
  # - No new issues for 3-5 searches → slow interval
  # - No new issues for 6+ searches → very slow interval
  #
  # With provider cache, these can be more aggressive since searches are local
  rapid_search_interval: 0.5 # Search every 30 minutes when finding new issues
  normal_search_interval: 2 # Search every 2 hours by default
  slow_search_interval: 12 # Search twice daily when no new issues
  very_slow_search_interval: 48 # Search every 2 days when very quiet

  # Note: empty_search_threshold is hardcoded in SearchScheduler (currently 3)
  # After 3 consecutive empty searches, interval increases to slow
  # After 6 consecutive empty searches, interval increases to very_slow

logging:
  level: INFO
  log_file: ./local/logs/periodical_manager.log

server:
  host: 0.0.0.0
  port: 8000

# CORS allowed origins
# Controls which browser origins can make requests to the API.
# Default: ["*"] (all origins) — RESTRICT THIS IN PRODUCTION.
#
# Examples:
#   cors_origins: ["http://localhost:8000"]
#   cors_origins: ["https://curator.myserver.com", "http://192.168.1.10:8000"]
#
# Can also be set via environment variable (comma-separated):
#   CURATOR_CORS_ORIGINS=https://curator.myserver.com,http://192.168.1.10:8000
cors_origins: ["*"]

# JWT Authentication Secret
# Used for signing JWT tokens for user authentication
# If not specified, a secret will be auto-generated on first run and saved to config
# IMPORTANT: Keep this secret! Anyone with this can create valid authentication tokens
jwt_secret: null # Auto-generated if null/empty


# ==============================================================================
# Advanced Configuration Notes
# ==============================================================================

# Environment Variable Overrides:
# The following environment variables can override config file settings:
#
# Storage paths:
#   CURATOR_CONFIG_PATH - Path to config file (default: local/config/config.yaml)
#   CURATOR_DB_PATH - Database file path
#   CURATOR_DOWNLOAD_DIR - Download directory
#   CURATOR_LIBRARY_DIR - Organization directory
#   CURATOR_CACHE_DIR - Cache directory
#
# Logging:
#   CURATOR_LOG_FILE - Log file path
#   CURATOR_LOG_LEVEL - Log level (DEBUG, INFO, WARNING, ERROR)
#
# Server:
#   CURATOR_HOST - Server host (default: 0.0.0.0)
#   CURATOR_PORT - Server port (default: 8000)
#
# CORS / Security:
#   CURATOR_CORS_ORIGINS - Comma-separated allowed origins (e.g. "https://curator.example.com")
#
# Credentials (keep API keys out of config.yaml by using env vars instead):
#   CURATOR_NEWSNAB_API_KEY - Newsnab/Prowlarr provider api_key
#   CURATOR_TORZNAB_API_KEY - Torznab provider api_key
#   CURATOR_SABNZBD_API_KEY - SABnzbd api_key
#   CURATOR_NZBGET_PASSWORD - NZBGet password
#   CURATOR_QBITTORRENT_PASSWORD - qBittorrent password
#
# Special modes:
#   CURATOR_DRY_RUN - Set to "true" for dry run mode (no file moves)
#   DISABLE_OCR - Set to "true" to disable OCR features
#   USE_GPU - Set to "0" for CPU-only mode (default)