Prototyp

app/extractors/base.py

@@ -0,0 +1,22 @@
+"""Platform detection: decides whether a URL is a video or a website."""
+import re
+
+VIDEO_PATTERNS = [
+    r"youtube\.com/watch",
+    r"youtu\.be/",
+    r"tiktok\.com/",
+    r"instagram\.com/reel",
+    r"instagram\.com/p/",
+    r"vimeo\.com/",
+    r"twitch\.tv/",
+    r"twitter\.com/.+/status",
+    r"x\.com/.+/status",
+    r"facebook\.com/.*video",
+    r"reddit\.com/r/.*/comments",  # reddit video posts
+]
+
+_VIDEO_RE = re.compile("|".join(VIDEO_PATTERNS), re.IGNORECASE)
+
+
+def is_video_url(url: str) -> bool:
+    return bool(_VIDEO_RE.search(url))

app/extractors/video.py

@@ -0,0 +1,206 @@
+"""Video recipe extraction: yt-dlp → caption check → Whisper → GPT-4."""
+from __future__ import annotations
+import asyncio
+import json
+import logging
+import os
+import uuid
+from pathlib import Path
+from typing import AsyncGenerator
+
+from app.config import get_settings
+from app.models import Recipe, StatusEvent
+from app.services.openai_service import (
+    check_caption_has_recipe,
+    extract_recipe_from_text,
+    transcribe_audio,
+)
+
+log = logging.getLogger(__name__)
+
+MAX_DURATION_WARN = 20 * 60  # 20 minutes in seconds
+MAX_FILESIZE = "50m"
+WHISPER_MAX_BYTES = 24 * 1024 * 1024  # 25MB limit, buffer to 24MB
+
+
+async def extract_video(url: str) -> AsyncGenerator[StatusEvent | Recipe, None]:
+    yield StatusEvent(stage="detecting", message="Erkenne Plattform...", progress=5)
+
+    # Step 1: Get metadata without downloading
+    yield StatusEvent(stage="metadata", message="Lade Video-Metadaten...", progress=10)
+    metadata = await _get_metadata(url)
+
+    title = metadata.get("title", "")
+    description = metadata.get("description", "") or ""
+    tags = metadata.get("tags") or []
+    duration = metadata.get("duration") or 0
+
+    # Step 2: Check if caption/description contains a complete recipe
+    yield StatusEvent(stage="caption", message="Prüfe Video-Beschreibung...", progress=20)
+
+    caption_text = f"{title}\n\n{description}"
+    has_recipe_in_caption = await check_caption_has_recipe(caption_text)
+
+    if has_recipe_in_caption:
+        yield StatusEvent(stage="extracting", message="Extrahiere Rezept aus Beschreibung...", progress=50)
+        enriched_text = f"Titel: {title}\n\nBeschreibung:\n{description}"
+        recipe = await extract_recipe_from_text(enriched_text, url)
+        if recipe.image_url is None:
+            recipe.image_url = _get_thumbnail(metadata)
+        if not recipe.tags and tags:
+            recipe.tags = [str(t) for t in tags[:5]]
+        yield StatusEvent(stage="done", message="Rezept aus Beschreibung extrahiert.", progress=95)
+        yield recipe
+        return
+
+    # Step 3: Audio download + transcription fallback
+    if duration > MAX_DURATION_WARN:
+        yield StatusEvent(
+            stage="warning",
+            message=f"Video ist {duration // 60} Min. lang – Transkription kann dauern.",
+            progress=25,
+        )
+
+    yield StatusEvent(stage="downloading", message="Lade Audio herunter...", progress=30)
+
+    settings = get_settings()
+    audio_path = Path(settings.tmp_dir) / f"{uuid.uuid4()}.mp3"
+
+    try:
+        await _download_audio(url, audio_path)
+
+        # Check file size for Whisper limit
+        if audio_path.exists():
+            size = audio_path.stat().st_size
+            if size > WHISPER_MAX_BYTES:
+                raise RuntimeError(
+                    f"Audio-Datei zu groß ({size // 1024 // 1024} MB). "
+                    "Whisper unterstützt max. 25MB. Bitte kürzeres Video verwenden."
+                )
+        else:
+            raise RuntimeError("Audio-Download fehlgeschlagen.")
+
+        yield StatusEvent(stage="transcribing", message="Transkribiere Audio (Whisper)...", progress=55)
+        transcript = await transcribe_audio(audio_path)
+
+        if not transcript.strip():
+            raise RuntimeError("Transkription leer. Kein gesprochener Text erkannt.")
+
+        yield StatusEvent(stage="extracting", message="Extrahiere Rezept aus Transkription...", progress=75)
+
+        enriched_text = (
+            f"Video-Titel: {title}\n"
+            f"Beschreibung: {description[:500]}\n\n"
+            f"Transkription:\n{transcript}"
+        )
+        recipe = await extract_recipe_from_text(enriched_text, url)
+        if recipe.image_url is None:
+            recipe.image_url = _get_thumbnail(metadata)
+        if not recipe.tags and tags:
+            recipe.tags = [str(t) for t in tags[:5]]
+
+        yield StatusEvent(stage="done", message="Rezept extrahiert.", progress=95)
+        yield recipe
+
+    except FileNotFoundError:
+        raise RuntimeError(
+            "yt-dlp nicht gefunden. Bitte sicherstellen, dass yt-dlp installiert ist."
+        )
+    except Exception:
+        raise
+    finally:
+        if audio_path.exists():
+            try:
+                audio_path.unlink()
+            except Exception as e:
+                log.warning("Could not delete tmp file %s: %s", audio_path, e)
+
+
+def _ffmpeg_args() -> list[str]:
+    """Return --ffmpeg-location args if configured, else empty list."""
+    loc = get_settings().ffmpeg_location
+    return ["--ffmpeg-location", loc] if loc else []
+
+
+async def _get_metadata(url: str) -> dict:
+    """Run yt-dlp --dump-json to get metadata without downloading."""
+    proc = await asyncio.create_subprocess_exec(
+        "yt-dlp",
+        "--dump-json",
+        "--no-playlist",
+        *_ffmpeg_args(),
+        url,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=60)
+    if proc.returncode != 0:
+        err = stderr.decode(errors="replace")
+        _check_tiktok_error(err, url)
+        raise RuntimeError(f"yt-dlp Metadaten-Fehler: {err[:500]}")
+    try:
+        return json.loads(stdout.decode())
+    except json.JSONDecodeError:
+        return {}
+
+
+async def _download_audio(url: str, output_path: Path) -> None:
+    """Download audio only via yt-dlp subprocess (non-blocking)."""
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    proc = await asyncio.create_subprocess_exec(
+        "yt-dlp",
+        "--no-playlist",
+        "--extract-audio",
+        "--audio-format", "mp3",
+        "--audio-quality", "5",
+        "--max-filesize", MAX_FILESIZE,
+        *_ffmpeg_args(),
+        "--output", str(output_path).replace(".mp3", ".%(ext)s"),
+        url,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    try:
+        stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=300)
+    except asyncio.TimeoutError:
+        proc.kill()
+        raise RuntimeError("Audio-Download Timeout (5 Min.) überschritten.")
+
+    if proc.returncode != 0:
+        err = stderr.decode(errors="replace")
+        _check_tiktok_error(err, url)
+        raise RuntimeError(f"Audio-Download fehlgeschlagen: {err[:500]}")
+
+    # yt-dlp may output with different extension before conversion
+    # Try to find the actual file
+    if not output_path.exists():
+        parent = output_path.parent
+        stem = output_path.stem
+        for f in parent.glob(f"{stem}*"):
+            if f.suffix in (".mp3", ".m4a", ".webm", ".opus"):
+                f.rename(output_path)
+                break
+
+
+def _get_thumbnail(metadata: dict) -> str | None:
+    thumbnail = metadata.get("thumbnail")
+    if thumbnail:
+        return thumbnail
+    thumbnails = metadata.get("thumbnails") or []
+    if thumbnails:
+        # prefer largest
+        best = max(thumbnails, key=lambda t: (t.get("width", 0) or 0), default=None)
+        if best:
+            return best.get("url")
+    return None
+
+
+def _check_tiktok_error(stderr: str, url: str) -> None:
+    """Raise a clear error for TikTok login/cookie issues."""
+    if "tiktok" in url.lower():
+        if any(phrase in stderr.lower() for phrase in ["login", "cookie", "captcha", "403"]):
+            raise RuntimeError(
+                "TikTok erfordert Cookies für dieses Video. "
+                "Bitte exportiere deine Browser-Cookies und konfiguriere yt-dlp entsprechend."
+            )

app/extractors/website.py

@@ -0,0 +1,301 @@
+"""Website recipe extraction: recipe-scrapers → LD+JSON → GPT-4 fallback."""
+from __future__ import annotations
+import json
+import logging
+import re
+from typing import AsyncGenerator
+
+import httpx
+from bs4 import BeautifulSoup
+
+from app.models import Ingredient, Recipe, StatusEvent
+
+log = logging.getLogger(__name__)
+
+MAX_GPT_CHARS = 12_000
+
+
+async def extract_website(
+    url: str,
+) -> AsyncGenerator[StatusEvent | Recipe, None]:
+    """Yields StatusEvents then a final Recipe."""
+
+    yield StatusEvent(stage="detecting", message="Erkenne Rezeptseite...", progress=10)
+    yield StatusEvent(stage="fetching", message="Lade Seite...", progress=20)
+
+    # Fetch HTML once — reused by all tiers
+    try:
+        async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client:
+            resp = await client.get(
+                url,
+                headers={"User-Agent": "Mozilla/5.0 (compatible; Rezept0r/1.0)"},
+            )
+            resp.raise_for_status()
+            html = resp.text
+    except Exception as e:
+        raise RuntimeError(f"Seite konnte nicht geladen werden: {e}") from e
+
+    soup = BeautifulSoup(html, "lxml")
+
+    # Tier 1: recipe-scrapers (uses already-fetched HTML)
+    try:
+        from recipe_scrapers import scrape_html, NoSchemaFoundInWildMode
+        scraper = scrape_html(html, org_url=url, wild_mode=True)
+        recipe = _scraper_to_recipe(scraper, url)
+        if recipe.ingredients and recipe.steps:
+            yield StatusEvent(stage="parsing", message="Rezept geparst.", progress=70)
+            yield recipe
+            return
+    except Exception as e:
+        log.info("recipe-scrapers failed (%s), trying LD+JSON fallback", e)
+
+    yield StatusEvent(stage="parsing", message="Suche nach Schema.org-Daten...", progress=40)
+
+    # Tier 2: LD+JSON in page (including Next.js RSC payloads)
+    recipe = _try_ldjson(soup, url)
+    if recipe and (recipe.ingredients or recipe.steps):
+        yield StatusEvent(stage="parsing", message="Schema.org-Daten gefunden.", progress=70)
+        yield recipe
+        return
+
+    yield StatusEvent(stage="gpt", message="Extrahiere mit KI...", progress=55)
+
+    # Tier 3: GPT-4 fallback
+    from app.services.openai_service import extract_recipe_from_text
+    article_text = _extract_article_text(soup)
+    if not article_text.strip():
+        raise RuntimeError("Kein verwertbarer Text auf der Seite gefunden.")
+
+    truncated = article_text[:MAX_GPT_CHARS]
+    recipe = await extract_recipe_from_text(truncated, url)
+    yield StatusEvent(stage="done", message="Rezept extrahiert.", progress=100)
+    yield recipe
+
+
+def _scraper_to_recipe(scraper, url: str) -> Recipe:
+    def safe(fn):
+        try:
+            return fn()
+        except Exception:
+            return None
+
+    raw_ingredients = safe(scraper.ingredients) or []
+    ingredients = [Ingredient(name=i) for i in raw_ingredients if i]
+
+    raw_instructions = safe(scraper.instructions_list) or []
+    if not raw_instructions:
+        instr = safe(scraper.instructions) or ""
+        raw_instructions = [s.strip() for s in instr.split("\n") if s.strip()] if instr else []
+
+    nutrition_data = safe(scraper.nutrients) or {}
+    nutrition = None
+    if nutrition_data:
+        from app.models import NutritionInfo
+        kcal_str = nutrition_data.get("calories", "")
+        kcal = _parse_float(kcal_str)
+        if kcal:
+            nutrition = NutritionInfo(kcal_per_serving=kcal, source="extracted")
+
+    servings = None
+    yields_raw = safe(scraper.yields)
+    if yields_raw:
+        servings = _parse_int(yields_raw)
+
+    return Recipe(
+        title=safe(scraper.title) or "Unbekanntes Rezept",
+        description=safe(scraper.description),
+        image_url=safe(scraper.image),
+        source_url=url,
+        servings=servings,
+        prep_time_minutes=safe(scraper.prep_time),
+        cook_time_minutes=safe(scraper.cook_time),
+        ingredients=ingredients,
+        steps=raw_instructions,
+        nutrition=nutrition,
+        tags=list(safe(scraper.category) or []) if isinstance(safe(scraper.category), list)
+             else ([safe(scraper.category)] if safe(scraper.category) else []),
+    )
+
+
+def _is_recipe_type(data: dict) -> bool:
+    t = data.get("@type")
+    if isinstance(t, list):
+        return "Recipe" in t
+    return t == "Recipe"
+
+
+def _parse_ldjson_candidates(data) -> list[dict]:
+    """Return all Recipe-type dicts from a parsed JSON-LD structure."""
+    candidates: list[dict] = []
+    if isinstance(data, dict):
+        if "@graph" in data:
+            for item in data["@graph"]:
+                if isinstance(item, dict) and _is_recipe_type(item):
+                    candidates.append(item)
+        elif _is_recipe_type(data):
+            candidates.append(data)
+    elif isinstance(data, list):
+        for item in data:
+            if isinstance(item, dict) and _is_recipe_type(item):
+                candidates.append(item)
+    return candidates
+
+
+def _try_ldjson(soup: BeautifulSoup, url: str) -> Recipe | None:
+    # Standard <script type="application/ld+json"> tags
+    for script in soup.find_all("script", type="application/ld+json"):
+        try:
+            data = json.loads(script.string or "")
+        except (json.JSONDecodeError, TypeError):
+            continue
+        for item in _parse_ldjson_candidates(data):
+            return _ldjson_to_recipe(item, url)
+
+    # Next.js RSC streaming payloads: self.__next_f.push([1,"<json-encoded-string>"])
+    for script in soup.find_all("script"):
+        text = script.string or ""
+        if "recipeInstructions" not in text and "@type" not in text:
+            continue
+        m = re.search(r'self\.__next_f\.push\(\[1,"(.+)"\]\)\s*$', text, re.DOTALL)
+        if not m:
+            continue
+        try:
+            decoded = json.loads('"' + m.group(1) + '"')
+            data = json.loads(decoded)
+        except (json.JSONDecodeError, ValueError):
+            continue
+        for item in _parse_ldjson_candidates(data):
+            return _ldjson_to_recipe(item, url)
+
+    return None
+
+
+def _ldjson_to_recipe(data: dict, url: str) -> Recipe:
+    from app.models import NutritionInfo
+
+    ingredients_raw = data.get("recipeIngredient", [])
+    ingredients = [Ingredient(name=i) for i in ingredients_raw if isinstance(i, str)]
+
+    instructions_raw = data.get("recipeInstructions", [])
+    steps: list[str] = []
+    if isinstance(instructions_raw, str):
+        steps = [s.strip() for s in instructions_raw.split("\n") if s.strip()]
+    elif isinstance(instructions_raw, list):
+        for item in instructions_raw:
+            if isinstance(item, str):
+                steps.append(item)
+            elif isinstance(item, dict):
+                item_type = item.get("@type", "")
+                # HowToSection: recurse into itemListElement
+                if item_type == "HowToSection":
+                    for sub in item.get("itemListElement", []):
+                        if isinstance(sub, dict):
+                            text = sub.get("text", "") or sub.get("name", "")
+                            if text:
+                                steps.append(text)
+                else:
+                    text = item.get("text", "") or item.get("name", "")
+                    if text:
+                        steps.append(text)
+
+    nutrition = None
+    nutr_data = data.get("nutrition", {})
+    if isinstance(nutr_data, dict):
+        kcal = _parse_float(nutr_data.get("calories", ""))
+        if kcal:
+            nutrition = NutritionInfo(kcal_per_serving=kcal, source="extracted")
+
+    yields_raw = data.get("recipeYield", data.get("yield", ""))
+    servings = None
+    if isinstance(yields_raw, list) and yields_raw:
+        servings = _parse_int(str(yields_raw[0]))
+    elif isinstance(yields_raw, (str, int)):
+        servings = _parse_int(str(yields_raw))
+
+    keywords = data.get("keywords", "")
+    if isinstance(keywords, str):
+        tags = [k.strip() for k in keywords.split(",") if k.strip()]
+    elif isinstance(keywords, list):
+        tags = [str(k).strip() for k in keywords if k]
+    else:
+        tags = []
+
+    categories = data.get("recipeCategory", [])
+    if isinstance(categories, str):
+        tags += [categories]
+    elif isinstance(categories, list):
+        tags += categories
+
+    return Recipe(
+        title=data.get("name", "Unbekanntes Rezept"),
+        description=data.get("description"),
+        image_url=_extract_image_url(data.get("image")),
+        source_url=url,
+        servings=servings,
+        prep_time_minutes=_parse_duration(data.get("prepTime")),
+        cook_time_minutes=_parse_duration(data.get("cookTime")),
+        ingredients=ingredients,
+        steps=steps,
+        nutrition=nutrition,
+        tags=list(set(tags))[:10],
+    )
+
+
+def _extract_article_text(soup: BeautifulSoup) -> str:
+    for tag in soup.find_all(["script", "style", "nav", "footer", "header", "aside"]):
+        tag.decompose()
+    for selector in ["article", "main", '[class*="recipe"]', '[id*="recipe"]']:
+        el = soup.select_one(selector)
+        if el:
+            return el.get_text(separator="\n", strip=True)
+    return soup.get_text(separator="\n", strip=True)
+
+
+def _extract_image_url(image) -> str | None:
+    if not image:
+        return None
+    if isinstance(image, str):
+        return image
+    if isinstance(image, list) and image:
+        first = image[0]
+        return first.get("url") if isinstance(first, dict) else str(first)
+    if isinstance(image, dict):
+        return image.get("url") or image.get("contentUrl")
+    return None
+
+
+def _parse_float(s: str | None) -> float | None:
+    if not s:
+        return None
+    import re
+    m = re.search(r"[\d.]+", str(s))
+    try:
+        return float(m.group()) if m else None
+    except ValueError:
+        return None
+
+
+def _parse_int(s: str | None) -> int | None:
+    if not s:
+        return None
+    import re
+    m = re.search(r"\d+", str(s))
+    try:
+        return int(m.group()) if m else None
+    except ValueError:
+        return None
+
+
+def _parse_duration(iso: str | None) -> int | None:
+    """Parse ISO 8601 duration (PT1H30M) → minutes."""
+    if not iso:
+        return None
+    import re
+    hours = re.search(r"(\d+)H", iso)
+    minutes = re.search(r"(\d+)M", iso)
+    total = 0
+    if hours:
+        total += int(hours.group(1)) * 60
+    if minutes:
+        total += int(minutes.group(1))
+    return total or None