import subprocess
import sys
import os
import time
import random
import re

def install_dependencies():
    # map: pip package -> python import name
    required = {
        "requests": "requests",
        "beautifulsoup4": "bs4",
        "undetected-chromedriver": "undetected_chromedriver",
        "urllib3": "urllib3",
        "selenium": "selenium",
    }

    for pip_name, import_name in required.items():
        try:
            __import__(import_name)
        except ImportError:
            print(f"Missing package '{pip_name}'. Installing automatically via pip...")
            subprocess.check_call([sys.executable, "-m", "pip", "install", pip_name])
            print(f"    ✓ Installed {pip_name}")

# Only run auto-install when running as a normal script (not frozen by PyInstaller)
if not getattr(sys, "frozen", False):
    install_dependencies()


import requests
import urllib3
import undetected_chromedriver as uc
from selenium import webdriver


urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

def _parse_chrome_major_from_version_text(text: str):
    # Examples:
    # "Google Chrome 150.0.7871.129"
    # "Chromium 125.0.6422.0"
    m = re.search(r'(\d+)\.(\d+)\.(\d+)\.(\d+)', text)
    if not m:
        m = re.search(r'Chrome\s+(\d+)\.', text, re.IGNORECASE)
    if not m:
        return None
    return int(m.group(1))

def get_chrome_major_version():
    """
    Cross-platform Chrome major version detection.
    Tries: Windows registry -> executable --version -> common binary names.
    """
    try:
        # --- Windows: registry first ---
        if sys.platform == "win32":
            import winreg

            uninstall_roots = [
                r"SOFTWARE\Google\Chrome",
                r"SOFTWARE\WOW6432Node\Google\Chrome",
            ]

            for root_path in uninstall_roots:
                try:
                    # BLBeacon is commonly present and contains version-like data.
                    key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, root_path + r"\BLBeacon")
                    val = winreg.QueryValueEx(key, "version")[0]
                    major = _parse_chrome_major_from_version_text(val)
                    if major:
                        return major
                except Exception:
                    pass

                # Fallback: Program Files install location guess + --version
            # If registry didn’t work, fall through to executable probing.

        # --- Executable probing (Windows/macOS/Linux) ---
        candidates = []

        if sys.platform == "win32":
            # Common install paths
            candidates += [
                r"C:\Program Files\Google\Chrome\Application\chrome.exe",
                r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe",
            ]
            # Also allow using PATH name
            candidates += ["chrome.exe", "GoogleChrome.exe", "chrome"]

        elif sys.platform == "darwin":
            candidates += [
                "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
                "/Applications/Chromium.app/Contents/MacOS/Chromium",
                "google-chrome",
                "chromium",
            ]
        else:
            # Linux
            candidates += [
                "google-chrome",
                "google-chrome-stable",
                "chromium-browser",
                "chromium",
            ]

        # Try each candidate with "--version"
        for cmd in candidates:
            try:
                if os.path.isabs(cmd) and not os.path.exists(cmd):
                    continue

                p = subprocess.run(
                    [cmd, "--version"],
                    capture_output=True,
                    text=True,
                    check=False
                )
                out = (p.stdout or "") + (p.stderr or "")
                major = _parse_chrome_major_from_version_text(out.strip())
                if major:
                    return major
            except Exception:
                pass
    except Exception:
        pass

    return None


def make_driver(PROFILE_DIR):
    """
    Uses undetected-chromedriver with an explicit version_main when possible.
    Retries once if we hit a version-mismatch message.
    """
    options_uc = uc.ChromeOptions()
    # IMPORTANT: unique profile directory avoids lock / connect issues
    options_uc.add_argument(f"--user-data-dir={PROFILE_DIR}")

    major = get_chrome_major_version()
    print(f"Detected Chrome major version: {major}")

    # 1) Try UC with detected major if available
    try_versions = []
    if major is not None:
        try_versions.append(major)

    # If detection fails, UC will choose; but we at least retry once after a mismatch.
    if not try_versions:
        try_versions = [None]

    last_exc = None
    for vmain in try_versions:
        try:
            if vmain is None:
                return uc.Chrome(options=options_uc, headless=False, use_subprocess=True)
            return uc.Chrome(options=options_uc, headless=False, use_subprocess=True, version_main=vmain)
        except Exception as e:
            last_exc = e

            # Parse "only supports Chrome version XXX" and retry with that major
            msg = str(e)
            m = re.search(r"only supports Chrome version\s+(\d+)", msg, re.IGNORECASE)
            if m:
                supported_major = int(m.group(1))
                try:
                    return uc.Chrome(
                        options=options_uc,
                        headless=False,
                        use_subprocess=True,
                        version_main=supported_major
                    )
                except Exception as e2:
                    last_exc = e2

    raise last_exc
from urllib.parse import urlparse, urljoin, unquote
from bs4 import BeautifulSoup
import undetected_chromedriver as uc
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# ----------------------------
# User input / targets
# ----------------------------
user_input = input("Enter thread URL here, make sure it's the first page: ").strip()
FALLBACK_URL = "https://soyjak.forum"
TARGET_URL = user_input if user_input else FALLBACK_URL

parsed_target = urlparse(TARGET_URL)
BASE_DOMAIN = f"{parsed_target.scheme}://{parsed_target.netloc}"
HOST = parsed_target.netloc

thread_match = re.search(r'threads/([^/]+)', parsed_target.path)
if thread_match:
    folder_suffix = thread_match.group(1)
else:
    folder_suffix = f"thread_{int(time.time())}"

OUTPUT_DIR = os.path.join(".", f"{folder_suffix}")
ATTACH_DIR = os.path.join(OUTPUT_DIR, "Attachments")
PROFILE_DIR = r".\chrome_profile"

os.makedirs(ATTACH_DIR, exist_ok=True)

# ----------------------------
# Tuning
# ----------------------------
MAX_RETRIES = 3
REQUEST_TIMEOUT = 20
BACKOFF_BASE = 1.7

# XenForo attachment/link patterns vary a bit by setup/theme; this is a broad heuristic.
ATTACH_HREF_REGEX = re.compile(r'/(attachments|attachment)/', re.IGNORECASE)

# Common downloadable file extensions to avoid rewriting “non-file” links too aggressively.
COMMON_FILE_EXTS = {
    ".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp",
    ".mp4", ".webm", ".ogg", ".mp3", ".wav", ".m4a",
    ".pdf", ".zip", ".rar", ".7z", ".tar", ".gz",
    ".txt", ".csv", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
    ".svg"
}

# ----------------------------
# Helpers
# ----------------------------
def sanitize_filename(url):
    parsed = urlparse(url)
    path = unquote(parsed.path)
    filename = os.path.basename(path)
    filename = re.sub(r'\?.*$', '', filename)
    filename = re.sub(r'[^\w\-_.]', '_', filename)

    # If it still looks empty, generate one.
    if not filename or len(filename) > 200:
        filename = f"asset_{int(time.time())}_{random.randint(1000,9999)}"

    # If there's no extension, try to infer from path.
    if '.' not in filename:
        ext = os.path.splitext(path)[1].lower()
        if ext:
            filename += ext

    return filename


def normalize_url(url):
    if not url or url.startswith("data:"):
        return url
    if url.startswith("//"):
        return "https:" + url
    if url.startswith("/"):
        return urljoin(BASE_DOMAIN, url)
    if url.startswith("data/") or url.startswith("styles/") or url.startswith("attachments/"):
        return urljoin(BASE_DOMAIN, url)
    return url


def parse_srcset(srcset_value):
    """
    Returns list of URLs from srcset (ignores descriptors but uses them to pick the "best").
    Example: "url1 1x, url2 2x" or "url1 320w, url2 640w"
    """
    if not srcset_value:
        return []

    parts = [p.strip() for p in srcset_value.split(",") if p.strip()]
    results = []
    for part in parts:
        # part might be: "url 2x" or "url 640w" or just "url"
        tokens = part.split()
        url = tokens[0].strip()
        desc = tokens[1].strip() if len(tokens) > 1 else None
        results.append((url, desc))
    return results


def pick_best_srcset_candidate(srcset_value):
    """
    Picks best candidate using descriptor:
    - "2x" => highest x
    - "640w" => highest w
    - no descriptor => first URL
    """
    candidates = parse_srcset(srcset_value)
    if not candidates:
        return None

    def score(desc):
        if not desc:
            return 0
        m = re.match(r'(\d+(?:\.\d+)?)(x|w)$', desc)
        if not m:
            return 0
        val = float(m.group(1))
        return val

    best_url, _ = max(candidates, key=lambda c: score(c[1]))
    return best_url


def build_srcset_with_local_urls(img_tag, attr_name, session, referer):
    """
    Rewrites img/srcset-like attribute to local paths (best effort).
    Returns whether it rewrote something.
    """
    rewritten_any = False
    original = img_tag.get(attr_name)
    if not original:
        return False

    candidates = parse_srcset(original)
    if not candidates:
        return False

    new_parts = []
    for url, desc in candidates:
        url_norm = normalize_url(url)
        if not url_norm or not url_norm.startswith("http"):
            new_parts.append(f"{url} {desc}".strip())
            continue

        local = download_asset(session, url_norm, ATTACH_DIR, referer=referer)
        # download_asset returns the original URL if it couldn't fetch; that's okay.
        if local != url_norm:
            rewritten_any = True
        if desc:
            new_parts.append(f"{local} {desc}")
        else:
            new_parts.append(f"{local}")

    img_tag[attr_name] = ", ".join(new_parts)
    return rewritten_any


def get_img_candidate_urls(img_tag):
    urls = []

    for attr in ("src", "data-src"):
        v = img_tag.get(attr)
        if v:
            urls.append(v)

    # We'll handle srcset separately because it can contain multiple URLs.
    for attr in ("srcset", "data-srcset"):
        v = img_tag.get(attr)
        if v:
            candidates = parse_srcset(v)
            for u, _desc in candidates:
                urls.append(u)

    # Normalize and filter
    out = []
    for u in urls:
        nu = normalize_url(u)
        if nu and nu != "" and not nu.startswith("data:"):
            out.append(nu)
    return out


def setup_requests_session(driver):
    session = requests.Session()

    try:
        ua = driver.execute_script("return navigator.userAgent;")
    except Exception:
        ua = "Mozilla/5.0 (compatible; ThreadArchiver/1.0)"

    session.headers.update({
        "User-Agent": ua,
        "Referer": BASE_DOMAIN,
        "Accept": "*/*",
        "Accept-Language": "en-US,en;q=0.9",
        "Connection": "keep-alive"
    })

    # Copy cookies with a safer domain/path scope.
    for cookie in driver.get_cookies():
        name = cookie.get("name")
        value = cookie.get("value")
        if not name:
            continue

        domain = cookie.get("domain") or HOST
        path = cookie.get("path") or "/"

        # Ensure domain is in the right form for requests.
        domain = domain.lstrip(".") if isinstance(domain, str) else HOST
        session.cookies.set(name, value, domain=domain, path=path)

    return session


def download_asset(session, url, target_folder, referer=None):
    """
    Downloads a single URL if possible and returns the local relative path (if saved),
    otherwise returns the original URL.
    """
    original = url
    url = normalize_url(url)

    if not url or url.startswith("data:"):
        return original
    if not url.startswith("http"):
        return original

    local_name = sanitize_filename(url)
    dest = os.path.join(target_folder, local_name)

    # Already downloaded
    if os.path.exists(dest):
        return os.path.relpath(dest, OUTPUT_DIR).replace("\\", "/")

    headers = {"Referer": referer or BASE_DOMAIN}
    for attempt in range(MAX_RETRIES + 1):
        try:
            resp = session.get(url, headers=headers, timeout=REQUEST_TIMEOUT, stream=True, verify=False)

            if resp.status_code == 200:
                os.makedirs(os.path.dirname(dest), exist_ok=True)
                with open(dest, "wb") as f:
                    for chunk in resp.iter_content(8192):
                        if chunk:
                            f.write(chunk)
                print(f"    ✓ {local_name}")
                return os.path.relpath(dest, OUTPUT_DIR).replace("\\", "/")

            # Retry for transient blocks / throttling
            if resp.status_code in (403, 429):
                wait = (BACKOFF_BASE ** attempt) + random.random()
                print(f"    … HTTP {resp.status_code} retry {attempt+1}/{MAX_RETRIES+1} ({local_name}) in {wait:.1f}s")
                time.sleep(wait)
                continue

            print(f"    ✗ HTTP {resp.status_code}: {original}")
            return original

        except Exception as e:
            if attempt >= MAX_RETRIES:
                print(f"    ✗ Exception, giving up: {original} ({type(e).__name__}: {e})")
                return original

            wait = (BACKOFF_BASE ** attempt) + random.random()
            print(f"    … Exception ({type(e).__name__}), retry in {wait:.1f}s: {original}")
            time.sleep(wait)

    return original


# ----------------------------
# HTML normalization (clean layout)
# ----------------------------
def create_clean_layout(soup, page_number, total_pages=892):
    title_tag = soup.find("h1", class_="p-title-value") or soup.find("title")
    title_text = title_tag.get_text(strip=True) if title_tag else "Soyjak Forum"

    prev_html = f'<a href="page_{page_number-1}.html">← Previous</a>' if page_number > 1 else ''
    next_html = f'<a href="page_{page_number+1}.html">Next →</a>' if page_number < total_pages else ''

    new_html = f"""
    <!DOCTYPE html>
    <html lang="en">
    <head>
        <meta charset="utf-8">
        <title>{title_text} - Page {page_number}</title>
        <style>
            body {{ font-family: Arial, sans-serif; margin: 0; padding: 15px; background: #f4f4f4; line-height: 1.5; }}
            .header {{ background: #2c3e50; color: white; padding: 15px; margin-bottom: 20px; border-radius: 8px; }}
            .message {{ background: white; border: 1px solid #ddd; border-radius: 8px; margin-bottom: 20px; padding: 15px; display: flex; gap: 15px; }}
            .avatar {{ flex-shrink: 0; width: 75px; }}
            .avatar img {{ width: 75px; height: 75px; border-radius: 8px; object-fit: cover; }}
            .message-body {{ flex: 1; min-width: 0; }}
            .username {{ font-weight: bold; font-size: 1.15em; margin-bottom: 4px; }}
            .badges img {{ max-height: 28px; margin: 2px 4px 2px 0; vertical-align: middle; }}
            .content img, .content video, .content audio {{ max-width: 100%; height: auto; display: block; margin: 12px 0; border-radius: 6px; }}
            .pagination {{ text-align: center; margin: 40px 0; padding: 15px; background: white; border-radius: 8px; }}
            .pagination a, .pagination button {{ padding: 8px 16px; margin: 0 5px; background: #3498db; color: white; text-decoration: none; border: none; border-radius: 4px; cursor: pointer; }}
        </style>
    </head>
    <body>
        <div class="header">
            <h1>{title_text}</h1>
            <p>Page {page_number}</p>
        </div>
    """

    # XenForo commonly uses article.message for posts, but keep your original selection.
    for post in soup.find_all("article", class_="message"):
        # Username
        username_tag = post.find("a", class_="username") or post.find(".username") or post.find(".message-name")
        username = username_tag.get_text(strip=True) if username_tag else "Unknown"

        # Avatar search (aggressive)
        avatar = None
        for selector in ["img.avatar", "a.avatar img", "img[data-user-id]", "img.avatar-u", ".message-avatar img"]:
            avatar = post.select_one(selector)
            if avatar and avatar.get("src"):
                break

        avatar_src = avatar.get("src", "") if avatar else ""

        # Badges
        badges = post.find("div", class_="featuredBadges")
        badges_html = str(badges) if badges else ""

        # Content
        content = post.find("div", class_="message-content") or post.find("article") or post
        content_html = str(content) if content else "<p>No content</p>"

        new_html += f"""
        <div class="message">
            <div class="avatar">
                <img src="{avatar_src}" alt="avatar">
            </div>
            <div class="message-body">
                <div class="username">{username}</div>
                <div class="badges">{badges_html}</div>
                <div class="content">{content_html}</div>
            </div>
        </div>
        """

    new_html += f"""
        <div class="pagination">
            {prev_html}
            <strong>Page {page_number}</strong>
            {next_html}
            
            <div style="margin-top:15px;">
                <input type="number" id="pageInput" placeholder="Go to page" min="1" max="{total_pages}" style="padding:8px; width:130px;">
                <button onclick="goToPage()">Go</button>
            </div>
        </div>

        <script>
            function goToPage() {{
                let page = document.getElementById('pageInput').value.trim();
                if (page) window.location.href = `page_${{page}}.html`;
            }}
        </script>
    </body></html>
    """

    return BeautifulSoup(new_html, "html.parser")


def rewrite_images_and_media(clean_soup, session, referer):
    """
    Downloads/replaces:
    - img/src + img/data-src
    - img/srcset + img/data-srcset (rewritten so the saved HTML shows local files)
    - video/audio source tags
    """
    # Images
    for img in clean_soup.find_all("img"):
        # Rewrite src + data-src to a downloaded local asset (best candidate)
        best_srcset_url = pick_best_srcset_candidate(img.get("srcset")) or pick_best_srcset_candidate(img.get("data-srcset"))
        candidates = []

        for attr in ("src", "data-src"):
            v = img.get(attr)
            if v:
                candidates.append(normalize_url(v))

        if best_srcset_url:
            candidates.append(normalize_url(best_srcset_url))

        # Download one "primary" candidate to fill src/data-src
        primary_url = None
        for c in candidates:
            if c and c.startswith("http"):
                primary_url = c
                break

        if primary_url:
            local = download_asset(session, primary_url, ATTACH_DIR, referer=referer)
            if local:
                if img.get("src") is not None:
                    img["src"] = local
                if img.get("data-src") is not None:
                    img["data-src"] = local

        # Rewrite whole srcset/data-srcset so browser doesn't pick a remote URL variant.
        try:
            if img.get("srcset"):
                build_srcset_with_local_urls(img, "srcset", session, referer)
            if img.get("data-srcset"):
                build_srcset_with_local_urls(img, "data-srcset", session, referer)
        except Exception:
            pass

    # Videos & audios
    for media in clean_soup.find_all(["video", "audio"]):
        # Rewrite direct src if present
        if media.get("src"):
            local = download_asset(session, media["src"], ATTACH_DIR, referer=referer)
            if local:
                media["src"] = local

        # Rewrite <source> tags
        for source in media.find_all("source"):
            if source.get("src"):
                local = download_asset(session, source["src"], ATTACH_DIR, referer=referer)
                if local:
                    source["src"] = local

    return clean_soup


def rewrite_attachment_links(clean_soup, session, referer):
    """
    Downloads and rewrites XenForo-style attachment links.
    Best-effort heuristic:
    - anchors whose href contains /attachments/ or /attachment/
    - and/or has a recognizable file extension in the path
    """
    for a in clean_soup.find_all("a", href=True):
        href = a.get("href")
        if not href:
            continue

        href_norm = normalize_url(href)
        if not href_norm or not href_norm.startswith("http"):
            continue

        parsed = urlparse(href_norm)
        path = parsed.path.lower()
        ext = os.path.splitext(path)[1] if "." in path else ""

        looks_like_attachment = bool(ATTACH_HREF_REGEX.search(path)) or any(ext == e for e in COMMON_FILE_EXTS)

        if not looks_like_attachment:
            continue

        local = download_asset(session, href_norm, ATTACH_DIR, referer=referer)
        # If it downloaded, local will be a relative path under OUTPUT_DIR
        if local and local != href_norm:
            a["href"] = local

    return clean_soup


# ----------------------------
# Main scraping/archiving loop
# ----------------------------
def archive_thread():
    for d in [OUTPUT_DIR, ATTACH_DIR]:
        os.makedirs(d, exist_ok=True)

    print("Launching browser...")

    driver = make_driver(PROFILE_DIR)
    try:
        driver.get(BASE_DOMAIN)
        time.sleep(5)

        current_url = TARGET_URL
        page_number = 1

        # total_pages remains best-effort (your original hard-coded value).
        total_pages = 892

        while current_url:
            print(f"\n=== Scraping Page {page_number} ===")
            driver.get(current_url)

            WebDriverWait(driver, 25).until(EC.presence_of_element_located((By.CLASS_NAME, "p-pageWrapper")))
            time.sleep(4)

            # Scroll to trigger lazy loading
            last = driver.execute_script("return document.body.scrollHeight")
            for _ in range(40):
                driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
                time.sleep(1.5)
                new = driver.execute_script("return document.body.scrollHeight")
                if new == last:
                    break
                last = new

            # Refresh session after page load (cookies may update)
            session = setup_requests_session(driver)
            referer = current_url

            soup = BeautifulSoup(driver.page_source, "html.parser")
            print("Building clean layout...")

            clean_soup = create_clean_layout(soup, page_number, total_pages=total_pages)

            # Download/replace media
            print("Rewriting and downloading images/media...")
            clean_soup = rewrite_images_and_media(clean_soup, session, referer)

            # Download/replace attachment links
            print("Rewriting and downloading attachment links...")
            clean_soup = rewrite_attachment_links(clean_soup, session, referer)

            file_path = os.path.join(OUTPUT_DIR, f"page_{page_number}.html")
            with open(file_path, "w", encoding="utf-8") as f:
                f.write(str(clean_soup))

            print(f"✓ Saved page_{page_number}.html")

            # Next page navigation (your original logic)
            try:
                next_btn = driver.find_element(By.CSS_SELECTOR, "a.pageNav-jump--next")
                next_url = next_btn.get_attribute("href")
                current_url = urljoin(BASE_DOMAIN, next_url) if next_url else None
                page_number += 1
                time.sleep(3)
            except Exception:
                print("✅ End of thread reached.")
                current_url = None

    finally:
        driver.quit()
        print("\nDone!")


if __name__ == "__main__":
    archive_thread()