import os
import re
import sys
import time
import random
import subprocess
import shutil
import tempfile
import threading
from pathlib import Path
from urllib.parse import urljoin, urlparse, unquote
import requests
import urllib3
import undetected_chromedriver as uc
from bs4 import BeautifulSoup
from PIL import Image, UnidentifiedImageError
from selenium.webdriver.support.ui import WebDriverWait
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
BASE_DOMAIN = 'https://soyjak.forum'
MEMBERS_URL = f'{BASE_DOMAIN}/members/list/'
OUTPUT_DIR = Path('soyjak_archive')
PROFILE_DIR = Path('soyjak_chrome_profile')
MIN_MESSAGES = 100
PAGE_DELAY = (0.8, 1.8)
PROFILE_DELAY = (0.5, 1.2)
REQUEST_TIMEOUT = 15
FFMPEG_BINARY = shutil.which('ffmpeg')
CHROME_VERSION_MAIN = None
MAX_PROFILES = None
print_lock = threading.Lock()

def log(message):
    with print_lock:
        print(message, flush=True)

def normalize_url(url):
    if not url:
        return None
    url = url.strip()
    if url.startswith('data:') or url.startswith('blob:'):
        return None
    return urljoin(BASE_DOMAIN + '/', url)

def safe_folder_name(username):
    username = unquote(username).strip()
    username = re.sub('[<>:"/\\\\|?*\\x00-\\x1f]', '_', username)
    username = username.rstrip('. ')
    if not username:
        username = 'unknown_user'
    if username.upper() in {'CON', 'PRN', 'AUX', 'NUL', *(f'COM{i}' for i in range(1, 10)), *(f'LPT{i}' for i in range(1, 10))}:
        username = '_' + username
    return username[:150]

def safe_filename_from_url(url):
    name = os.path.basename(urlparse(url).path)
    name = re.sub('[^a-zA-Z0-9._-]', '_', name)
    return name or 'download'

def make_driver(profile_dir):
    profile_dir = os.path.abspath(str(profile_dir))
    os.makedirs(profile_dir, exist_ok=True)
    candidates = [shutil.which(name) for name in ('chrome', 'google-chrome', 'chromium', 'chromium-browser')]
    candidates += [os.path.join(os.environ.get('PROGRAMFILES', 'C:\\Program Files'), 'Google', 'Chrome', 'Application', 'chrome.exe'), os.path.join(os.environ.get('PROGRAMFILES(X86)', 'C:\\Program Files (x86)'), 'Google', 'Chrome', 'Application', 'chrome.exe'), os.path.join(os.environ.get('LOCALAPPDATA', ''), 'Google', 'Chrome', 'Application', 'chrome.exe')]
    binary = next((p for p in candidates if p and os.path.isfile(p)), None)
    if not binary:
        raise RuntimeError('Chrome executable not found. Install Google Chrome or set its path in make_driver().')

    def launch(use_saved_profile):
        options = uc.ChromeOptions()
        options.binary_location = binary
        for arg in ('--disable-gpu', '--disable-background-networking', '--disable-background-timer-throttling', '--disable-backgrounding-occluded-windows', '--disable-component-update', '--disable-sync', '--no-first-run', '--no-default-browser-check', '--disable-dev-shm-usage', '--remote-debugging-port=0'):
            options.add_argument(arg)
        options.add_experimental_option('prefs', {'profile.default_content_setting_values.notifications': 2, 'profile.default_content_setting_values.geolocation': 2})
        kwargs = {'options': options, 'headless': False, 'use_subprocess': True}
        if use_saved_profile:
            kwargs['user_data_dir'] = profile_dir
        if CHROME_VERSION_MAIN is not None:
            kwargs['version_main'] = CHROME_VERSION_MAIN
        return uc.Chrome(**kwargs)
    try:
        return launch(True)
    except Exception as exc:
        log(f'Chrome launch with saved profile failed: {exc}')
        log('Retrying with a temporary Chrome profile...')
        return launch(False)

def wait_for_page(driver, timeout=25):
    WebDriverWait(driver, timeout).until(lambda d: d.execute_script('return document.readyState') in ('interactive', 'complete'))

def setup_requests_session(driver):
    session = requests.Session()
    try:
        user_agent = driver.execute_script('return navigator.userAgent;')
    except Exception:
        user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome Safari/537.36'
    session.headers.update({'User-Agent': user_agent, 'Referer': BASE_DOMAIN + '/', 'Accept': '*/*', 'Accept-Language': 'en-US,en;q=0.9'})
    for cookie in driver.get_cookies():
        name = cookie.get('name')
        if not name:
            continue
        kwargs = {'path': cookie.get('path') or '/'}
        if cookie.get('domain'):
            kwargs['domain'] = cookie['domain']
        session.cookies.set(name, cookie.get('value', ''), **kwargs)
    return session

def parse_message_count(member):
    for dl in member.select('dl.pairs'):
        dt = dl.find('dt')
        dd = dl.find('dd')
        if not dt or not dd:
            continue
        if dt.get_text(' ', strip=True).rstrip(':').lower() == 'messages':
            value = dd.get_text('', strip=True)
            value = value.replace(',', '').replace(' ', '')
            match = re.search('\\d+', value)
            if match:
                return int(match.group())
    return None

def scrape_member_page(html_text):
    soup = BeautifulSoup(html_text, 'html.parser')
    members = []
    for row in soup.select('.contentRow-main'):
        anchor = row.select_one('h3.contentRow-header a.username[href*="/members/"]')
        if not anchor:
            continue
        username = anchor.get_text(' ', strip=True)
        profile_url = normalize_url(anchor.get('href'))
        if not username or not profile_url:
            continue
        message_count = parse_message_count(row)
        if message_count is None or message_count <= MIN_MESSAGES:
            continue
        members.append({'username': username, 'profile_url': profile_url, 'messages': message_count})
    return (members, soup)

def get_next_page(soup, current_url):
    selectors = ['a.pageNav-jump--next', 'a[rel="next"]', '.pageNav a.pageNav-jump--next']
    for selector in selectors:
        link = soup.select_one(selector)
        if link and link.get('href'):
            return urljoin(current_url, link['href'])
    return None

def collect_members(driver):
    log('\n========== PHASE 1: COLLECT MEMBERS ==========')
    members = []
    seen_profiles = set()
    seen_pages = set()
    page_url = MEMBERS_URL
    page_number = 0
    while page_url and page_url not in seen_pages:
        seen_pages.add(page_url)
        page_number += 1
        log(f'[LIST] Page {page_number}: {page_url}')
        try:
            driver.get(page_url)
            wait_for_page(driver)
            time.sleep(random.uniform(*PAGE_DELAY))
            page_members, soup = scrape_member_page(driver.page_source)
            added = 0
            for member in page_members:
                key = member['profile_url'].rstrip('/').lower()
                if key in seen_profiles:
                    continue
                seen_profiles.add(key)
                members.append(member)
                added += 1
            log(f'[LIST] Found {len(page_members)} qualifying entries; added {added} new members.')
            next_url = get_next_page(soup, page_url)
            if not next_url:
                log('[LIST] Next button is gone. Pagination finished.')
                break
            page_url = next_url
        except Exception as exc:
            log(f'[ERROR] Failed to process list page: {exc}')
            log('Stopping pagination to avoid skipping unknown pages.')
            break
    log(f'\nCollected {len(members)} unique members over {MIN_MESSAGES} messages.')
    return members

def is_catbox_url(url):
    if not url:
        return False
    parsed = urlparse(url.strip())
    host = (parsed.hostname or '').lower().rstrip('.')
    return parsed.scheme.lower() == 'https' and (host == 'catbox.moe' or host.endswith('.catbox.moe'))

def extract_music_url(soup):
    audio = soup.select_one('audio#profileMusic')
    if not audio:
        return None
    candidates = [source.get('src') for source in audio.find_all('source', src=True)]
    if audio.get('src'):
        candidates.append(audio.get('src'))
    for candidate in candidates:
        url = normalize_url(candidate)
        if url and is_catbox_url(url):
            return url
    return None

def extract_avatar_url(soup):
    avatar = soup.select_one('.memberHeader-avatar a.avatar img, .memberHeader-avatar img, img.avatar[data-user-id]')
    if not avatar:
        return None
    candidates = []
    for item in avatar.get('srcset', '').split(','):
        parts = item.strip().split()
        if not parts:
            continue
        descriptor = parts[1] if len(parts) > 1 else '1x'
        match = re.match('(\\d+(?:\\.\\d+)?)(x|w)$', descriptor)
        score = float(match.group(1)) if match else 1.0
        if match and match.group(2) == 'w':
            score /= 1920.0
        candidates.append((score, parts[0]))
    if candidates:
        src = max(candidates, key=lambda item: item[0])[1]
    else:
        src = avatar.get('src')
    return normalize_url(src)

def extract_banner_url(soup):
    banner = soup.select_one('.memberProfileBanner[style]')
    if not banner:
        return None
    style = banner.get('style', '')
    match = re.search('background-image\\s*:\\s*url\\(\\s*\n            (?:\n                ["\'](?P<quoted>[^"\']+)["\']\n                |\n                (?P<plain>[^)]+)\n            )\n            \\s*\\)', style, re.I | re.X)
    if not match:
        return None
    return normalize_url((match.group('quoted') or match.group('plain')).strip())

def extract_background_url(soup):
    for style_tag in soup.find_all('style'):
        css = style_tag.get_text(' ', strip=False)
        html_rule = re.search('html\\s*\\{([^}]*)\\}', css, re.I | re.S)
        if not html_rule:
            continue
        rule = html_rule.group(1)
        if not re.search('background(?:-image)?\\s*:', rule, re.I):
            continue
        urls = re.findall('url\\(\\s*(?:[\\"\']([^\\"\']+)[\\"\']|([^)]+))\\s*\\)', rule, re.I)
        for quoted, plain in urls:
            candidate = (quoted or plain).strip()
            if candidate and (not candidate.lower().startswith('data:')):
                return normalize_url(candidate)
    return None

def download_to_temp(session, url):
    response = session.get(url, timeout=REQUEST_TIMEOUT, stream=True, verify=True, allow_redirects=True)
    response.raise_for_status()
    temp = tempfile.NamedTemporaryFile(prefix='soyjak_asset_', suffix='_' + safe_filename_from_url(url), delete=False)
    temp_path = temp.name
    try:
        with temp:
            for chunk in response.iter_content(chunk_size=128 * 1024):
                if chunk:
                    temp.write(chunk)
        if os.path.getsize(temp_path) == 0:
            raise ValueError('Downloaded file is empty.')
        return temp_path
    except Exception:
        try:
            os.remove(temp_path)
        except OSError:
            pass
        raise
    finally:
        response.close()

def save_image(session, url, destination_stem):
    if not url:
        return None
    temp_path = None
    converted_path = None
    try:
        temp_path = download_to_temp(session, url)
        with Image.open(temp_path) as image:
            image_format = (image.format or '').upper()
            image.verify()
        if image_format in ('GIF', 'PNG'):
            extension = '.gif' if image_format == 'GIF' else '.png'
            destination = destination_stem.with_suffix(extension)
            shutil.move(temp_path, str(destination))
            temp_path = None
            return destination
        with Image.open(temp_path) as image:
            converted = image.convert('RGBA')
            converted_path = str(destination_stem.with_name(destination_stem.name + '_converted.png'))
            converted.save(converted_path, format='PNG')
        destination = destination_stem.with_suffix('.png')
        shutil.move(converted_path, str(destination))
        converted_path = None
        return destination
    except (requests.RequestException, OSError, ValueError, UnidentifiedImageError) as exc:
        log(f'    [IMAGE ERROR] {url}: {exc}')
        return None
    finally:
        for path in (temp_path, converted_path):
            if path:
                try:
                    os.remove(path)
                except OSError:
                    pass

def save_music_as_mp3(session, url):
    if not is_catbox_url(url):
        log(f'    [MUSIC SKIP] Not a Catbox HTTPS URL: {url}')
        return None
    if not FFMPEG_BINARY:
        log('    [MUSIC ERROR] FFmpeg was not found on PATH. Install FFmpeg and restart the script.')
        return None
    input_path = None
    output_path = None
    try:
        input_path = download_to_temp(session, url)
        output_file = tempfile.NamedTemporaryFile(prefix='soyjak_music_', suffix='.mp3', delete=False)
        output_path = output_file.name
        output_file.close()
        result = subprocess.run([FFMPEG_BINARY, '-nostdin', '-hide_banner', '-loglevel', 'error', '-y', '-i', input_path, '-vn', '-codec:a', 'libmp3lame', '-q:a', '2', output_path], capture_output=True, text=True, timeout=300, check=False)
        if result.returncode != 0:
            error = (result.stderr or '').strip()
            raise RuntimeError(error[-1000:] or 'FFmpeg conversion failed.')
        if not os.path.isfile(output_path) or os.path.getsize(output_path) == 0:
            raise RuntimeError('FFmpeg produced no MP3 output.')
        result_path = output_path
        output_path = None
        return result_path
    except (requests.RequestException, OSError, subprocess.TimeoutExpired, RuntimeError, ValueError) as exc:
        log(f'    [MUSIC ERROR] {url}: {exc}')
        return None
    finally:
        for path in (input_path, output_path):
            if path:
                try:
                    os.remove(path)
                except OSError:
                    pass

def archive_profile(driver, session, member):
    username = member['username']
    profile_url = member['profile_url']
    messages = member['messages']
    log(f'\n[PROFILE] {username} ({messages} messages): {profile_url}')
    converted_music = None
    try:
        driver.get('view-source:' + profile_url)
        try:
            wait_for_page(driver)
        except Exception:
            pass
        time.sleep(random.uniform(*PROFILE_DELAY))
        soup = None
        try:
            response = session.get(profile_url, timeout=REQUEST_TIMEOUT, allow_redirects=True)
            response.raise_for_status()
            soup = BeautifulSoup(response.text, 'html.parser')
            response.close()
        except requests.RequestException as exc:
            log(f'    [INFO] HTTP profile fetch failed; using browser DOM: {exc}')
        if soup is None or not soup.select_one('audio#profileMusic'):
            driver.get(profile_url)
            wait_for_page(driver)
            soup = BeautifulSoup(driver.page_source, 'html.parser')
        music_url = extract_music_url(soup)
        if not music_url:
            log('    [SKIP] No usable Catbox audio#profileMusic source.')
            return 'skipped'
        converted_music = save_music_as_mp3(session, music_url)
        if not converted_music:
            log('    [SKIP] Music download or MP3 conversion failed; no folder created.')
            return 'skipped'
        folder = OUTPUT_DIR / safe_folder_name(username)
        folder.mkdir(parents=True, exist_ok=True)
        music_destination = folder / 'profilemusic.mp3'
        shutil.move(converted_music, str(music_destination))
        converted_music = None
        log(f'    [OK] {music_destination}')
        for label, url, stem in (('avatar', extract_avatar_url(soup), folder / 'avatar'), ('banner', extract_banner_url(soup), folder / 'banner'), ('background', extract_background_url(soup), folder / 'background')):
            if not url:
                log(f'    [INFO] No {label} URL found.')
                continue
            result = save_image(session, url, stem)
            if result:
                log(f'    [OK] {result.name}')
        return 'saved'
    except Exception as exc:
        log(f'    [ERROR] Profile failed: {exc}')
        return 'error'
    finally:
        if converted_music:
            try:
                os.remove(converted_music)
            except OSError:
                pass


def main():
    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
    driver = None
    session = None
    saved = 0
    skipped = 0
    errors = 0
    try:
        log('Starting Chrome...')
        driver = make_driver(PROFILE_DIR)
        log('Open the forum and sign in if necessary.')
        log('The script will start collecting members in 15 seconds.')
        time.sleep(15)
        session = setup_requests_session(driver)
        members = collect_members(driver)
        if not members:
            log('\nNo qualifying members were collected. Check that the member list loaded and that you have permission to view it.')
            return
        manifest_path = OUTPUT_DIR / 'members_over_100_messages.csv'
        import csv
        with open(manifest_path, 'w', newline='', encoding='utf-8-sig') as file:
            writer = csv.DictWriter(file, fieldnames=['username', 'messages', 'profile_url'])
            writer.writeheader()
            writer.writerows(members)
        log(f'\nMember list saved to: {manifest_path}')
        log(f'Profiles to inspect: {len(members)}')
        profile_limit = MAX_PROFILES
        if profile_limit is not None:
            members = members[:profile_limit]
            log(f'Testing limit enabled: {len(members)} profiles')
        log('\n========== PHASE 2: ARCHIVE PROFILES ==========')
        for index, member in enumerate(members, start=1):
            log(f'\nProgress: {index}/{len(members)}')
            result = archive_profile(driver, session, member)
            if result == 'saved':
                saved += 1
            elif result == 'skipped':
                skipped += 1
            else:
                errors += 1
        log('\n========== FINISHED ==========')
        log(f'Profiles inspected: {len(members)}')
        log(f'Profiles with music processed: {saved}')
        log(f'Profiles skipped (no music source): {skipped}')
        log(f'Errors: {errors}')
        log(f'Output directory: {OUTPUT_DIR.resolve()}')
    except KeyboardInterrupt:
        log('\nInterrupted by user.')
    finally:
        if session:
            session.close()
        if driver:
            try:
                driver.quit()
            except Exception:
                pass
if __name__ == '__main__':
    main()