Работа с Selenium для парсинга ЦИАН?

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
import time
import re

# --- Параметры ---
MAX_RESULTS = 10
URL = "https://www.cian.ru/kupit-kvartiru/" # Москва
# URL = "https://zvenigorod.cian.ru/kupit-kvartiru/"  # Московская Область

# --- Настройка Selenium ---
options = webdriver.ChromeOptions()
# options.add_argument('--headless') # Управляет видимостью окна браузера
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/129.0.0.0 Safari/537.36')
options.add_argument('accept=text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8')
options.add_argument('accept-language=ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7')

try:
    driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)
except Exception as e:
    print("Ошибка инициализации WebDriver:", e)
    raise SystemExit

try:
    driver.get(URL)
    time.sleep(5)

    # Прокрутка для подгрузки карточек
    for _ in range(7):
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(3)

    WebDriverWait(driver, 30).until(
        EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'div[data-name="LinkArea"]'))
    )

    data = []
    seen_links = set()
    output_count = 0

    # Снимок всех карточек
    all_cards = driver.find_elements(By.CSS_SELECTOR, 'div[data-name="LinkArea"]')

    for idx in range(len(all_cards)):
        if output_count >= MAX_RESULTS:
            break

        # Пересобираем список карточек
        cards = driver.find_elements(By.CSS_SELECTOR, 'div[data-name="LinkArea"]')
        if idx >= len(cards):
            break
        card = cards[idx]

        try:
            # Скроллим к карточке
            driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", card)
            time.sleep(0.5)

            # Получаем ссылку и фильтруем дубликаты
            try:
                link_el = card.find_element(By.CSS_SELECTOR, 'a[href*="/sale/flat/"]')
                link = link_el.get_attribute('href')
            except Exception:
                link = "N/A"

            if link != "N/A" and link in seen_links:
                continue
            if link != "N/A":
                seen_links.add(link)

            # Перед кликом — сколько телефонов уже на странице
            prev_phone_count = len(driver.find_elements(By.CSS_SELECTOR, 'span[data-mark="PhoneValue"]'))

            # Клик по кнопке телефона
            try:
                phone_buttons = card.find_elements(By.CSS_SELECTOR, 'button[data-mark="PhoneButton"]')
                if phone_buttons:
                    btn = phone_buttons[0]
                    try:
                        driver.execute_script("arguments[0].click();", btn)
                    except Exception:
                        try:
                            btn.click()
                        except Exception:
                            pass
                    # Ждём появления нового телефона
                    try:
                        WebDriverWait(driver, 5).until(
                            lambda d: len(d.find_elements(By.CSS_SELECTOR, 'span[data-mark="PhoneValue"]')) > prev_phone_count
                        )
                    except Exception:
                        pass
                else:
                    # Глобальный клик (редко нужно)
                    all_phone_buttons = driver.find_elements(By.CSS_SELECTOR, 'button[data-mark="PhoneButton"]')
                    if all_phone_buttons:
                        try:
                            driver.execute_script("arguments[0].click();", all_phone_buttons[0])
                            WebDriverWait(driver, 3).until(
                                lambda d: len(d.find_elements(By.CSS_SELECTOR, 'span[data-mark="PhoneValue"]')) > prev_phone_count
                            )
                        except Exception:
                            pass
            except Exception:
                pass

            # Ожидание для загрузки контактов и телефона
            time.sleep(20)

            # Парсим HTML всей страницы
            html = driver.page_source
            soup = BeautifulSoup(html, 'lxml')
            current_card = soup.select('div[data-name="LinkArea"]')[idx]

            # Данные карточки
            title_elem = current_card.select_one('span[data-mark="OfferTitle"]')
            subtitle_elem = current_card.select_one('span[data-mark="OfferSubtitle"]')
            price_elem = current_card.select_one('span[data-mark="MainPrice"], span[class*="-price-"], div[class*="-price-"]')
            address_elem = current_card.select_one(
                'div[class*="-labels-"], div[data-name="Geo"], div[class*="-address-"], div[class*="-geo-"], span[class*="-subtitle-"]')
            area_elem = current_card.select_one('div[data-name="DescriptionItem"], div[class*="-info-"], span[class*="-info-"]')

            title = title_elem.get_text(strip=True) if title_elem else "N/A"
            subtitle = subtitle_elem.get_text(strip=True) if subtitle_elem else ""
            title_combined = (title + " " + subtitle).strip() if title != "N/A" or subtitle else (title if title != "N/A" else "N/A")
            price = price_elem.get_text(strip=True) if price_elem else "N/A"
            area = area_elem.get_text(strip=True) if area_elem else "N/A"

            # Адрес
            address = "N/A"
            if address_elem:
                geo_labels = address_elem.select('a[data-name="GeoLabel"]')
                address_parts = [lbl.get_text(strip=True) for lbl in geo_labels]
                address = ", ".join(address_parts) if address_parts else address_elem.get_text(strip=True)
            if not address or address == "N/A":
                full_text = current_card.get_text(" ", strip=True)
                address_match = re.search(
                    r'(Москва|Московская область)[\s,:\-А-Яа-яЁё0-9\.,]+', full_text)
                if address_match:
                    address = address_match.group(0).strip()

            # Нормализация площади/цены
            if area == "N/A" or not re.match(r'^\d+[,\.]?\d*\s*м²$', area):
                full_text = current_card.get_text(" ", strip=True)
                area_match = re.search(r'(\d+[,\.]?\d*)\s*м²', full_text)
                area = (area_match.group(1).replace('.', ',') + ' м²') if area_match else area

            if price == "N/A":
                full_text = current_card.get_text(" ", strip=True)
                price_match = re.search(r'(\d[\d\s]*)\s*₽', full_text)
                price = (price_match.group(1).strip() + ' ₽') if price_match else price

            # Телефон (ваш подход: последний номер после клика)
            phone = "N/A"
            try:
                page_phones = driver.find_elements(By.CSS_SELECTOR, 'span[data-mark="PhoneValue"]')
                if page_phones:
                    phone = page_phones[-1].text.strip()
            except Exception:
                print(f"Объявление {idx + 1}: Телефон не найден")

            # Контакты: ищем ближайший блок контактов перед телефоном
            contacts = {}
            contact_blocks = soup.select('div[class*="_93444fe79c--contact--"]')
            if contact_blocks:
                # Ищем телефон в DOM и ближайший предшествующий блок контактов
                phone_elems = soup.select('span[data-mark="PhoneValue"]')
                if phone_elems:
                    phone_elem = phone_elems[-1]  # Последний телефон
                    # Ищем ближайший предшествующий div[class*="_93444fe79c--contact--"]
                    parent = phone_elem.find_previous('div', class_=re.compile('_93444fe79c--contact--'))
                    if parent:
                        type_elem = parent.select_one('span[class*="text_textTransform__uppercase"]')
                        name_elem = parent.select_one('span[class*="fontSize_16px"]')
                        if type_elem and name_elem:
                            contact_type = type_elem.get_text(strip=True)
                            contact_name = name_elem.get_text(strip=True)
                            contacts[contact_type] = contact_name
            if not contacts:
                print(f"Объявление {idx + 1}: Контакты не найдены")

            # Пропуск пустых карточек
            if title_combined == "N/A" and price == "N/A" and address == "N/A":
                continue

            output_count += 1
            record = {
                "title": title_combined,
                "price": price,
                "address": address,
                "area": area,
                "link": link,
                "phone": phone,
                "contacts": contacts
            }
            data.append(record)

            print(f"Объявление {output_count}: Заголовок: {title_combined}, Цена: {price}, Адрес: {address}, "
                  f"Площадь: {area}, Телефон: {phone}, Контакты: {contacts}, Ссылка: {link}")

            # Debug: сохранить HTML карточки
            try:
                with open(f'объявление-{output_count}.html', 'w', encoding='utf-8') as f:
                    f.write(str(current_card))
            except Exception:
                pass

        except Exception as ex:
            print(f"Ошибка при обработке карточки {idx + 1}: {ex}")
            continue
finally:
    driver.quit()
    time.sleep(1)