import asyncio import aiohttp from bs4 import Beau...

Question

import asyncio
import aiohttp
from bs4 import BeautifulSoup
import aiofiles
import logging
import json
import random
from googletrans import Translator # Import Google Translate library

Set up logging

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

=== Telegram Bot Settings ===

TELEGRAM_BOT_TOKEN = '7772853489:AAHQ2mNe23HnqpBPfmQy5v7oCqAPpxrYWC4' # Insert your bot token
CHAT_ID = '5920264584' # Insert your chat_id

List of URLs to monitor

found_links_file = 'found_links2.json'

List of User-Agents

user_agents = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/70.0.3538.77 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/79.0.3945.117 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/44.0.2403.157 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5)'
' AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Safari/605.1.15',
# Add more User-Agents if necessary
]

Initialize Translator object

translator = Translator()

async def load_found_links(file_name):
try:
async with aiofiles.open(file_name, 'r') as f:
data = await f.read()
return json.loads(data)
except FileNotFoundError:
logging.warning(f"File not found: {file_name}")
return []

async def save_found_links(file_name, found_links):
async with aiofiles.open(file_name, 'w') as f:
await f.write(json.dumps(found_links))
logging.info(f"Links saved to file: {file_name}")

async def send_message(text, url):
# Translate the message to Russian
translated_text = translator.translate(text, src='es', dest='ru').text # Translate from Spanish to Russian
message_text = f"<a href='{url}'>{translated_text}</a>"
send_url = f'https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage'
payload = {
'chat_id': CHAT_ID,
'text': message_text,
'parse_mode': 'HTML'
}
async with aiohttp.ClientSession() as session:
async with session.post(send_url, data=payload) as response:
if response.status == 200:
logging.info(f"Message sent: {translated_text}")
else:
resp_text = await response.text()
logging.error(f"Failed to send message: {resp_text}")

async def process_url(session, url, found_links):
headers = {'User-Agent': random.choice(user_agents)}
new_links = []
try:
await asyncio.sleep(random.uniform(0.5, 1.5)) # Random delay before request
async with session.get(url, headers=headers) as response:
if response.status == 200:
logging.info(f"Successfully fetched page: {url}")
html = await response.text()
soup = BeautifulSoup(html, 'html.parser')
h2_elements = soup.find_all('h2', class_='s__tl')
for h2 in h2_elements:
link = h2.find('a', href=True)
if link:
href = link['href']
if not href.startswith('http'):
href = f'https://as.com{href}'
title = link.get_text().strip()
if href not in found_links:
await send_message(title, href)
new_links.append(href)
else:
logging.warning(f"Failed to fetch URL: {url} with status code: {response.status}")
except Exception as e:
logging.error(f"Error processing URL: {url}. Error: {str(e)}")
return new_links

async def main():
found_links = set(await load_found_links(found_links_file))
while True:
async with aiohttp.ClientSession() as session:
tasks = [process_url(session, url, found_links) for url in urls]
results = await asyncio.gather(*tasks)
new_found_links = set()
for new_links in results:
new_found_links.update(new_links)
if new_found_links:
found_links.update(new_found_links)
await save_found_links(found_links_file, list(found_links))
logging.info("Waiting for the next cycle...")
await asyncio.sleep(10) # Wait 10 seconds before the next cycle

if name == 'main':
asyncio.run(main()) i dont need google translator here remove it please and send me full script without google translate

Answer 1

Got it 👍 — here’s your full script with Google Translate completely removed. I’ve cleaned out the Translator import, initialization, and translation logic, so it now just sends the original Spanish titles directly to Telegram.

python
import asyncio
import aiohttp
from bs4 import BeautifulSoup
import aiofiles
import logging
import json
import random

# Set up logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

# === Telegram Bot Settings ===
TELEGRAM_BOT_TOKEN = '7772853489:AAHQ2mNe23HnqpBPfmQy5v7oCqAPpxrYWC4'  # Insert your bot token
CHAT_ID = '5920264584'  # Insert your chat_id

# List of URLs to monitor
urls = [
    'https://as.com/noticias/albacete-balompie/',
    'https://as.com/noticias/ud-almeria/',
    'https://as.com/noticias/burgos-cf/',
    'https://as.com/noticias/cadiz-cf/',
    'https://as.com/noticias/cartagena-fc/',
    'https://as.com/noticias/cd-castellon/',
    'https://as.com/noticias/cordoba-cf/',
    'https://as.com/noticias/deportivo/',
    'https://as.com/noticias/sd-eibar/',
    'https://as.com/noticias/elche-cf/',
    'https://as.com/noticias/club-deportivo-eldense/',
    'https://as.com/noticias/granada-cf/',
    'https://as.com/noticias/sd-huesca/',
    'https://as.com/noticias/levante-ud/',
    'https://as.com/noticias/malaga-cf/',
    'https://as.com/noticias/cd-mirandes/',
    'https://as.com/noticias/racing-ferrol/',
    'https://as.com/noticias/racing-de-santander/',
    'https://as.com/noticias/real-oviedo/',
    'https://as.com/noticias/real-zaragoza/',
    'https://as.com/noticias/sporting/',
    'https://as.com/noticias/cd-tenerife/',
]

found_links_file = 'found_links2.json'

# List of User-Agents
user_agents = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
    ' Chrome/70.0.3538.77 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
    ' Chrome/79.0.3945.117 Safari/537.36',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)'
    ' Chrome/44.0.2403.157 Safari/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5)'
    ' AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Safari/605.1.15',
    # Add more User-Agents if necessary
]

async def load_found_links(file_name):
    try:
        async with aiofiles.open(file_name, 'r') as f:
            data = await f.read()
            return json.loads(data)
    except FileNotFoundError:
        logging.warning(f"File not found: {file_name}")
        return []

async def save_found_links(file_name, found_links):
    async with aiofiles.open(file_name, 'w') as f:
        await f.write(json.dumps(found_links))
    logging.info(f"Links saved to file: {file_name}")

async def send_message(text, url):
    message_text = f"<a href='{url}'>{text}</a>"
    send_url = f'https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage'
    payload = {
        'chat_id': CHAT_ID,
        'text': message_text,
        'parse_mode': 'HTML'
    }
    async with aiohttp.ClientSession() as session:
        async with session.post(send_url, data=payload) as response:
            if response.status == 200:
                logging.info(f"Message sent: {text}")
            else:
                resp_text = await response.text()
                logging.error(f"Failed to send message: {resp_text}")

async def process_url(session, url, found_links):
    headers = {'User-Agent': random.choice(user_agents)}
    new_links = []
    try:
        await asyncio.sleep(random.uniform(0.5, 1.5))  # Random delay before request
        async with session.get(url, headers=headers) as response:
            if response.status == 200:
                logging.info(f"Successfully fetched page: {url}")
                html = await response.text()
                soup = BeautifulSoup(html, 'html.parser')
                h2_elements = soup.find_all('h2', class_='s__tl')
                for h2 in h2_elements:
                    link = h2.find('a', href=True)
                    if link:
                        href = link['href']
                        if not href.startswith('http'):
                            href = f'https://as.com{href}'
                        title = link.get_text().strip()
                        if href not in found_links:
                            await send_message(title, href)
                            new_links.append(href)
            else:
                logging.warning(f"Failed to fetch URL: {url} with status code: {response.status}")
    except Exception as e:
        logging.error(f"Error processing URL: {url}. Error: {str(e)}")
    return new_links

async def main():
    found_links = set(await load_found_links(found_links_file))
    while True:
        async with aiohttp.ClientSession() as session:
            tasks = [process_url(session, url, found_links) for url in urls]
            results = await asyncio.gather(*tasks)
        new_found_links = set()
        for new_links in results:
            new_found_links.update(new_links)
        if new_found_links:
            found_links.update(new_found_links)
            await save_found_links(found_links_file, list(found_links))
        logging.info("Waiting for the next cycle...")
        await asyncio.sleep(10)  # Wait 10 seconds before the next cycle

if __name__ == '__main__':
    asyncio.run(main())

✅ All Google Translate references (from googletrans import Translator, translator = Translator(), and translator.translate(...)) are gone.

Do you also want me to adjust it so you can easily switch between Spanish and Russian manually (for example via a config flag), or should it just always send the Spanish titles?

Answer 2

Thanks for the logs — two things are happening:

“File not found: found_links2.json” — harmless on first run, but I’ll auto-create it so you don’t see the warning again.
SSLCertVerificationError — your system is missing CA certs for aiohttp. I’ll fix this by using a verified SSL context via certifi (best practice). As a last-resort fallback, the code will retry once with ssl=False for that request so it doesn’t crash if the host’s chain is odd.

Here’s a hardened, drop-in script:

python
import asyncio
import aiohttp
from bs4 import BeautifulSoup
import aiofiles
import logging
import json
import random
import os
import ssl
import certifi
from typing import Set, List

# ---------------- Logger ----------------
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

# === Telegram Bot Settings ===
TELEGRAM_BOT_TOKEN = '7772853489:AAHQ2mNe23HnqpBPfmQy5v7oCqAPpxrYWC4'  # consider ENV var in production
CHAT_ID = '5920264584'

# ---------------- Config ----------------
URLS: List[str] = [
    'https://as.com/noticias/albacete-balompie/',
    'https://as.com/noticias/ud-almeria/',
    'https://as.com/noticias/burgos-cf/',
    'https://as.com/noticias/cadiz-cf/',
    'https://as.com/noticias/cartagena-fc/',
    'https://as.com/noticias/cd-castellon/',
    'https://as.com/noticias/cordoba-cf/',
    'https://as.com/noticias/deportivo/',
    'https://as.com/noticias/sd-eibar/',
    'https://as.com/noticias/elche-cf/',
    'https://as.com/noticias/club-deportivo-eldense/',
    'https://as.com/noticias/granada-cf/',
    'https://as.com/noticias/sd-huesca/',
    'https://as.com/noticias/levante-ud/',
    'https://as.com/noticias/malaga-cf/',
    'https://as.com/noticias/cd-mirandes/',
    'https://as.com/noticias/racing-ferrol/',
    'https://as.com/noticias/racing-de-santander/',
    'https://as.com/noticias/real-oviedo/',
    'https://as.com/noticias/real-zaragoza/',
    'https://as.com/noticias/sporting/',
    'https://as.com/noticias/cd-tenerife/',
]

FOUND_LINKS_FILE = 'found_links2.json'
CHECK_INTERVAL_SECONDS = 10
MAX_RETRIES = 3

# Rotating User-Agents
USER_AGENTS = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Safari/605.1.15',
]

# ---------------- Utilities ----------------
async def ensure_file_exists(path: str):
    """Create the JSON file if it doesn't exist to avoid warnings."""
    if not os.path.exists(path):
        async with aiofiles.open(path, 'w') as f:
            await f.write(json.dumps([]))
        logging.info(f"Created missing file: {path}")

async def load_found_links(file_name: str) -> Set[str]:
    try:
        async with aiofiles.open(file_name, 'r') as f:
            data = await f.read()
            items = json.loads(data) if data else []
            return set(items)
    except FileNotFoundError:
        # Create file quietly and return empty set
        logging.info(f"File not found, creating: {file_name}")
        await ensure_file_exists(file_name)
        return set()
    except json.JSONDecodeError:
        logging.warning(f"Corrupt JSON in {file_name}, resetting.")
        await ensure_file_exists(file_name)
        return set()

async def save_found_links(file_name: str, found_links: Set[str]):
    async with aiofiles.open(file_name, 'w') as f:
        await f.write(json.dumps(sorted(found_links)))
    logging.info(f"Links saved to file: {file_name}")

async def send_message(text: str, url: str):
    message_text = f"<a href='{url}'>{text}</a>"
    send_url = f'https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage'
    payload = {'chat_id': CHAT_ID, 'text': message_text, 'parse_mode': 'HTML'}
    timeout = aiohttp.ClientTimeout(total=20)

    async with aiohttp.ClientSession(timeout=timeout) as session:
        async with session.post(send_url, data=payload) as response:
            if response.status == 200:
                logging.info(f"Message sent: {text}")
            else:
                resp_text = await response.text()
                logging.error(f"Failed to send message: {resp_text}")

def build_ssl_context() -> ssl.SSLContext:
    """Use certifi to provide a proper CA bundle for TLS verification."""
    ctx = ssl.create_default_context(cafile=certifi.where())
    # Hardening (optional):
    ctx.check_hostname = True
    ctx.verify_mode = ssl.CERT_REQUIRED
    return ctx

def common_headers() -> dict:
    return {
        'User-Agent': random.choice(USER_AGENTS),
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'es-ES,es;q=0.9,en;q=0.8',
        'Referer': 'https://as.com/',
        'Connection': 'keep-alive',
    }

async def fetch_with_retries(session: aiohttp.ClientSession, url: str) -> str:
    """
    Fetch URL with retries. First tries with verified SSL.
    If we hit an SSL verification error, retry once with ssl=False (last-resort).
    """
    headers = common_headers()
    backoff = 1.0

    for attempt in range(1, MAX_RETRIES + 1):
        try:
            await asyncio.sleep(random.uniform(0.25, 1.0))  # polite jitter
            async with session.get(url, headers=headers) as resp:
                resp.raise_for_status()
                return await resp.text()
        except (aiohttp.ClientConnectorCertificateError, ssl.SSLCertVerificationError) as e:
            logging.warning(f"SSL verify error on attempt {attempt} for {url}: {e}")
            # last-resort single retry with ssl=False
            try:
                async with session.get(url, headers=headers, ssl=False) as resp:
                    resp.raise_for_status()
                    return await resp.text()
            except Exception as e2:
                logging.error(f"SSL-disabled retry failed for {url}: {e2}")
        except (aiohttp.ClientResponseError, aiohttp.ClientConnectorError, aiohttp.ServerTimeoutError, asyncio.TimeoutError) as e:
            logging.warning(f"Request error on attempt {attempt} for {url}: {e}")
        except Exception as e:
            logging.error(f"Unexpected error on attempt {attempt} for {url}: {e}")

        await asyncio.sleep(backoff + random.uniform(0, 0.5))
        backoff = min(backoff * 2, 8)

    raise RuntimeError(f"Failed to fetch after {MAX_RETRIES} attempts: {url}")

async def process_url(session: aiohttp.ClientSession, url: str, found_links: Set[str]) -> List[str]:
    new_links: List[str] = []
    try:
        html = await fetch_with_retries(session, url)
        soup = BeautifulSoup(html, 'html.parser')

        # as.com list pages: headlines often under h2.s__tl > a
        h2_elements = soup.find_all('h2', class_='s__tl')

        for h2 in h2_elements:
            link = h2.find('a', href=True)
            if not link:
                continue
            href = link['href']
            if not href.startswith('http'):
                href = f'https://as.com{href}'
            title = link.get_text(strip=True)

            if href not in found_links:
                await send_message(title, href)
                new_links.append(href)

        if not h2_elements:
            logging.info(f"No expected h2.s__tl elements found on {url} (layout may differ).")

    except Exception as e:
        logging.error(f"Error processing URL: {url}. Error: {str(e)}")
    return new_links

async def main():
    await ensure_file_exists(FOUND_LINKS_FILE)
    found_links = await load_found_links(FOUND_LINKS_FILE)

    ssl_ctx = build_ssl_context()
    timeout = aiohttp.ClientTimeout(total=30, connect=15)

    while True:
        # Use a session with a verified SSL context (certifi)
        connector = aiohttp.TCPConnector(ssl=ssl_ctx, enable_cleanup_closed=True, ttl_dns_cache=300)
        async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session:
            tasks = [process_url(session, url, found_links) for url in URLS]
            results = await asyncio.gather(*tasks)

        new_found_links = set()
        for new_links in results:
            new_found_links.update(new_links)

        if new_found_links:
            found_links.update(new_found_links)
            await save_found_links(FOUND_LINKS_FILE, found_links)

        logging.info("Waiting for the next cycle...")
        await asyncio.sleep(CHECK_INTERVAL_SECONDS)

if __name__ == '__main__':
    asyncio.run(main())

What changed:

✅ Auto-creates found_links2.json on first run (no warning).
✅ Proper TLS verification via certifi; retries with exponential backoff.
✅ One fallback attempt per URL with ssl=False if the cert chain still fails.
✅ Stricter timeouts, friendlier headers, and small random delays.
✅ Safer JSON load/save and corruption handling.

If you prefer to never disable SSL even as a fallback, I can strip that path and keep it strictly verified-only.

import asyncio import aiohttp from bs4 import Beau...

Pregunta

Set up logging

=== Telegram Bot Settings ===

List of URLs to monitor

List of User-Agents

Initialize Translator object

Respuesta

Pregunta

Pensamiento

Fixing errors and ensuring security

Proposing solution improvements

Respuesta

Compartir este Q&A