monoai.tools

 2def domain_whois(domain:str):
 3
 4    """
 5    Get the whois for a given domain:
 6    
 7    Args:
 8        domain (string): the domain
 9    """
10
11    try:
12        from whois import whois
13    except ImportError:
14        raise ImportError("whois is not installed. Please install it with 'pip install whois'")
15
16    result = whois(domain)
17    return str(result)

 2def search_web_with_duckduckgo(query: str, max_results: int = 10, exclude_domains: list[str] = None):
 3    """Search the web using DuckDuckGo search engine.
 4    
 5    Args:
 6        query: The query to search for
 7        max_results: The maximum number of results to return. Default is 5
 8        exclude_domains: The domains to exclude from the search. Default is None
 9    
10    Returns:
11        A dictionary containing:
12            data: The search results as a list of dictionaries
13            text: The results merged into a single string        
14    """
15    search_engine = _DuckDuckGoSearch(max_results, exclude_domains)
16    response, text_response = search_engine.search(query)
17    return {"data": response, "text": text_response}

20def search_web_with_tavily(query: str, max_results: int = 10, exclude_domains: list[str] = None):
21    """Search the web using Tavily search engine.
22    
23    Args:
24        query: The query to search for
25        max_results: The maximum number of results to return. Default is 5
26        exclude_domains: The domains to exclude from the search. Default is None
27    
28    Returns:
29        A dictionary containing:
30            data: The search results as a list of dictionaries
31            text: The results merged into a single string        
32    """
33    search_engine = _TavilySearch(max_results, exclude_domains)
34    response, text_response = search_engine.search(query)
35    return {"data": response, "text": text_response}

 5def scrape_web_with_requests(url: str):
 6    """Scrape a webpage using the requests library.
 7    
 8    This function uses the requests library for basic web scraping.
 9    It's fast and lightweight but doesn't handle JavaScript-rendered content.
10    
11    Args:
12        url: The URL to scrape
13    
14    Returns:
15        A dictionary containing:
16            html: The HTML content of the page
17            text: The extracted text content from the page
18    
19    Raises:
20        requests.RequestException: If the request fails
21    """
22    scraper = _RequestsScraper()
23    response, text_response = scraper.scrape(url)
24    return {"html": response, "text": text_response}

27def scrape_web_with_selenium(url: str, headless: bool = True, wait_time: int = 10):
28    """Scrape a webpage using Selenium WebDriver.
29    
30    This function uses Selenium to handle dynamic content and JavaScript.
31    It's useful for websites that require JavaScript execution to load content.
32    
33    Args:
34        url: The URL to scrape
35        headless: Whether to run Chrome in headless mode. Default is True
36        wait_time: Maximum time to wait for elements to load in seconds. Default is 10
37    
38    Returns:
39        A dictionary containing:
40            html: The HTML content of the page after JavaScript execution
41            text: The extracted text content from the page
42    
43    Raises:
44        Exception: If scraping fails
45    """
46    scraper = _SeleniumScraper(headless=headless, wait_time=wait_time)
47    response, text_response = scraper.scrape(url)
48    return {"html": response, "text": text_response}

51def scrape_web_with_tavily(url: str, deep: bool = False):
52    """Scrape a webpage using the Tavily API.
53    
54    This function uses the Tavily API for advanced content extraction.
55    Tavily provides clean, structured content extraction but doesn't return raw HTML.
56    
57    Args:
58        url: The URL to scrape
59        deep: Whether to use advanced extraction mode. Default is False
60    
61    Returns:
62        A dictionary containing:
63            html: None (not available with Tavily)
64            text: The extracted and cleaned content from the page
65    
66    Raises:
67        Exception: If the Tavily API call fails
68    """
69    scraper = _TavilyScraper(deep=deep)
70    response, text_response = scraper.scrape(url)
71    return {"html": response, "text": text_response}

 74def scrape_web(url: str, engine: str = "requests", deep: bool = False):
 75    """Scrape a webpage using the specified engine.
 76    
 77    This is a convenience function that dispatches to the appropriate
 78    scraping function based on the engine parameter.
 79    
 80    Args:
 81        url: The URL to scrape
 82        engine: The engine to use (requests, tavily, selenium). Default is requests
 83        deep: If using tavily, whether to use the advanced extraction mode. Default is False
 84    
 85    Returns:
 86        A dictionary containing:
 87            html: The HTML content of the page (not available if using tavily)
 88            text: The content of the page merged into a single string
 89            
 90    Raises:
 91        ValueError: If an invalid engine is specified
 92    """
 93    
 94    if engine == "requests":
 95        return scrape_web_with_requests(url)
 96    elif engine == "tavily":
 97        return scrape_web_with_tavily(url, deep=deep)
 98    elif engine == "selenium":
 99        return scrape_web_with_selenium(url)
100    else:
101        raise ValueError(f"Invalid engine: {engine} (must be 'requests', 'tavily', or 'selenium')")