monoai.tools

 2class WebSearch():
 3
 4    """
 5    WebSearch is a tool that allows you to search the web. 
 6    It uses the DuckDuckGo or Tavily API to search the web.
 7
 8    Examples
 9    --------
10
11    ```python
12    websearch = WebSearch(engine="duckduckgo", max_results=5)
13    result = websearch.search("What is the capital of France?")
14    print(result["text"]) # print the result merged into a single string
15    print(result["data"]) # print the result as a list of dictionaries
16    ```
17    """
18
19    def __init__(self, engine: str = "duckduckgo", max_results: int = 5, exclude_domains: list[str] = None):
20
21        """
22        Initialize the WebSearch tool.
23
24        Parameters:
25        ----------
26        engine: str, optional
27            The search engine to use (duckduckgo or tavily, default is duckduckgo)
28        max_results: int, optional
29            The maximum number of results to return (default is 5)
30        exclude_domains: list[str], optional
31            The domains to exclude from the search (default is None)
32        """
33
34        if engine == "duckduckgo":
35            self._engine = _DuckDuckGoSearch(max_results, exclude_domains)
36        elif engine == "tavily":
37            self._engine = _TavilySearch(max_results, exclude_domains)
38        else:
39            raise ValueError(f"Invalid engine: {engine} (must be 'duckduckgo' or 'tavily')")
40
41    def search(self, query: str):
42
43        """
44        Search the web.
45
46        Parameters:
47        ----------
48        query: str
49            The query to search for
50
51        Returns:
52        -------
53        dict
54            The text response from the search engine.
55        """
56        response, text_response = self._engine.search(query)
57        return {"data": response, "text": text_response}

 2class WebScraping():
 3
 4    """
 5    WebScraping is a tool that allows you to scrape the web.
 6    It uses the requests library to scrape the web.
 7
 8    Examples
 9    --------
10
11    ```python
12    webscraping = WebScraping()
13    result = webscraping.scrape("https://www.scrapethissite.com")
14    print(result["html"]) # print the html of the page
15    print(result["text"]) # print the content of the page merged into a single string
16    ```
17    """
18
19    def __init__(self, engine:str = "requests", deep:bool = False):
20
21        """
22        Initialize the WebScraping tool.
23
24        Parameters:
25        ----------
26        engine: str, optional
27            The engine to use (requests, tavily, selenium. Default is requests)
28        deep: bool, optional
29            If using tavily, whether to use the advanced extraction mode (default is False)
30        """
31
32        if engine == "requests":
33            self._engine = _RequestsScraper()
34        elif engine == "tavily":
35            self._engine = _TavilyScraper(deep=deep)
36        elif engine == "selenium":
37            self._engine = _SeleniumScraper()
38        else:
39            raise ValueError(f"Invalid engine: {engine} (must be 'requests', 'tavily', or 'selenium')")
40
41    def scrape(self, url: str):
42
43        """
44        Scrape a webpage.
45
46        Parameters:
47        ----------
48        url: str
49            The url to scrape
50
51        Returns:
52        -------
53        dict
54            The response from the scraper.
55            html: str
56            The html of the page (not available if using tavily)
57            text: str
58            The content of the page merged into a single string
59        """
60        response, text_response = self._engine.scrape(url)
61        return {"html": response, "text": text_response}