fix: web search

util/agent/web_searcher.py

@@ -53,10 +53,13 @@ async def search_from_bing(keyword: str) -> str:
     logger.info("web_searcher - search_from_bing: " + keyword)
     results = await google.search(keyword, 5)
     if len(results) == 0:
+        logger.debug("search google failed")
         results = await bing_search.search(keyword, 5)
     if len(results) == 0:
+        logger.debug("search bing failed")
         results = await sogo_search.search(keyword, 5)
     if len(results) == 0:
+        logger.debug("search sogo failed")
         return "没有搜索到结果"
     ret = ""
     idx = 1

util/search_engine_scraper/engine.py

@@ -39,7 +39,6 @@ class SearchEngine():
         headers = self.headers
         headers["Referer"] = url
         headers["User-Agent"] = random.choice(USER_AGENTS)
-        print(headers)
         if data:
             async with ClientSession() as session:
                 async with session.post(url, headers=headers, data=data, timeout=self.TIMEOUT) as resp:

util/search_engine_scraper/google.py

@@ -1,3 +1,4 @@
+import os
 from googlesearch import search
 
 try:
@@ -10,12 +11,14 @@ except ImportError:
 from typing import List
 
 class Google(SearchEngine):
+    def __init__(self) -> None:
+        super().__init__()
+        self.proxy = os.environ.get("HTTPS_PROXY")
     
     async def search(self, query: str, num_results: int) -> List[SearchResult]:
-        index = 1
         results = []
         try:
-            ls = search(query, advanced=True, num_results=num_results, timeout=3)
+            ls = search(query, advanced=True, num_results=num_results, timeout=3, proxy=self.proxy)
             for i in ls:
                 results.append(SearchResult(title=i.title, url=i.url, snippet=i.description))
         except: