laoxong/AstrBot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 修复一部分超限的报错

Browse Source 
perf: web search稳定性和精确度优化
This commit is contained in:
Soulter
2023-09-27 22:06:08 +08:00
parent 553f1b8d83
commit 08e7d4d0c6
4 changed files with 59 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/provider/provider_openai_official.py
+4 -3
View File
@@ -119,7 +119,7 @@ class ProviderOpenAIOfficial(Provider):
retry = 0
response = None
err = ''
while retry < 5:
while retry < 15:
try:
response = openai.ChatCompletion.create(
messages=req,
@@ -142,9 +142,10 @@ class ProviderOpenAIOfficial(Provider):
gu.log("token超限, 清空对应缓存")
self.session_dict[session_id] = []
cache_data_list, new_record, req = self.wrap(prompt, session_id)
elif 'Limit: 3 / min. Please try again in 20s.' in str(e):
time.sleep(60)
elif 'Limit: 3 / min. Please try again in 20s.' in str(e) or "OpenAI response error" in str(e):
time.sleep(30)
else:
time.sleep(5)
gu.log(str(e), level=gu.LEVEL_ERROR)
err = str(e)
retry+=1
model/provider/provider_rev_chatgpt.py
+2 -1
View File
@@ -152,7 +152,7 @@ class ProviderRevChatGPT(Provider):
res = ''
err_msg = ''
err_cnt = 0
while err_cnt < 5:
while err_cnt < 15:
try:
res = self.request_text(prompt, selected_revstat['obj'])
selected_revstat['busy'] = False
@@ -175,6 +175,7 @@ class ProviderRevChatGPT(Provider):
raise Exception("触发RevChatGPT请求频率限制。请1小时后再试，或者切换账号。")
gu.log(f"请求异常: {str(e)}", level=gu.LEVEL_WARNING, tag="RevChatGPT")
err_cnt += 1
time.sleep(3)
raise Exception(f'回复失败。原因：{err_msg}。如果您设置了多个账号，可以使用/switch指令切换账号。输入/switch查看详情。')
util/func_call.py
+4 -3
View File
@@ -1,7 +1,7 @@
import json
import util.general_utils as gu
import time
class FuncCallJsonFormatError(Exception):
def __init__(self, msg):
self.msg = msg
@@ -48,7 +48,7 @@ class FuncCall():
funccall_prompt = """
我正实现function call功能，该功能旨在让你变成给定的问题到给定的函数的解析器（意味着你不是创造函数）。
下面会给你提供可能用到的函数相关信息和一个问题，你需要将其转换成给定的函数调用。
- 你的返回信息只含json，严格仿照以下内容（不含注释）:
- 你的返回信息只含json，请严格仿照以下内容（不含注释），必须含有`res`,`func_call`字段:
```
{
"res": string // 如果没有找到对应的函数，那么你可以在这里正常输出内容。如果有，这里是空字符串。
@@ -111,7 +111,7 @@ class FuncCall():
invoke_func_res = ""
if len(res["func_call"]) > 0:
if "func_call" in res and len(res["func_call"]) > 0:
task_list = res["func_call"]
invoke_func_res_list = []
@@ -174,6 +174,7 @@ class FuncCall():
raise e
if "The message you submitted was too long" in str(e):
# 如果返回的内容太长了，那么就截取一部分
time.sleep(3)
invoke_func_res = invoke_func_res[:int(len(invoke_func_res) / 2)]
after_prompt = """
函数返回以下内容："""+invoke_func_res+"""
util/gplugin.py
+49 -15
View File
@@ -16,6 +16,7 @@ def special_fetch_zhihu(link: str) -> str:
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
}
response = requests.get(link, headers=headers)
response.encoding = "utf-8"
soup = BeautifulSoup(response.text, "html.parser")
if "zhuanlan.zhihu.com" in link:
@@ -32,12 +33,13 @@ def web_keyword_search_via_bing(keyword) -> str:
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
}
url = "https://cn.bing.com/search?q="+keyword
url = "https://www.bing.com/search?q="+keyword
_cnt = 0
_detail_store = []
while _cnt < 5:
try:
response = requests.get(url, headers=headers)
response.encoding = "utf-8"
soup = BeautifulSoup(response.text, "html.parser")
res = []
ols = soup.find(id="b_results")
@@ -51,19 +53,29 @@ def web_keyword_search_via_bing(keyword) -> str:
"desc": desc,
"link": link,
})
if len(_detail_store) < 2 and "zhihu.com" in link:
try:
_detail_store.append(special_fetch_zhihu(link)[:800])
except BaseException as e:
print(f"zhihu parse err: {str(e)}")
if len(res) >= 5: # 限制5条
break
if len(_detail_store) >= 3:
continue
# 爬取前两条的网页内容
if "zhihu.com" in link:
try:
_detail_store.append(special_fetch_zhihu(link)[100:800])
except BaseException as e:
print(f"zhihu parse err: {str(e)}")
else:
try:
_detail_store.append(fetch_website_content(link)[100:1000])
except BaseException as e:
print(f"fetch_website_content err: {str(e)}")
except Exception as e:
print(f"bing parse err: {str(e)}")
if len(res) == 0:
break
if len(_detail_store) > 0:
ret = f"{str(res)} \n来源知乎的具体资料: {str(_detail_store)}"
ret = f"{str(res)} \n具体网页内容: {str(_detail_store)}"
else:
ret = f"{str(res)}"
return str(ret)
@@ -71,6 +83,7 @@ def web_keyword_search_via_bing(keyword) -> str:
print(f"bing fetch err: {str(e)}")
_cnt += 1
time.sleep(1)
print("fail to fetch bing info, using sougou.")
return web_keyword_search_via_sougou(keyword)
@@ -96,9 +109,22 @@ def web_keyword_search_via_sougou(keyword) -> str:
"title": title,
"link": link,
})
except:
pass
ret = f"{str(res)} \n全部内容: {tidy_text(soup.text)}"
if len(res) >= 5: # 限制5条
break
except Exception as e:
gu.log(f"sougou parse err: {str(e)}", tag="web_keyword_search_via_sougou", level=gu.LEVEL_ERROR)
# 爬取网页内容
_detail_store = []
for i in res:
if _detail_store >= 3:
break
try:
_detail_store.append(fetch_website_content(i["link"])[100:1000])
except BaseException as e:
print(f"fetch_website_content err: {str(e)}")
ret = f"{str(res)}"
if len(_detail_store) > 0:
ret += f"\n网页内容: {str(_detail_store)}"
return ret
def fetch_website_content(url):
@@ -107,9 +133,10 @@ def fetch_website_content(url):
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
}
response = requests.get(url, headers=headers)
response.encoding = "utf-8"
soup = BeautifulSoup(response.text, "html.parser")
res = soup.text
res = res.replace("\n", "")
res = res.replace("\n", "").replace(" ", " ").replace("\r", "").replace("\t", "")
with open(f"temp_{time.time()}.html", "w", encoding="utf-8") as f:
f.write(res)
return res
@@ -117,20 +144,26 @@ def fetch_website_content(url):
def web_search(question, provider, session_id):
new_func_call = FuncCall(provider)
new_func_call.add_func("web_keyword_search_via_bing", [{
"type": "string",
"name": "keyword",
"brief": "必应搜索的关键词(分词，尽量保留所有信息)"
}],
}],
"在必应搜索引擎上搜索给定的关键词，并且返回第一页的搜索结果列表(标题,简介和链接)",
web_keyword_search_via_bing
)
new_func_call.add_func("fetch_website_content", [{
"type": "string",
"name": "url",
"brief": "网址"
}],
"获取网址的内容",
fetch_website_content
)
func_definition1 = new_func_call.func_dump()
question1 = f"{question} \n（只能调用一个函数。）"
try:
res1, has_func = new_func_call.func_call(question1, func_definition1, is_task=False, is_summary=False, session_id=session_id)
res1, has_func = new_func_call.func_call(question1, func_definition1, is_task=False, is_summary=False)
except BaseException as e:
res = provider.text_chat(question) + "\n(网页搜索失败, 此为默认回复)"
return res
@@ -153,6 +186,7 @@ def web_search(question, provider, session_id):
raise e
if "The message you submitted was too long" in str(e):
res2 = res2[:int(len(res2) / 2)]
time.sleep(3)
question3 = f"""请回答`{question}`问题。\n以下是相关材料，请直接拿此材料针对问题进行回答，再给参考链接, 参考链接首末有空格。```\n{res1}\n{res2}\n```\n"""
return res3
else: