fix(download): 修复文件下载和错误处理功能
- 添加了对 text/plain 和 application/zip 格式的支持 - 在详情页面增加了 URL 输出调试信息 - 修复了跳转页面时的 URL 和 referer 处理逻辑 - 添加了下载失败时的错误事件推送和提示 - 在前端实现了错误消息的类型区分和显示 - 优化了侧边栏菜单组件的配置参数
This commit is contained in:
@@ -39,9 +39,11 @@ class BookService:
|
||||
print(f"文件夹:{name} 已存在")
|
||||
|
||||
def isDownloadTarget(self, soup): # 是否是可以下载的文件
|
||||
return soup.headers.get('Content-Type') in ['"application/octet-stream"', "application/octet-stream"]
|
||||
return soup.headers.get('Content-Type') in ['"application/octet-stream"', "application/octet-stream",
|
||||
"text/plain","application/zip"]
|
||||
|
||||
def post_page(self, name, url): # 帖子页面
|
||||
print(f"详情页面 {self.setting["baseUrl"]}{url}")
|
||||
response = requests.get(f"{self.setting["baseUrl"]}{url}", headers=self.headers)
|
||||
print(f"详情页面状态码:{str(response.status_code)}")
|
||||
soup = BeautifulSoup(response.text, 'lxml')
|
||||
@@ -66,17 +68,20 @@ class BookService:
|
||||
def download_file(self, file_url, dir_name, file_name): # 下载文件
|
||||
file_res = requests.get(file_url, headers=self.headers, stream=True, allow_redirects=False)
|
||||
if file_res.status_code in (301, 302, 307, 308): # 非200状态码,重新请求
|
||||
location = file_res.headers["location"]
|
||||
print(f'跳转页面了:{location}')
|
||||
file_res = requests.get(location, headers={"referer": location}, stream=True)
|
||||
file_url = file_res.headers["location"]
|
||||
print(f'跳转页面了:{file_url}')
|
||||
file_res = requests.get(file_url, headers={"referer": file_url}, stream=True)
|
||||
print(f"下载状态码:{file_res.status_code} {file_url}")
|
||||
print(dir_name + "/" + file_name)
|
||||
yield f"data: {self.setting["targetDate"]} {file_name}\n\n"
|
||||
if self.isDownloadTarget(file_res):
|
||||
with open(f"download/{self.setting["targetDate"]}/{dir_name}/{file_name}", 'wb') as file2:
|
||||
for chunk in file_res.iter_content(chunk_size=1024):
|
||||
if chunk:
|
||||
file2.write(chunk)
|
||||
yield f"data: {self.setting["targetDate"]} {file_name}\n\n"
|
||||
else:
|
||||
yield f"event: mes_error\ndata: {self.setting["targetDate"]} {file_name}\n\n"
|
||||
print("不是可下载文件")
|
||||
|
||||
def section_page(self, page): # 版块页面
|
||||
print(f"当前页面 {page}")
|
||||
@@ -105,11 +110,13 @@ class BookService:
|
||||
posts_max_timestamp = datetime_to_timestamp(self.get_post_date(posts[-1])) # 帖子列表最大时间戳
|
||||
|
||||
# 本页最早的帖子时间戳小于等于目标时间戳 且 倒计时页数大于 0 则跳转到上一页继续查找
|
||||
if posts_min_timestamp >= target_date_timestamp and not self.one_date_loading and self.setting["targetCountdownPage"] > 0:
|
||||
if posts_min_timestamp >= target_date_timestamp and not self.one_date_loading and self.setting[
|
||||
"targetCountdownPage"] > 0:
|
||||
print("跳转到上一页")
|
||||
# yield f"data: 跳转到上一页\n\n"
|
||||
self.setting["targetCountdownPage"] = max(self.setting["targetCountdownPage"] - 1, 0)
|
||||
Setting.query.filter_by(name="book_download").update({"targetCountdownPage": self.setting["targetCountdownPage"]})
|
||||
Setting.query.filter_by(name="book_download").update(
|
||||
{"targetCountdownPage": self.setting["targetCountdownPage"]})
|
||||
db.session.commit()
|
||||
yield from self.section_page(self.setting["totalPage"] - self.setting["targetCountdownPage"])
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user