Compare commits
2
Commits
74fb9698cd
...
c64c9479a9
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c64c9479a9 | ||
|
|
da8cea5e08 |
@@ -0,0 +1 @@
|
|||||||
|
aa=9999999
|
||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import os
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
@@ -22,8 +24,8 @@ class BookService:
|
|||||||
self.setting["targetDate"] = timestamp_to_datetime(target_ts)
|
self.setting["targetDate"] = timestamp_to_datetime(target_ts)
|
||||||
|
|
||||||
app_dir = find_project_root()
|
app_dir = find_project_root()
|
||||||
print(app_dir)
|
self.download_path = app_dir / "download" # 下载目录
|
||||||
self.download_path = app_dir.parent / "download" # 下载目录
|
print(self.download_path)
|
||||||
self.today_timestamp = get_today_timestamp()
|
self.today_timestamp = get_today_timestamp()
|
||||||
self.one_date_loading = False
|
self.one_date_loading = False
|
||||||
self.session = requests.session()
|
self.session = requests.session()
|
||||||
@@ -54,7 +56,8 @@ class BookService:
|
|||||||
name = name.replace(old, new)
|
name = name.replace(old, new)
|
||||||
return name
|
return name
|
||||||
|
|
||||||
def create_folder(self, parent_dir: Path, folder_name: str) -> Path: # 创建文件夹
|
@staticmethod
|
||||||
|
def create_folder(parent_dir: Path, folder_name: str) -> Path: # 创建文件夹
|
||||||
"""创建文件夹(如果不存在)并返回路径"""
|
"""创建文件夹(如果不存在)并返回路径"""
|
||||||
target = parent_dir / folder_name
|
target = parent_dir / folder_name
|
||||||
target.mkdir(parents=True, exist_ok=True)
|
target.mkdir(parents=True, exist_ok=True)
|
||||||
@@ -80,14 +83,19 @@ class BookService:
|
|||||||
return
|
return
|
||||||
yield from self.download_file(f"{self.setting["baseUrl"]}{i.select_one(
|
yield from self.download_file(f"{self.setting["baseUrl"]}{i.select_one(
|
||||||
'a')['href']}", name, i.select_one('a').string)
|
'a')['href']}", name, i.select_one('a').string)
|
||||||
|
# yield from self.download_file(f"{self.setting["baseUrl"]}{i.select_one(
|
||||||
|
# 'a')['href']}", name, self.legitimate_naming(i.select_one('a').string))
|
||||||
except RetryError as e:
|
except RetryError as e:
|
||||||
print('重试三次也不成功')
|
print('重试三次也不成功')
|
||||||
|
|
||||||
@retry(stop=stop_after_attempt(3))
|
@retry(stop=stop_after_attempt(3))
|
||||||
def download_file(self, file_url: str, dir_name: str, file_name: str): # 下载文件
|
def download_file(self, file_url: str, dir_name: str, file_name: str): # 下载文件
|
||||||
try:
|
try:
|
||||||
response = self.session.get(file_url, stream=True)
|
response = self.session.get(file_url, stream=True, allow_redirects=False)
|
||||||
response.raise_for_status()
|
if response.status_code in (301, 302, 307, 308): # 非200状态码,重新请求
|
||||||
|
file_url = response.headers["location"]
|
||||||
|
print(f'跳转页面了:{file_url}')
|
||||||
|
response = self.session.get(file_url, headers={"referer": file_url}, stream=True)
|
||||||
print(f"下载状态码:{response.status_code} {file_url}")
|
print(f"下载状态码:{response.status_code} {file_url}")
|
||||||
print(dir_name + "/" + file_name)
|
print(dir_name + "/" + file_name)
|
||||||
content_type = response.headers.get('Content-Type', '')
|
content_type = response.headers.get('Content-Type', '')
|
||||||
@@ -95,7 +103,8 @@ class BookService:
|
|||||||
yield f"event: mes_error\ndata: {self.setting["targetDate"]} {file_name}\n\n"
|
yield f"event: mes_error\ndata: {self.setting["targetDate"]} {file_name}\n\n"
|
||||||
print("不是可下载文件")
|
print("不是可下载文件")
|
||||||
return
|
return
|
||||||
target_dir = f"download/{self.setting["targetDate"]}/{dir_name}/{file_name}"
|
target_dir = self.download_path / self.setting["targetDate"] / dir_name / file_name
|
||||||
|
print(target_dir)
|
||||||
with open(target_dir, 'wb') as f:
|
with open(target_dir, 'wb') as f:
|
||||||
for chunk in response.iter_content(chunk_size=1024):
|
for chunk in response.iter_content(chunk_size=1024):
|
||||||
if chunk:
|
if chunk:
|
||||||
@@ -185,6 +194,7 @@ class BookService:
|
|||||||
return element.select_one(".by em span").string
|
return element.select_one(".by em span").string
|
||||||
|
|
||||||
def book_download(self):
|
def book_download(self):
|
||||||
|
print(os.getenv('aa'))
|
||||||
self.create_folder(self.download_path, self.setting["targetDate"]) # 创建文件夹
|
self.create_folder(self.download_path, self.setting["targetDate"]) # 创建文件夹
|
||||||
target_page = self.setting["totalPage"] - self.setting["targetCountdownPage"] # 目标页
|
target_page = self.setting["totalPage"] - self.setting["targetCountdownPage"] # 目标页
|
||||||
yield from self.section_page(target_page) # 前往目标页
|
yield from self.section_page(target_page) # 前往目标页
|
||||||
|
|||||||
Reference in New Issue
Block a user