|
|
@@ -42,7 +42,7 @@ else:
|
|
|
|
|
|
|
|
|
def open_browser(target_urls):
|
|
|
- all_data = {}
|
|
|
+ # all_data = {}
|
|
|
|
|
|
for target_url in target_urls:
|
|
|
pages = '/{}'
|
|
|
@@ -115,10 +115,11 @@ def open_browser(target_urls):
|
|
|
page.close()
|
|
|
browser.close()
|
|
|
|
|
|
- all_data[title] = urls
|
|
|
+ # all_data[title] = urls # 所有数据爬取完成再存
|
|
|
+ save_data({title: urls})
|
|
|
|
|
|
# 获取所有 url 数据之后, 存数据库
|
|
|
- return all_data
|
|
|
+ # return all_data
|
|
|
|
|
|
|
|
|
def download_img(load_data, target_file_path):
|
|
|
@@ -325,8 +326,7 @@ if __name__ == "__main__":
|
|
|
print(f'目标链接是:{targets}')
|
|
|
|
|
|
if step == 1:
|
|
|
- all_data = open_browser(targets)
|
|
|
- save_data(all_data)
|
|
|
+ open_browser(targets)
|
|
|
elif step == 2:
|
|
|
# 开始读取数据
|
|
|
load_data = load_data()
|
|
|
@@ -337,8 +337,7 @@ if __name__ == "__main__":
|
|
|
print('下载完成, 程序退出')
|
|
|
elif step == 3:
|
|
|
# 保存 img 链接
|
|
|
- all_data = open_browser(targets)
|
|
|
- save_data(all_data)
|
|
|
+ open_browser(targets)
|
|
|
|
|
|
# 开始读取数据
|
|
|
load_data = load_data()
|