++

flaticon/flaticon.py

@@ -46,11 +46,12 @@ def open_browser(target_urls):
 
     link_count = 1
     for target_url in target_urls:
-        print(f'开始获取 {target_url} 数据\n当前链接是第 {link_count} 个\n共 {len(target_urls)} 个链接')
+        print(f'\n开始获取 {target_url} 数据, 当前链接是第 {link_count} 个, 共 {len(target_urls)} 个链接')
         link_count += 1
         pages = '/{}'
         urls = []
         title = ''  # 存放当前页面的title
+        total_page_count = 0  # 合集总共页数
         with sync_playwright() as playwright:
             if local_proxy:
                 browser = playwright.chromium.launch(
@@ -67,18 +68,11 @@ def open_browser(target_urls):
             for page_count in range(1, 999):
                 try:
                     goto_url = target_url + pages.format(page_count)
-                    page.goto(goto_url, timeout=5000)
+                    page.goto(goto_url, timeout=8000)
                 except Exception as e:
-                    print(e)
-                    print(f'页面加载失败：url：{goto_url}')
-
-                try:
-                    # 检查一下当前页面是不是 404
-                    page.wait_for_selector(not_find_page_selector, state="attached", timeout=2000)
-                    print(f'总页数是 {page_count - 1} 在 url: {goto_url}')
-                    break
-                except:
                     pass
+                    # print(e)
+                    # print(f'页面加载失败：url：{goto_url}')
 
                 if page_count == 1:
                     # 获取title
@@ -91,6 +85,14 @@ def open_browser(target_urls):
                     invalid_chars = ['<', '>', ':', '"', '/', '\\', '|', '?', '*', '.', '  ', 'Icon Pack ']
                     for char in invalid_chars:
                         title = title.replace(char, '')
+                else:
+                    try:
+                        # 检查一下当前页面是不是 404
+                        page.wait_for_selector(not_find_page_selector, state="attached", timeout=2000)
+                        total_page_count = page_count - 1
+                        break
+                    except:
+                        pass
 
                 for i in range(1, img_count + 1):
                     # 选择所有的<a>标签
@@ -113,13 +115,14 @@ def open_browser(target_urls):
                             img_sequence_num += 1
                             break
 
-            print(f'所有图片URL已获取。总共图片 {len(urls)}')
+            print(f'所有图片URL已获取。总页数: 共 {total_page_count} 页, 总共图片 {len(urls)},  正在写入数据库...')
 
             page.close()
             browser.close()
 
             # all_data[title] = urls  # 所有数据爬取完成再存
             save_data({title: urls})
+        print(f'{title} 已保存')
 
     # 获取所有 url 数据之后, 存数据库
     # return all_data
@@ -204,9 +207,9 @@ def save_data(data_item):
                     data['url']
                 ))
                 conn.commit()
-                print(f"数据 {data['url']} 插入成功")
+                # print(f"数据 {data['url']} 保存成功")
             else:
-                print(f"数据 {data['url']} 已存在，未插入")
+                print(f"数据 {data['url']} 已存在，跳过")
 
     # 关闭数据库连接
     cursor.close()
@@ -326,8 +329,6 @@ if __name__ == "__main__":
                 print('在 target_link.txt 中未找到目标链接')
                 exit(0)
 
-    print(f'目标链接是：{targets}')
-
     if step == 1:
         open_browser(targets)
     elif step == 2:

flaticon/target_link.txt

@@ -1,2 +1,86 @@
-https://www.flaticon.com/packs/editorial-design-24
-https://www.flaticon.com/packs/space-347
+https://www.flaticon.com/packs/plumbing-22
+https://www.flaticon.com/packs/travel-agent
+https://www.flaticon.com/packs/allergies-73
+https://www.flaticon.com/packs/bakery-210
+https://www.flaticon.com/packs/world-aids-day-12
+https://www.flaticon.com/packs/take-away-67
+https://www.flaticon.com/packs/online-learning-247
+https://www.flaticon.com/packs/sakura-festival-22
+https://www.flaticon.com/packs/ecology-and-environment-21
+https://www.flaticon.com/packs/emergency-service-4
+https://www.flaticon.com/packs/home-screen-apps-32
+https://www.flaticon.com/packs/summer-380
+https://www.flaticon.com/packs/accommodation-and-hotel-2
+https://www.flaticon.com/packs/notification-alert-2
+https://www.flaticon.com/packs/notification-alert-3
+https://www.flaticon.com/packs/using-technology-3
+https://www.flaticon.com/packs/bicycle-element-11
+https://www.flaticon.com/packs/international-food-24
+https://www.flaticon.com/packs/food-and-delivery-12
+https://www.flaticon.com/packs/bear-3
+https://www.flaticon.com/packs/baby-208
+https://www.flaticon.com/packs/international-day-of-families-8
+https://www.flaticon.com/packs/emergency-services-68
+https://www.flaticon.com/packs/dogs-36
+https://www.flaticon.com/packs/summer-party-51
+https://www.flaticon.com/packs/womens-day-113
+https://www.flaticon.com/packs/mobile-technology-19
+https://www.flaticon.com/packs/chat-52
+https://www.flaticon.com/packs/machine-learning-9
+https://www.flaticon.com/packs/cosmetic-21
+https://www.flaticon.com/packs/approval-7
+https://www.flaticon.com/packs/buenos-aires-4
+https://www.flaticon.com/packs/maps-and-navigation-41
+https://www.flaticon.com/packs/electronics-157
+https://www.flaticon.com/packs/currency-243
+https://www.flaticon.com/packs/productivity-29
+https://www.flaticon.com/packs/neighborhood-watch
+https://www.flaticon.com/packs/remote-collaboration
+https://www.flaticon.com/packs/remote-working
+https://www.flaticon.com/packs/lawmaker
+https://www.flaticon.com/packs/files-77
+https://www.flaticon.com/packs/marketing-303
+https://www.flaticon.com/packs/automotive-ecommerce
+https://www.flaticon.com/packs/rental-3
+https://www.flaticon.com/packs/natural-disaster-70
+https://www.flaticon.com/packs/cloud-algorithm-7
+https://www.flaticon.com/packs/luck-19
+https://www.flaticon.com/packs/photo-editing-tools-28
+https://www.flaticon.com/packs/recession-13
+https://www.flaticon.com/packs/summer-424
+https://www.flaticon.com/packs/digital-transformation-19
+https://www.flaticon.com/packs/daily-planner-23
+https://www.flaticon.com/packs/online-learning-262
+https://www.flaticon.com/packs/online-learning-263
+https://www.flaticon.com/packs/copywriting-110
+https://www.flaticon.com/packs/big-data-analysis
+https://www.flaticon.com/packs/graphic-design-277
+https://www.flaticon.com/packs/photo-stickers-2
+https://www.flaticon.com/packs/summer-434
+https://www.flaticon.com/packs/automotive-ecommerce-2
+https://www.flaticon.com/packs/dentist-111
+https://www.flaticon.com/packs/home-security-31
+https://www.flaticon.com/packs/health-checkup-29
+https://www.flaticon.com/packs/legal-laws-justice-1
+https://www.flaticon.com/packs/genius-12
+https://www.flaticon.com/packs/summer-446
+https://www.flaticon.com/packs/multimedia-and-entertainment-19
+https://www.flaticon.com/packs/tropical-flower-design-7
+https://www.flaticon.com/packs/vacation-73
+https://www.flaticon.com/packs/music-248
+https://www.flaticon.com/packs/rc-racing-4
+https://www.flaticon.com/packs/debt-21
+https://www.flaticon.com/packs/baby-212
+https://www.flaticon.com/packs/water-park-74
+https://www.flaticon.com/packs/health-165
+https://www.flaticon.com/packs/miscellany-health-4
+https://www.flaticon.com/packs/music-262
+https://www.flaticon.com/packs/miscellany-health-8
+https://www.flaticon.com/packs/neuromarketing-3
+https://www.flaticon.com/packs/photography-239
+https://www.flaticon.com/packs/car-repair-43
+https://www.flaticon.com/packs/quit-smoking-64
+https://www.flaticon.com/packs/nuclear-energy-66
+https://www.flaticon.com/packs/spring-avatars-20
+https://www.flaticon.com/packs/smart-city-81
+https://www.flaticon.com/packs/bags-7

test/get_flaticon_all_url.py

@@ -7,7 +7,7 @@ import random
 import httpx
 
 all_urls = []
-for i in range(1, 10):
+for i in range(21, 30):
     url = 'https://www.flaticon.com/lineal-color-icons/{}'.format(i)
 
     headers = {
@@ -25,6 +25,6 @@ for i in range(1, 10):
 
     time.sleep(random.uniform(1, 2))
 
-with open('urls.txt', 'w', encoding='utf-8') as file:
+with open('flaticon_urls.txt', 'w', encoding='utf-8') as file:
     for url in all_urls:
         file.write(url + '\n')

test/urls.txt

@@ -1,216 +0,0 @@
-https://www.flaticon.com/packs/social-media-87
-https://www.flaticon.com/packs/mentoring-and-training-2
-https://www.flaticon.com/packs/education-633
-https://www.flaticon.com/packs/alphabet-and-numbers-20
-https://www.flaticon.com/packs/discussion-39
-https://www.flaticon.com/packs/planning-and-organization-calendars-2
-https://www.flaticon.com/packs/nature-196
-https://www.flaticon.com/packs/food-delivery-187
-https://www.flaticon.com/packs/contact-us-194
-https://www.flaticon.com/packs/origami-24
-https://www.flaticon.com/packs/art-design-35
-https://www.flaticon.com/packs/bubble-tea-2
-https://www.flaticon.com/packs/birthday-240
-https://www.flaticon.com/packs/electrician-tools-and-elements-43
-https://www.flaticon.com/packs/internet-security-110
-https://www.flaticon.com/packs/birthday-237
-https://www.flaticon.com/packs/museum-163
-https://www.flaticon.com/packs/hygiene-routine-67
-https://www.flaticon.com/packs/kids-activities-12
-https://www.flaticon.com/packs/fashion-week-2
-https://www.flaticon.com/packs/teenager-2
-https://www.flaticon.com/packs/farming-162
-https://www.flaticon.com/packs/presentation-30
-https://www.flaticon.com/packs/health-checkup-26
-https://www.flaticon.com/packs/summer-370
-https://www.flaticon.com/packs/contacts-communication-1
-https://www.flaticon.com/packs/learning-266
-https://www.flaticon.com/packs/international-childrens-day-8
-https://www.flaticon.com/packs/language-learning-5
-https://www.flaticon.com/packs/human-body-25
-https://www.flaticon.com/packs/space-347
-https://www.flaticon.com/packs/farming-158
-https://www.flaticon.com/packs/frequently-asked-questions-faq-11
-https://www.flaticon.com/packs/creative-tools-53
-https://www.flaticon.com/packs/south-korea-7
-https://www.flaticon.com/packs/technology-186
-https://www.flaticon.com/packs/birthday-234
-https://www.flaticon.com/packs/happiness-72
-https://www.flaticon.com/packs/cyberpunk-15
-https://www.flaticon.com/packs/entertainment-103
-https://www.flaticon.com/packs/coding-32
-https://www.flaticon.com/packs/ninja-55
-https://www.flaticon.com/packs/ocean-23
-https://www.flaticon.com/packs/product-features-9
-https://www.flaticon.com/packs/world-pride-day-41
-https://www.flaticon.com/packs/party-and-celebration-65
-https://www.flaticon.com/packs/jobs-and-professions-avatars-22
-https://www.flaticon.com/packs/digital-nomad-44
-https://www.flaticon.com/packs/blogger-and-influencer-62
-https://www.flaticon.com/packs/cardiovascular-disease-2
-https://www.flaticon.com/packs/education-665
-https://www.flaticon.com/packs/task-and-project-management-10
-https://www.flaticon.com/packs/yokai-8
-https://www.flaticon.com/packs/audio-and-video-43
-https://www.flaticon.com/packs/physics-49
-https://www.flaticon.com/packs/social-media-slang-bubbles-7
-https://www.flaticon.com/packs/international-day-of-yoga-9
-https://www.flaticon.com/packs/business-799
-https://www.flaticon.com/packs/happiness-74
-https://www.flaticon.com/packs/success-96
-https://www.flaticon.com/packs/magic-111
-https://www.flaticon.com/packs/japan-146
-https://www.flaticon.com/packs/family-life-70
-https://www.flaticon.com/packs/document-and-paper-1
-https://www.flaticon.com/packs/human-body-28
-https://www.flaticon.com/packs/gaming-pc-3
-https://www.flaticon.com/packs/career-women-4
-https://www.flaticon.com/packs/editorial-design-24
-https://www.flaticon.com/packs/automotive-ecommerce-1
-https://www.flaticon.com/packs/network-and-sharing-2
-https://www.flaticon.com/packs/medicine-108
-https://www.flaticon.com/packs/indoor-plants-2
-https://www.flaticon.com/packs/work-in-progress-2
-https://www.flaticon.com/packs/cinema-184
-https://www.flaticon.com/packs/labour-day-116
-https://www.flaticon.com/packs/ecommerce-593
-https://www.flaticon.com/packs/agile-32
-https://www.flaticon.com/packs/friendship-108
-https://www.flaticon.com/packs/wedding-439
-https://www.flaticon.com/packs/fair-trade-13
-https://www.flaticon.com/packs/education-topics-1
-https://www.flaticon.com/packs/grocery-104
-https://www.flaticon.com/packs/voting-elections-32
-https://www.flaticon.com/packs/superhero-44
-https://www.flaticon.com/packs/vegan-62
-https://www.flaticon.com/packs/aviation-92
-https://www.flaticon.com/packs/accounting-86
-https://www.flaticon.com/packs/new-media
-https://www.flaticon.com/packs/festivals-and-holidays-2
-https://www.flaticon.com/packs/bakery-226
-https://www.flaticon.com/packs/police-88
-https://www.flaticon.com/packs/festa-junina-46
-https://www.flaticon.com/packs/music-242
-https://www.flaticon.com/packs/language-learning-14
-https://www.flaticon.com/packs/gastronomy-66
-https://www.flaticon.com/packs/hotel-service-87
-https://www.flaticon.com/packs/nursing-home-2
-https://www.flaticon.com/packs/family-130
-https://www.flaticon.com/packs/city-elements-107
-https://www.flaticon.com/packs/mexico-87
-https://www.flaticon.com/packs/youth-25
-https://www.flaticon.com/packs/desert-86
-https://www.flaticon.com/packs/pets-106
-https://www.flaticon.com/packs/travel-483
-https://www.flaticon.com/packs/international-day-of-families-5
-https://www.flaticon.com/packs/international-day-of-yoga-14
-https://www.flaticon.com/packs/workout-app-8
-https://www.flaticon.com/packs/birthday-232
-https://www.flaticon.com/packs/college-72
-https://www.flaticon.com/packs/advertisement-46
-https://www.flaticon.com/packs/cleaning-236
-https://www.flaticon.com/packs/fashion-accessories-7
-https://www.flaticon.com/packs/international-childrens-day-14
-https://www.flaticon.com/packs/reward-and-badges-28
-https://www.flaticon.com/packs/climate-change-111
-https://www.flaticon.com/packs/web-buttons-23
-https://www.flaticon.com/packs/people-diversity
-https://www.flaticon.com/packs/greenhouse-33
-https://www.flaticon.com/packs/taxi-service-67
-https://www.flaticon.com/packs/essentials-118
-https://www.flaticon.com/packs/seo-and-sem-19
-https://www.flaticon.com/packs/wrestling-2
-https://www.flaticon.com/packs/travel-531
-https://www.flaticon.com/packs/cinco-de-mayo-48
-https://www.flaticon.com/packs/domotics-52
-https://www.flaticon.com/packs/tropical-79
-https://www.flaticon.com/packs/customer-self-service-2
-https://www.flaticon.com/packs/world-cuisine-2
-https://www.flaticon.com/packs/cyber-crimes-50
-https://www.flaticon.com/packs/family-133
-https://www.flaticon.com/packs/wedding-442
-https://www.flaticon.com/packs/mind-process-20
-https://www.flaticon.com/packs/butcher-69
-https://www.flaticon.com/packs/selection-and-cursors-66
-https://www.flaticon.com/packs/sales-140
-https://www.flaticon.com/packs/fathers-day-56
-https://www.flaticon.com/packs/bill-and-payment-method-2
-https://www.flaticon.com/packs/addictions-34
-https://www.flaticon.com/packs/greek-alphabet-2
-https://www.flaticon.com/packs/diet-and-nutrition-11
-https://www.flaticon.com/packs/birds-36
-https://www.flaticon.com/packs/medicine-112
-https://www.flaticon.com/packs/world-cuisine-1
-https://www.flaticon.com/packs/banking-182
-https://www.flaticon.com/packs/copywriting-105
-https://www.flaticon.com/packs/herbs-and-spices-8
-https://www.flaticon.com/packs/morning-routine-89
-https://www.flaticon.com/packs/world-pride-day-32
-https://www.flaticon.com/packs/edutainment-8
-https://www.flaticon.com/packs/buenos-aires
-https://www.flaticon.com/packs/hindu-gods-18
-https://www.flaticon.com/packs/coronavirus-249
-https://www.flaticon.com/packs/tiger-2
-https://www.flaticon.com/packs/in-the-village-63
-https://www.flaticon.com/packs/space-337
-https://www.flaticon.com/packs/kids-costumes-7
-https://www.flaticon.com/packs/graduation-12
-https://www.flaticon.com/packs/coffee-shop-220
-https://www.flaticon.com/packs/sweets-and-candies-58
-https://www.flaticon.com/packs/dental-care-73
-https://www.flaticon.com/packs/gastronomy-63
-https://www.flaticon.com/packs/luck-16
-https://www.flaticon.com/packs/medieval-121
-https://www.flaticon.com/packs/business-model-canvas-7
-https://www.flaticon.com/packs/playground-96
-https://www.flaticon.com/packs/business-805
-https://www.flaticon.com/packs/hazardous-waste-and-materials-2
-https://www.flaticon.com/packs/biometrics-6
-https://www.flaticon.com/packs/cannabis-53
-https://www.flaticon.com/packs/fathers-day-55
-https://www.flaticon.com/packs/cinco-de-mayo-45
-https://www.flaticon.com/packs/ecommerce-609
-https://www.flaticon.com/packs/furniture-and-household-37
-https://www.flaticon.com/packs/mind-process-19
-https://www.flaticon.com/packs/summer-384
-https://www.flaticon.com/packs/magic-and-fairy-tale-2
-https://www.flaticon.com/packs/bear-1
-https://www.flaticon.com/packs/protection-and-security-60
-https://www.flaticon.com/packs/dogs-37
-https://www.flaticon.com/packs/contact-us-215
-https://www.flaticon.com/packs/music-store-lineal-color-3-of-3
-https://www.flaticon.com/packs/construction-business-3
-https://www.flaticon.com/packs/reward-and-badges-34
-https://www.flaticon.com/packs/gym-129
-https://www.flaticon.com/packs/japanese-culture-3
-https://www.flaticon.com/packs/argentina-18
-https://www.flaticon.com/packs/dragon-boat-festival-21
-https://www.flaticon.com/packs/coffee-119
-https://www.flaticon.com/packs/mall-58
-https://www.flaticon.com/packs/essentials-116
-https://www.flaticon.com/packs/homeware-47
-https://www.flaticon.com/packs/chat-51
-https://www.flaticon.com/packs/contact-us-213
-https://www.flaticon.com/packs/keys-and-locks-13
-https://www.flaticon.com/packs/digital-nomad-40
-https://www.flaticon.com/packs/online-marketing-80
-https://www.flaticon.com/packs/workout-app-4
-https://www.flaticon.com/packs/nanotechnology-18
-https://www.flaticon.com/packs/beauty-175
-https://www.flaticon.com/packs/company-structure-30
-https://www.flaticon.com/packs/graphic-design-272
-https://www.flaticon.com/packs/winery-8
-https://www.flaticon.com/packs/world-pride-day-54
-https://www.flaticon.com/packs/kids-costumes-9
-https://www.flaticon.com/packs/activism-15
-https://www.flaticon.com/packs/world-blood-donor-day-2
-https://www.flaticon.com/packs/victoria-day-2
-https://www.flaticon.com/packs/mental-health-25
-https://www.flaticon.com/packs/food-and-delivery-8
-https://www.flaticon.com/packs/animal-head-13
-https://www.flaticon.com/packs/workaholic-10
-https://www.flaticon.com/packs/text-editor-83
-https://www.flaticon.com/packs/online-game-15
-https://www.flaticon.com/packs/notes-and-tasks-32
-https://www.flaticon.com/packs/education-topics-3
-https://www.flaticon.com/packs/world-pride-day-44