ResourceCollection/flaticon/step0_get_flaticon_all_url.py

# -*- coding: utf-8 -*-

import re
import time
import random

import httpx

all_urls = []
for i in range(151, 201):
    url = 'https://www.flaticon.com/stickers-packs/{}'.format(i)

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
    }

    resp = httpx.get(url, headers=headers)
    resp.encoding = 'utf-8'
    page = resp.text
    url_list = re.findall('data-pin-url="(.*?)"', page)

    for u in url_list:
        if '{{link}}' not in u:
            all_urls.append(u)

    time.sleep(random.uniform(1, 2))

with open('url_file_0_flaticon_urls.txt', 'w', encoding='utf-8') as file:
    for url in all_urls:
        file.write(url + '\n')