[python] URL 초당 200개 호출 (비동기)

 

 

 

import asyncio
import aiohttp
import time
from urllib import parse
import urllib3
import matplotlib.pyplot as plt
import numpy as np
from time import sleep
from datetime import datetime, timedelta


urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

async def fetch(session, url):
    try:
        async with session.get(url, ssl=False, timeout=aiohttp.ClientTimeout(total=600)) as response:
            return await response.text()
    except aiohttp.ClientConnectorError as e:
        print(f"Connection Error: {e}")
    except asyncio.TimeoutError:
        print("Request timed out")

async def fetch_all(urls):
    async with aiohttp.ClientSession() as session:
        tasks = []
        for url in urls:
            tasks.append(fetch(session, url))
        return await asyncio.gather(*tasks, return_exceptions=True)

async def main():
    url = "https://도메인.도메인.kr"  # 호출하려는 API URL
    urls = []

    with open(CSV_FILE) as data_file:
        for line in data_file:
            keyword = parse.quote(line.strip())
            urls.append(HOST + "/home/1.0/total/search?sort=RANK&inputKeyword=" + keyword + "&searchKeyword=" + keyword + "&page=1&perPage=20")
            if (len(urls) % CHUNK == 0):
                start_time = time.time()
                results = await fetch_all(urls)
                end_time = time.time()
                y.append((time.time() - start_time ) * 1000)
                print("Shot!!! ::: " + str(end_time))
                sleep(0.5)
                urls = []

#     print(f"Time taken: {end_time - start_time} seconds")
#     print(f"Number of responses: {len(results)}")

if __name__ == "__main__":
    y = []
    plt.rcParams['font.family'] = 'AppleGothic'

    now = datetime.now()
    start_time = now.strftime("%Y-%m-%d %H:%M:%S")
    CHUNK = 200
    HOST = "https://도메인.도메인.kr"
    CSV_FILE = "../keyword_10000.csv"
    asyncio.run(main())
    x = np.arange(1, len(y) + 1)
    plt.plot(x, y, label='응답시간 ms')  # 첫 번째 선, 레이블 추가
    plt.legend()
    plt.title('API 응답시간')
    plt.xlabel( str(CHUNK)+'건 씩 / '+str(len(y)) +'회 ')
    plt.ylabel('시간 ms')
    print("Api "+str(CHUNK)+"건 호출 평균 : " + str(round(np.mean(y), 2)))
    now = datetime.now()
    end_time = now.strftime("%Y-%m-%d %H:%M:%S")
    print("Run time :: " + start_time + " ~ " + end_time)
    # 그래프 보여주기
    plt.show()