self_example/Spider/Chapter06_异步爬虫/asynchttp/aiohttpTest.py

65 lines
2.2 KiB
Python

# -*- encoding:utf-8 -*-
'''
@Author : dingjiawen
@Date : 2023/12/6 16:57
@Usage : aiohttp库的使用
@Desc :
@参考:https://github.dev/Python3WebSpider/AsyncTest demo12
'''
import aiohttp
import asyncio
async def fetch(session, url):
async with session.get(url) as response:
return await response.text(), response.status
async def main():
async with aiohttp.ClientSession() as session:
html, status = await fetch(session, 'https://cuiqingcai.com')
print(f'html: {html[:100]}...')
print(f'status: {status}')
# 给url参数
async def main1():
params = {'name': 'germey', 'age': 25}
async with aiohttp.ClientSession() as session:
async with session.get('https://httpbin.org/get', params=params) as response:
print(await response.text())
'''
session还支持其他请求类型:
session.post('https://httpbin.org/post', data=b'data')
session.put('https://httpbin.org/put', data=b'data')
session.delete('https://httpbin.org/delete')
session.head('https://httpbin.org/get')
session.options('https://httpbin.org/get')
session.patch('https://httpbin.org/patch', data=b'data')
'''
# 返回的response对象
async def main2():
data = {'name': 'germey', 'age': 25}
# 有些返回字段前面需要加await有些则不需要,原则是,如果返回的是一个协程对象(如async修饰的方法),
# 那么前面就要加await,具体可以看aiohttp的API,其链接为 https://docs.aiohttp.org/en/stable/client_reference.html
async with aiohttp.ClientSession() as session:
async with session.post('https://httpbin.org/post', data=data) as response:
print('status:', response.status)
print('headers:', response.headers)
print('body:', await response.text())
print('bytes:', await response.read())
print('json:', await response.json())
# 超时设置
async def main3():
timeout = aiohttp.ClientTimeout(total=0.1)
async with aiohttp.ClientSession(timeout=timeout) as session:
async with session.get('https://httpbin.org/get') as response:
print('status:', response.status)
if __name__ == '__main__':
asyncio.run(main2())