Python爬虫

a-c-dream 发布于

urlib 基本使用

1
2
3
4
5
6
7
8
9
10
11
import urllib.request
# 定义一个url
url = 'http://www.baidu.com'
# 模拟浏览器向服务器发送请求
reponse = urllib.request.urlopen(url)
# 获取响应中的源码
# read方法 返回字节形式的二进制数据
# 二进制 => 字符串 解码 decode('编码格式')
content = reponse.read().decode('utf-8')

print(content)

urllib_1个类型和6个方法

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
import urllib.request

url = 'http://www.baidu.com'

response = urllib.request.urlopen(url)

# response 的类型是HTTPResponse
print(type(response))

# 一个字节一个字节读
content = response.read()
print(content)

# 返回多少个字节
content  = response.read(5)
print(content)

# 读一行
content = response.readline().decode('utf-8')
print(content)


# content = response.readlines()
print(content)

# 返回状态码 200 表示成功
print(response.getcode())
# 返回URL地址
print(response.geturl())
# 返回响应头
print(response.getheaders())

urllib下载

1
2
3
4
5
6
7
8
9
10
11
from urllib import request
# 注意文件后缀名
# 下载网页
url_pege = 'http://www.baidu.com'
request.urlretrieve(url_pege,'baidu.html')
# 下载图片
url_img = 'https://qiniu.sukoshi.xyz/src/images/68686407_p0.jpg'
request.urlretrieve(url_img,'background.jpg')
# 下载视频
url_voide = 'https://v26-web.douyinvod.com/bfe545f287bb76d73ce757462aafe4f0/62b7dcc1/video/tos/cn/tos-cn-ve-15c001-alinc2/c8084eabd77548138985e2de8eae5a65/?a=6383&ch=0&cr=0&dr=0&cd=0%7C0%7C0%7C0&cv=1&br=4194&bt=4194&cs=0&ds=6&ft=X1nbLXvvBQxBUEayJ8Z.wNnOYZlc6oN_F2bLTJOn0iZm&mime_type=video_mp4&qs=0&rc=aWg1OTk3NzVoNGdoM2k4ZkBpM29zZWk6ZmV4ZDMzNGkzM0A1MWAxXzBjX2AxMTIyMS9fYSNwX2ZscjRfamBgLS1kLS9zcw%3D%3D'
request.urlretrieve(url_voide,'dy.mp4')

urllib请求对象的定制

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
from urllib import request
# url组成
# https://www.baidu.com/s?wd=abab
# http/https    www.baidu.com   80/443      s       wd=     #
# 协议            主机              端口号     路径      参数      锚点
# http  80
# https 443
url = 'https://www.baidu.com'

headers = {
    'User-Agent': ''}
# 因为urlopen方法中不能存储字典 所以headers不能传进去
# 请求对象的定制
# 因为参数顺序的问题 不能直接写url 和 headers 中间还有data 所以要用关键字传参
my_request = request.Request(url=url,headers=headers)
response = request.urlopen(my_request)

content = response.read().decode('utf-8')
print(content)

User-Agent寻找

按F12，在网络里找到请求标头
20220626122624

使用urlencode转化data

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
import urllib.request
import urllib.parse

base_url = 'https://www.baidu.com/s?'
data = {
    'wd':'周杰伦',
    'sex':'男',
    'location':'中国台湾省'

}
new_data = urllib.parse.urlencode(data)
url = base_url+new_data
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36 Edg/103.0.1264.37'}
my_request = urllib.request.Request(url=url,headers=headers)
response = urllib.request.urlopen(my_request)
content = response.read().decode('utf8')
print(content)

post请求

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
import json
import urllib.request
import urllib.parse

url = 'https://fanyi.baidu.com/sug'
data = {'kw':'hello'}
# post请求的参数，必须进行编码
data = urllib.parse.urlencode(data).encode('utf-8')
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36 Edg/103.0.1264.37'}
# post请求的参数是不会拼接在url后面的，而是需要放在请求对象定制的参数中

my_request = urllib.request.Request(url,data,headers)
response = urllib.request.urlopen(my_request)
content = response.read().decode('utf-8')
# string to json
obj = json.loads(content)
print(obj)

post请求 请求头cookie

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
import urllib.request
import urllib.parse
import json

url = 'https://fanyi.baidu.com/v2transapi?from=en&to=zh'
data = {'from': 'en',
        'to': 'zh',
        'query': 'he',
        'transtype': 'realtime',
        'simple_means_flag': '3',
        'sign': '6565.326292',
        'token': 'd69923d02b5ce21bd78eadec3d969c20',
        'domain': 'common', }
data = urllib.parse.urlencode(data).encode('utf-8')
headers = {'Accept': '*/*',
           # 'Accept-Encoding': 'gzip, deflate, br',
           'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
           'Acs-Token': '1656226807977_1656232769580_4qV3gbnD96ZBZYBAlsg9IqeVfDXXSHy35b1/cm1fi34m12+86eTJD/33zOvUatfvr3IDlzSTebCaAMd6gE30mXH626jp6Sj7oyg9aDFiNngEc2CCzQdHxOJR/Z80oioh1T5aUd8A7giOqce64x8jbU0SPeRYPsSEClfl3b5vPM+byBGCITngsrourA1pNU+DylYOkjqT1eb36wDrNAmWRciI08bgzYb/5w3g89Spd96EEBvbvvZnLuQpCfrFKtaKw5GcYhqW3WJnRB6LKdAPqwHkGhvD6I5i+uLGfdf0m3+TiZflKJ/BEnJkhHczPmaxpX8j0kOXWUPPno3KCk7Itg==',
           'Connection': 'keep-alive',
           'Content-Length': '131',
           'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
           'Cookie': 'BIDUPSID=1BA07B2602EA9BFFD701DCFA7F74EDD3; PSTM=1601728311; __yjs_duid=1_f495d725353b4ced5e1fecd1bc2a8b9d1619439168319; BAIDUID=994FA9AA5A632117AFF8C9A6D4673920:FG=1; H_WISE_SIDS=107311_110085_114550_127969_164869_174434_176399_179350_181536_184716_185268_186844_189037_189326_189755_190617_190791_191068_191257_191287_192206_192387_193283_193557_193824_194085_194519_195003_195329_195342_195537_195631_196045_196425_196444_197241_197383_197469_197711_197782_197957_198072_198264_198515_198650_199082_199305_199469_199569_199752_199754_200037_200128_200271_200444_200542_200549_200762_201042_201055_201186_201192_201233_201328_201445_201546_201554_201577_201581_201600_201702_201733_201818_201935_201970_201978_201983_201995_202113_202146_202177_202392_202478_202554_202562_202822_202832_202911_202917_202926; MAWEBCUID=web_NCzqSxWJFrHzfffIsIvelcJGRhbSaelrinZtupZLrMkJqbLiJr; MCITY=-173%3A; BDUSS=XJ3U2RBQUlrT3pVdUl4RlY4SzB0ZHE5Qjh5OFNySFRBbEVManItSWw3Vmg1WjlpRVFBQUFBJCQAAAAAAAAAAAEAAAAMcnVQzt7Lrtbz0-PBywAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFYeGJhWHhiO; BDUSS_BFESS=XJ3U2RBQUlrT3pVdUl4RlY4SzB0ZHE5Qjh5OFNySFRBbEVManItSWw3Vmg1WjlpRVFBQUFBJCQAAAAAAAAAAAEAAAAMcnVQzt7Lrtbz0-PBywAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGFYeGJhWHhiO; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; BDSFRCVID_BFESS=SI4OJeC62RoGgSODRdvPMAhuY7N1qPbTH6aVLmiSAG-CwvvCUvIEEG0PKM8g0KCMHnJ9ogKKymOTHuPF_2uxOjjg8UtVJeC6EG0Ptf8g0f5; H_BDCLCKID_SF_BFESS=tbIOoI0btC83qR5gMJ5q-n3HKUrL5t_XbI6y3JjOHJOoDDvbyjOcy4LdjG5N2MbML2ORbJbKLp6Nj-5XjUIaKq8V3-Aq5l5raC5g5l0E0-3-flOa5xrpQfbQ0-RuqP-jW27uanrFfR7JOpvwhfnxybLrQRPH-Rv92DQMVU52QqcqEIQHQT3m5-5bbN3ht6T2-DA_oDD-fxK; Hm_lvt_64ecd82404c51e03dc91cb9e8c025574=1656232679; APPGUIDE_10_0_2=1; REALTIME_TRANS_SWITCH=1; FANYI_WORD_SWITCH=1; HISTORY_SWITCH=1; SOUND_SPD_SWITCH=1; SOUND_PREFER_SWITCH=1; Hm_lpvt_64ecd82404c51e03dc91cb9e8c025574=1656232696; ab_sr=1.0.1_YThkYjM2ODhhYjgzMTYzYjVlMDE0ZmE5OTg1NzgwYzM5MTY5NDdlZjI2NTk2ZjFkZDA4MmM3MTJiMzlmNDcwZDU2MzE3NWZhZThjNGIyOGUxYjNlYWM2OTk1Y2MxMDkwZWU4NWFmOGQ0ODkwODI3YzU5ZTBmMzkzZWQyN2Q1MzM1MWQ2NTUzNGU1MTU4ZTU1MDBkOGZiMGQzNDUwNzgzNDVlMmU2Y2Y4ZjhmYTZmZjY5MWRiNjQ2Zjc5OTJiMjJh',
           'Host': 'fanyi.baidu.com',
           'Origin': 'https://fanyi.baidu.com',
           'Referer': 'https://fanyi.baidu.com/',
           'sec-ch-ua': '" Not;A Brand";v="99", "Microsoft Edge";v="103", "Chromium";v="103"',
           'sec-ch-ua-mobile': '?0',
           'sec-ch-ua-platform': '"Windows"',
           'Sec-Fetch-Dest': 'empty',
           'Sec-Fetch-Mode': 'cors',
           'Sec-Fetch-Site': 'same-origin',
           'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36 Edg/103.0.1264.37',
           'X-Requested-With': 'XMLHttpRequest', }
my_request = urllib.request.Request(url, data, headers)
respones = urllib.request.urlopen(my_request)
content = respones.read().decode('utf-8')
obj = json.loads(content)
print(obj)

ajax的get请求豆瓣电影第一页

使用open时要注意设置编码

1
2
3
4
5
6
7
8
9
10
import urllib.request
import urllib.parse
url = 'https://movie.douban.com/j/chart/top_list?type=24&interval_id=100%3A90&action=&start=0&limit=20'
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36 Edg/103.0.1264.37'}
request = urllib.request.Request(url=url,headers=headers)
response = urllib.request.urlopen(request)
content = response.read().decode('utf-8')
fp = open('douban.json','w',encoding='utf-8')
fp.write(content)
fp.close()

ajax的get请求豆瓣电影前十页

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
# https://movie.douban.com/j/chart/top_list?type=24&interval_id=100%3A90&action=&start=0&limit=20
# https://movie.douban.com/j/chart/top_list?type=24&interval_id=100%3A90&action=&start=20&limit=20
import urllib.request
import urllib.parse


def create_request(page):
    base_url = 'https://movie.douban.com/j/chart/top_list?type=24&interval_id=100%3A90&action=&'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36 Edg/103.0.1264.37'}
    data = {
        'start': (page - 1) * 20,
        'limit': 20
    }
    data = urllib.parse.urlencode(data)
    url = base_url + data

    request = urllib.request.Request(url=url, headers=headers)
    return request


def get_data(request):
    response = urllib.request.urlopen(request)
    content = response.read().decode('utf-8')
    return content


def save_data(page,content):
    with open('douban_' + str(page) + '.json', 'w', encoding='utf-8') as fp:
        fp.write(content)
        fp.close()


if __name__ == '__main__':
    start = eval(input('起始页'))
    end = eval(input('终止页'))
    for page in range(start, end + 1):
        request = create_request(page)
        content = get_data(request)
        save_data(page,content)

爬虫中的异常

简介:

1.HTTPError类是URLError类的子类
2.导入的包urllib.error.HTTPError  urllib.error.URLEtrror
3.http错误: http错误是针对浏览器无法连接到服务器而增加出来的错误提示。引导并告诉浏览者该页是哪里出了问题。
4.通过urllib发送请求的时候，有可能会发送失败，这个时候如果想让你的代码更加的健壮，可以通过try-except进行捕获异常，异常有两类，URLError\HTTPError

cookie登录

cookie
    中携带登录信息，有了登录后的cookie，可以通过携带者cookie进入到任何页面
referer
    判断当前路径是不是由上个路径进来的

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
import urllib.request
import urllib.parse

url = 'https://user.qzone.qq.com/1139439715/infocenter'
header ={
 # ':authority': 'user.qzone.qq.com',
 # ':method': 'GET',
 # ':path': '/1139439715/infocenter',
 # ':scheme': 'https',
 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
 # 'accept-encoding': 'gzip, deflate, br',
 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
 'cache-control': 'max-age=0',
 'cookie': 'pgv_pvid=2343005899; RK=vg6c0RK3NV; ptcz=fccdc91f3e752825b2e1748495af1dba3ffbdadb67e0a0ba7eba2544fa9eef7d; pgv_pvi=6892274688; tvfe_boss_uuid=f8970ec8c04a0d86; o_cookie=1139439715; QZ_FE_WEBP_SUPPORT=1; pac_uid=1_1139439715; _tc_unionid=588561c3-324c-4732-9419-1c7baabb1b00; cpu_performance_v8=1; uin=o1139439715; skey=@NJeZaUKii; p_uin=o1139439715; pt4_token=COV697sGUjsLBeRsnm2wFQaCc7w5jl0NQWN6H6alxUk_; p_skey=0bYvaeCqt*lqjqK4cv0d-UdnKRVZ04GNySozqHhjG6E_; Loading=Yes; qz_screen=2560x1440; pgv_info=ssid=s7761440768',
 'if-modified-since': 'Mon, 27 Jun 2022 02:07:06 GMT',
 'sec-ch-ua': '" Not;A Brand";v="99", "Microsoft Edge";v="103", "Chromium";v="103"',
 'sec-ch-ua-mobile': '?0',
 'sec-ch-ua-platform': '"Windows"',
 'sec-fetch-dest': 'document',
 'sec-fetch-mode': 'navigate',
 'sec-fetch-site': 'none',
 'sec-fetch-user': '?1',
 'upgrade-insecure-requests': '1',
 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36 Edg/103.0.1264.37', }
request = urllib.request.Request(url=url,headers=header)
response = urllib.request.urlopen(request)
content = response.read().decode('utf-8')
fp = open('qq.html','w',encoding='utf-8')
fp.write(content)
fp.close()


代理ip

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
import urllib.request
import urllib.parse
url = 'https://www.baidu.com/s?wd=ip'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36 Edg/103.0.1264.37'}
request = urllib.request.Request(url=url,headers=headers)
proxies ={
    'http':'140.124.72.74:8088'
}
handler = urllib.request.ProxyHandler(proxies=proxies)
opener = urllib.request.build_opener(handler)
response = opener.open(request)
cotent = response.read().decode('utf-8')
fp = open('ip.html','w',encoding='utf-8')
fp.write(cotent)
fp.close()

代理池

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
import urllib.request

proxies_pool = [{'http': '140.124.72.74:8088'}, {'http': '112.192.234.123:8088'},{'http':'211.138.121.37:82'}]
import random
proxies = random.choice(proxies_pool)
url = 'https://www.baidu.com/s?wd=ip'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36 Edg/103.0.1264.37'}
request = urllib.request.Request(url = url,headers=headers)
handler = urllib.request.ProxyHandler(proxies=proxies)
opener = urllib.request.build_opener(handler)
response = opener.open(request)
content = response.read().decode('utf-8')
fp = open('ip_pool.html','w',encoding='utf-8')
fp.write(content)
fp.close()

xpath解析

1.本地文件             etree.parse
2.服务器响应文件        etree.HTML

站长素材

部分图片网站会使用懒加载，这时src要换成src2

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
# https://sc.chinaz.com/tupian/wenzitupian.html
# https://sc.chinaz.com/tupian/wenzitupian_2.html
import urllib.request
from lxml import etree

def create_request(page):
    url = ''
    if page == 1:
        url = 'https://sc.chinaz.com/tupian/wenzitupian.html'
    else:
        url = 'https://sc.chinaz.com/tupian/wenzitupian_' + str(page) + '.html'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36 Edg/103.0.1264.37'}
    request = urllib.request.Request(url=url, headers=headers)
    return request


def get_data(request):
    response = urllib.request.urlopen(request)
    content = response.read().decode('utf-8')
    return content


def down_load(content):
    tree = etree.HTML(content)
    name_list = tree.xpath('//div//a/img/@alt')
    src_list = tree.xpath('//div//a/img/@src')
    for i in range(len(name_list)):
        urllib.request.urlretrieve('https:'+src_list[i],'./charImg/'+name_list[i]+'.jpg')



if __name__ == '__main__':
    start_page = eval(input('start_page:'))
    end_page = eval(input('end_page:'))
    for page in range(start_page, end_page + 1):
        request = create_request(page)
        content = get_data(request)
        down_load(content)

request get请求

使用参数params传递
参数不需要urlencode编码
不需要请求对象的定制
请求资源中的？可以删除

1
2
3
4
5
6
7
8
9
10
import requests
url = 'https://www.baidu.com/s?'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36 Edg/103.0.1264.37'}
data = {
    'wd':'北京'
}
response = requests.get(url=url,params=data,headers=headers)
content = response.text
print(content)

requsets post请求

post请求是不需要编码的
post请求的参数是data
不需要请求对象的定制

1
2
3
4
5
6
7
8
9
10
11
12
13
14
import requests
url = 'https://fanyi.baidu.com/sug'
data = {
    'kw': 'eye'
}
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36 Edg/103.0.1264.37'}
response = requests.post(url=url,headers=headers,data=data)
response.encoding='utf-8'
content = response.text
import json
obj = json.loads(content)
print(obj)

requests 登录古诗文网

使用session对象，避免多次访问而引起的验证码不一致。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
import requests

url = 'https://so.gushiwen.cn/user/login.aspx?from=http%3a%2f%2fso.gushiwen.cn%2fuser%2fcollect.aspx'


headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36 Edg/103.0.1264.37'}
# response = requests.get(url=url,headers=headers)
session = requests.session()
response = session.get(url)
content = response.text
from bs4 import BeautifulSoup
soup = BeautifulSoup(content,'lxml')
__VIEWSTATE = soup.select('#__VIEWSTATE')[0].attrs.get('value')
__VIEWSTATEGENERATOR = soup.select('#__VIEWSTATEGENERATOR')[0].attrs.get('value')
code_url = 'https://so.gushiwen.cn'+soup.select('#imgCode')[0].attrs.get('src')
response_code = session.get(code_url)
content_code = response_code.content
fp = open('code.jpg','wb')
fp.write(content_code)
fp.close()
code_txt = input('请输入验证码')
data = {
    '__VIEWSTATE': __VIEWSTATE,
    '__VIEWSTATEGENERATOR': __VIEWSTATEGENERATOR,
    'from': 'http://so.gushiwen.cn/user/collect.aspx',
    'email': '1139439715@qq.com',
    'pwd': '020603',
    'code': code_txt,
    'denglu': '登录',
}

final_response = session.post(url=url,headers=headers,data=data)
final_content = final_response.text
fp = open('login.html','w',encoding='utf-8')
fp.write(final_content)
fp.close()

scrapy

1.创建爬虫项目
    scarapy startproject 项目的名字
    注：项目名不能数字开头也不能包含中文
2.创建爬虫文件
    要在spiders文件夹中创建爬虫文件
    cd 项目名\项目名\spiders
    创建爬虫文件
    scrapy genspider 爬虫文件的名字 要爬取的网页
    要爬取的网页一般不加http协议
3.运行
    scrapy crawl 爬虫的名字

忽略证书警告

方法一

使用urllib3的disable_warnings方法并设置verify为False即可

1
2
3
4
import urllib3
urllib3.disable_warnings()
response = requests.get('https://ssr2.scrape.center/',verify=False)
print(response.status_code)

方法二

使用logging.captureWarnings方法并设置verify为False即可

1
2
3
4
import logging
logging.captureWarnings(True)
response = requests.get('https://ssr2.scrape.center/',verify=False)
print(response.status_code)

timeout

设置超时时间

1
r= requests.get('https://www.httpbin.org/get',timeout=1)

分别设置连接和读取时间

1
r= requests.get('https://www.httpbin.org/get',timeout=(5,30))

身份验证

1
2
3
4
5
6
7
8

from requests.auth import HTTPBasicAuth
r = requests.get('https://ssr3.scrape.center/',auth=HTTPBasicAuth('admin','admin'))
print(r.status_code)

# 简写
r = requests.get('https://ssr3.scrape.center/',auth=('admin','admin'))
print(r.status_code)
a-c-dream

a-c-dream

贵在坚持

109
13
0
CATEGORYS
TAGS