爬取多页多列数据代码

赵乾舟

1. import requests
   
2. from pyquery import PyQuery as pq
   
3. from openpyxl import Workbook
   
4. wb = Workbook()
   
5. sheet = wb.active
   
6. sheet['a1'] = '标题'
   
7. sheet['b1'] = '作者'
   
8. url = 'https://club.coovm.com/forum-53-{pn}.html'
   
9. headers = {
   
10.     "user-agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"
    
11.     }
    
12. for page in range(1, 50):
    
13.     lasturl = url.format(pn=page)
    
14.     #print(lasturl)
    
15.     r = requests.get(url=lasturl, headers=headers).text
    
16.     d = pq(r)
    
17.     for i in d('#threadlisttableid tbody'):
    
18.         sub_d = pq(i)
    
19.         print(sub_d('.xst').text(),end=' ')
    
20.         print(sub_d('.by').text().split(' ')[0])
    
21.         sheet.append([sub_d('.xst').text(),sub_d('.by a').text().split(' ')[0]])
    
22. wb.save('mine4.xlsx')
    

复制代码