python编写爬虫时，某位同学所遇到的问题，以及排查办法

大家好，欢迎来到IT知识分享网。

某位同学在让我帮他调试一段代码

同学自己写的代码如下

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2020/3/5 20:20
# @Author : 小和尚不吃素
# @Site : 
# @File : zhihu.py
# @Software: PyCharm

import requests
from urllib.parse import urlencode
import json
from requests.exceptions import RequestException

def get_one_page():
    str_url ='https://www.zhihu.com/api/v3/feed/topstory/recommend?'
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36',
               'Cookie':'Cookie值'
               }
    data = {
        'session_token': 'bb725fb534057438d33b97f8be29f6d4',
        'desktop': 'true',
        'page_number':1,
        'limit': '6',
        'action': 'down',
        # 'after_id': 17,
        'ad_interval': '-1'
    }
    url = str_url+urlencode(data)
    print(url)
    try:
        response = requests.get(url, headers=headers)
        print(response.status_code)
        if response.status_code == 200:
            return response.json()
        return None
    except RequestException:
        print('请求索引页出错')
        return None


def parse_page_index(html):
    # print(html)
    data = html['data']
    # print(data)
    for i in data:
        str_1 = i['target']
        print(str_1)
        # str_2 = str_1['id']
        # print(str_2)
        # if
        str_3 = str_1['author']
        print(str_3)
        # str_1['']
        # str_4 = str_3['id']
        str_5 = str_3['question']
        print(str_5)
        # for z in i:
        #     print(z)
        print('\n')
    # data = json.loads(html)
    # print(data)
    # if data and 'data' in data:
    #     for item in data.get('data'):

            # print(item)
            # yield item['id']

if __name__ == '__main__':
    html = get_one_page()
    parse_page_index(html)

运行完成之后，报错如下：

看到这里，我们都知道报错的原因是第55行：str_5 = str_3[‘question’]出现错误我们下来就开始分析str_3,发现里面没有question这个key，那刀哥就分析data这个list的数据，发现在第46行代码：str_1 = i[‘target’]print(str_1)找到了question。那么我就把第55行代码：str_5 = str_3[‘question’]修改为：str_5 = str_1[‘question’]再次运行，发现循环到莫一行数据还是会提示这个错误，后来查找，有的字典中有question这个key有的没有那么刀哥就加了一个判断过滤了没有question这个key的情况，加的代码如下：if(‘question’ in str_1):str_5 = str_1[‘question’]print(str_5)

再次运行，发现程序再也不报错了。

这次调试给我们有这几个方面的思考：

1，写代码的时候一定要细心；

2，要随时输出数据，关注数据的变化；

3，在使用无论是python还是java（map类型），在需要通过key值获取value的时候，最好先进行判断key值是否存在；

最终修改之后，能够正常运行的代码如下：

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2020/3/5 20:20
# @Author : 小和尚不吃素
# @Site :
# @File : zhihu.py
# @Software: PyCharm

import requests
from urllib.parse import urlencode
import json
from requests.exceptions import RequestException

def get_one_page():
    str_url ='https://www.zhihu.com/api/v3/feed/topstory/recommend?'
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36',
               'Cookie':'Cookie值'
               }
    data = {
        'session_token': 'bb725fb534057438d33b97f8be29f6d4',
        'desktop': 'true',
        'page_number':1,
        'limit': '6',
        'action': 'down',
        # 'after_id': 17,
        'ad_interval': '-1'
    }
    url = str_url+urlencode(data)
    print(url)
    try:
        response = requests.get(url, headers=headers)
        print(response.status_code)
        if response.status_code == 200:
            return response.json()
        return None
    except RequestException:
        print('请求索引页出错')
        return None


def parse_page_index(html):
    # print(html)
    data = html['data']
    # print(data)
    for i in data:
        str_1 = i['target']
        print(str_1)
        # str_2 = str_1['id']
        # print(str_2)
        # if
        str_3 = str_1['author']
        print(str_3)
        # str_1['']
        # str_4 = str_3['id']
        if ('question' in str_1):
            str_5 = str_1['question']
            print(str_5)
        # for z in i:
        #     print(z)
        print('\n')
    # data = json.loads(html)
    # print(data)
    # if data and 'data' in data:
    #     for item in data.get('data'):

            # print(item)
            # yield item['id']

if __name__ == '__main__':
    html = get_one_page()
    parse_page_index(html)