本文将介绍如何使用Python爬取大众点评的店铺信息和评论数据,实现从店铺搜索到评论获取的完整流程。

功能概述

本爬虫程序主要实现以下功能:

  1. 店铺搜索:根据店铺名称搜索相关店铺
  2. 店铺选择:支持多店铺情况下的用户交互选择
  3. 信息提取:获取店铺的详细信息(评分、价格、地址等)
  4. 评论爬取:爬取店铺的所有评论数据
  5. 数据保存:将评论数据保存为CSV文件

技术栈

  • Python 3.x
  • requests:网络请求
  • lxml:HTML解析
  • re:正则表达式
  • csv:数据存储
  • random/time:请求控制和随机化

完整代码实现

1. 导入必要的库

import requests
import time
import string
import random
import csv
import os
from lxml import html
import re

2. 工具函数

def gen_query_id():
    """生成大众点评 mapi 需要的 queryid"""
    ts = str(int(time.time() * 1000))  # 毫秒时间戳
    rand = ''.join(random.choices(string.ascii_lowercase + string.digits, k=13))
    return f"{ts}_{rand}"

3. 请求头配置

# PC端请求头(用于店铺搜索和信息获取)
pc_headers = {
    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "accept-encoding": "gzip, deflate, br, zstd",
    "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6,zh-TW;q=0.5,ar;q=0.4",
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36 Edg/142.0.0.0",
    # ... 其他headers
}

# 移动端API请求头(用于评论获取)
m_headers = {
    'accept': 'application/json, text/plain, */*',
    'accept-encoding': 'gzip, deflate, br, zstd',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36 Edg/142.0.0.0',
    # ... 其他headers
}

4. 代理配置

tunnel_list = ["http://bievpxpq:doihzxnilk9c@31.59.33.35:6611"]
proxies = {
    "http": random.choice(tunnel_list),
    "https": random.choice(tunnel_list)
}

5. 店铺搜索功能

def get_shop_list(name):
    """获取店铺列表"""
    resp = requests.get(f'https://www.dianping.com/search/keyword/1/0_{name}', 
                       headers=pc_headers, timeout=10)
    
    if resp.status_code != 200:
        print("搜索失败,请检查网络或cookie设置")
        return []
    
    # 解析HTML
    tree = html.fromstring(resp.text)
    
    # 使用XPath提取信息
    shops = []
    li_list = tree.xpath('//div[@class="shop-list J_shop-list shop-all-list"]/ul/li')
    
    for li in li_list:
        try:
            shop_name = li.xpath('./div[2]/div/a/h4/text()')[0].strip()
            shop_url = li.xpath('./div[2]/div/a/@href')[0]
            # 从URL中提取shop_id
            shop_id = re.search(r'shop/([a-zA-Z0-9]+)', shop_url)
            shop_id = shop_id.group(1) if shop_id else None
            
            # 确保URL是完整的
            if not shop_url.startswith('http'):
                shop_url = 'https://www.dianping.com' + shop_url
            
            shops.append({
                'name': shop_name,
                'url': shop_url,
                'id': shop_id
            })
        except IndexError:
            continue
    
    return shops

6. 用户交互选择

def select_shop(shops):
    """让用户选择店铺"""
    if not shops:
        print("未找到相关店铺")
        return None
    
    print(f"\n找到 {len(shops)} 个相关店铺:")
    print("-" * 50)
    
    for i, shop in enumerate(shops, 1):
        print(f"{i}. {shop['name']}")
    
    print("-" * 50)
    
    if len(shops) == 1:
        selected_shop = shops[0]
        print(f"\n已选择唯一店铺: {selected_shop['name']}")
        return selected_shop
    else:
        while True:
            try:
                choice = input(f"\n请选择店铺 (1-{len(shops)}): ").strip()
                if not choice:
                    print("请输入选择编号")
                    continue
                
                choice_num = int(choice)
                if 1 <= choice_num <= len(shops):
                    selected_shop = shops[choice_num - 1]
                    print(f"\n已选择: {selected_shop['name']}")
                    return selected_shop
                else:
                    print(f"请输入 1-{len(shops)} 之间的数字")
            except ValueError:
                print("请输入有效的数字")

7. 店铺详细信息获取

def get_shop_details(url):
    """获取店铺详细信息"""
    try:
        resp = requests.get(url, headers=pc_headers, timeout=10)
        resp.raise_for_status()
        
        # 解析HTML
        tree = html.fromstring(resp.text)

        # 提取店铺名称
        shop_name = tree.xpath('//span[@class="shopName wx-text"]/text()')
        shop_name = shop_name[0].strip() if shop_name else "未找到"
        print(f"店铺名称: {shop_name}")

        # 提取评分
        rating_1 = tree.xpath('//div[@class="star-container star-45 wx-view"]/div[6]/text()')
        rating_2 = tree.xpath('//span[@class="scoreText wx-text"]/text()')
        rating_1 = rating_1[0].strip() if rating_1 else "未找到"
        rating_2 = rating_2[0].strip() if rating_2 else "未找到"
        print(f'评分: {rating_1} {rating_2}')

        # 提取评论数量
        review_count = tree.xpath('//span[@class="reviews wx-text"]/text()')
        review_count = review_count[0].strip() if review_count else "未找到"
        print(f'评论数量: {review_count}')

        # 提取人均价格
        avg_price = tree.xpath('//span[@class="price wx-text"]/text()')
        avg_price = avg_price[0].strip() if avg_price else "未找到"
        print(f'人均价格: {avg_price}')

        # 提取地址
        address = tree.xpath('//span[@class="addressText wx-text"]/text()')
        address = address[0].strip() if address else "未找到"
        address_2 = tree.xpath('//span[@class="desc-addr-txt wx-text"]/text()')
        address_2 = address_2[0].strip() if address_2 else "未找到"
        print(f'地址: {address} -- {address_2}')

        # 提取电话号码
        data = resp.text
        phone_matches = re.findall(r'"phoneNos":\s*\["?(\d+)"?\]', data)
        phone = phone_matches[0] if phone_matches else "未找到"
        print(f'电话号码: {phone}')

        # 返回提取的所有信息
        return {
            'shop_name': shop_name,
            'rating_1': rating_1,
            'rating_2': rating_2,
            'review_count': review_count,
            'avg_price': avg_price,
            'address': f"{address} -- {address_2}",
            'phone': phone
        }

    except Exception as e:
        print(f"解析错误: {e}")
        return None

8. 评论数据爬取

def fetch_comments(shop_id, start=0):
    """
    抓取一页评论
    :shop_id: 店铺ID
    :start: 分页偏移
    :return: (is_end, 当前页评论list[dict])
    """
    try:
        query_id = gen_query_id()
        url = (
            "https://mapi.dianping.com/mapi/review/outsidesiftedreviewlist.bin?"
            f"optimus_code=10&"
            f"optimus_partner=76&"
            f"optimus_risk_level=71&"
            f"reqsource=4&"
            f"filterid=800&"
            f"merge=1&"
            f"needfilter=1&"
            f"queryid={query_id}&"
            f"referid={shop_id}&"
            f"refertype=0&"
            f"start={start}&"
            f"multifilterids=%7B%22filterIds%22%3A%5B800%5D%7D&"
            f"yodaReady=h5&"
            f"csecplatform=4&"
            f"csecversion=4.1.1"
        )

        resp = requests.get(url, headers=m_headers, timeout=10)
        resp.raise_for_status()

        try:
            data = resp.json()
        except Exception:
            print("接口返回不是JSON格式")
            return True, []

        if not isinstance(data, dict):
            print("接口返回异常数据结构")
            return True, []

        review_list = data.get("list")
        if not isinstance(review_list, list):
            print("未获取到评论数据,可能被风控或无数据")
            return True, []

        is_end = data.get("isEnd", True)

        comments = []
        for item in review_list:
            if not isinstance(item, dict):
                continue

            feed_user = item.get("feedUser")
            if not isinstance(feed_user, dict):
                continue

            username = feed_user.get("userName")
            if not username or username == "商家回应":
                continue

            content = item.get("content", "")
            score_list = item.get("feedScoreList") or []

            score_text = ""
            if isinstance(score_list, list):
                score_text = " ".join(
                    s.get("text", "") for s in score_list if isinstance(s, dict)
                )

            comments.append({
                "username": username,
                "content": content.replace("\n", ""),
                "score_text": score_text,
                "shop_id": shop_id
            })

        return is_end, comments

    except requests.RequestException as e:
        print(f"请求异常: {e}")
        return True, []

9. 数据保存功能

def save_to_csv(comments, filename="dianping_comments.csv"):
    """将评论数据保存到CSV文件"""
    file_exists = os.path.isfile(filename)

    with open(filename, 'a', newline='', encoding='utf-8-sig') as csvfile:
        fieldnames = ['username', 'content', 'score_text', 'shop_id']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

        if not file_exists:
            writer.writeheader()

        for comment in comments:
            writer.writerow(comment)

    print(f"已保存 {len(comments)} 条评论到 {filename}")

10. 爬取指定店铺的所有评论

def crawl_all_comments(shop_id, shop_name):
    """
    爬取指定店铺的所有评论
    """
    start_index = 0
    all_comments = []
    page_count = 0

    print(f"\n开始抓取 '{shop_name}' 的评论...")

    while True:
        result = fetch_comments(shop_id, start=start_index)

        if not result:
            print("接口返回空结果,停止抓取")
            break

        is_end, comments = result

        if comments:
            all_comments.extend(comments)
            save_to_csv(comments)
            page_count += 1
            print(f"第 {page_count} 页抓取完成,共 {len(comments)} 条评论")

        if is_end:
            print(f"所有评论抓取完成!共 {page_count} 页,总计 {len(all_comments)} 条评论")
            break

        start_index += 14
        time.sleep(random.uniform(1, 2))

    return all_comments, page_count

10. 主程序流程

def main():
    # 第一步:搜索店铺
    name = input('请输入店铺名称: ').strip()
    if not name:
        print("店铺名称不能为空")
        return
    
    # 获取店铺列表
    shops = get_shop_list(name)
    
    # 让用户选择店铺
    selected_shop = select_shop(shops)
    
    if selected_shop:
        print(f"\n正在获取店铺详细信息...")
        print(f"店铺URL: {selected_shop['url']}")
        print(f"店铺ID: {selected_shop['id']}")
        
        # 第二步:获取选定店铺的详细信息
        shop_details = get_shop_details(selected_shop['url'])
        
        if shop_details:
            print("\n" + "="*50)
            print("店铺信息获取完成!")
            print("="*50)
            
            # 第三步:爬取所有评论
            print("\n开始爬取评论数据...")
            all_comments, page_count = crawl_all_comments(selected_shop['id'], selected_shop['name'])
            
            # 最终统计
            print(f"\n最终统计:")
            print(f"店铺名称: {selected_shop['name']}")
            print(f"总页数: {page_count}")
            print(f"总评论数: {len(all_comments)}")
            print(f"数据已保存到 dianping_comments.csv")
            
    else:
        print("未选择任何店铺")

if __name__ == "__main__":
    main()

11. 完整代码

import requests
import time
import string
import random
import csv
import os
from lxml import html
import re


# ----------------- 工具函数 -----------------
def gen_query_id():
    """生成大众点评 mapi 需要的 queryid"""
    ts = str(int(time.time() * 1000))  # 毫秒时间戳
    rand = ''.join(random.choices(string.ascii_lowercase + string.digits, k=13))
    return f"{ts}_{rand}"


# ----------------- 请求头 -----------------
m_headers = {
    'accept': 'application/json, text/plain, */*',
    'accept-encoding': 'gzip, deflate, br, zstd',
    'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6,zh-TW;q=0.5,ar;q=0.4',
    'connection': 'keep-alive',
    'cookie': '此处填写你自己的cookie',
    'dnt': '1',
    'host': 'mapi.dianping.com',
    'origin': 'https://m.dianping.com',
    'referer': 'https://m.dianping.com/',
    'sec-ch-ua': '"Chromium";v="142", "Microsoft Edge";v="142", "Not_A Brand";v="99"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Windows"',
    'sec-fetch-dest': 'empty',
    'sec-fetch-mode': 'cors',
    'sec-fetch-site': 'same-site',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36 Edg/142.0.0.0'
}

pc_headers = {
    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "accept-encoding": "gzip, deflate, br, zstd",
    "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6,zh-TW;q=0.5,ar;q=0.4",
    "cache-control": "max-age=0",
    "connection": "keep-alive",
    "cookie":"此处填写你自己的cookie",
    "dnt": "1",
    "host": "www.dianping.com",
    "referer": "https://www.dianping.com/search/keyword/3/0_%E5%9B%9B%E6%B5%B7",
    "sec-ch-ua": '"Chromium";v="142", "Microsoft Edge";v="142", "Not_A Brand";v="99"',
    "sec-ch-ua-mobile": "?0",
    "sec-ch-ua-platform": '"Windows"',
    "sec-fetch-dest": "document",
    "sec-fetch-mode": "navigate",
    "sec-fetch-site": "same-origin",
    "sec-fetch-user": "?1",
    "upgrade-insecure-requests": "1",
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36 Edg/142.0.0.0"
}

# ----------------- 代理 -----------------
tunnel_list = ["http://bievpxpq:doihzxnilk9c@31.59.33.35:6611"]
proxies = {
    "http": random.choice(tunnel_list),
    "https": random.choice(tunnel_list)
}


# ----------------- 店铺搜索和选择 -----------------
def get_shop_list(name):
    """获取店铺列表"""
    resp = requests.get(f'https://www.dianping.com/search/keyword/1/0_{name}', headers=pc_headers, timeout=10)
    print(f"搜索状态码: {resp.status_code}")

    if resp.status_code != 200:
        print("搜索失败,请检查网络或cookie设置")
        return []
    # 解析HTML
    tree = html.fromstring(resp.text)

    # 使用XPath提取信息
    shops = []
    li_list = tree.xpath('//div[@class="shop-list J_shop-list shop-all-list"]/ul/li')

    for li in li_list:
        try:
            shop_name = li.xpath('./div[2]/div/a/h4/text()')[0].strip()
            shop_url = li.xpath('./div[2]/div/a/@href')[0]
            # 从URL中提取shop_id
            shop_id = re.search(r'shop/([a-zA-Z0-9]+)', shop_url)
            shop_id = shop_id.group(1) if shop_id else None

            # 确保URL是完整的
            if not shop_url.startswith('http'):
                shop_url = 'https://www.dianping.com' + shop_url

            shops.append({
                'name': shop_name,
                'url': shop_url,
                'id': shop_id
            })
        except IndexError:
            continue

    return shops


def select_shop(shops):
    """让用户选择店铺"""
    if not shops:
        print("未找到相关店铺")
        return None

    print(f"\n找到 {len(shops)} 个相关店铺:")
    print("-" * 50)

    for i, shop in enumerate(shops, 1):
        print(f"{i}. {shop['name']}")

    print("-" * 50)

    if len(shops) == 1:
        # 如果只有一个店铺,直接返回
        selected_shop = shops[0]
        print(f"\n已选择唯一店铺: {selected_shop['name']}")
        return selected_shop
    else:
        # 多个店铺让用户选择
        while True:
            try:
                choice = input(f"\n请选择店铺 (1-{len(shops)}): ").strip()
                if not choice:
                    print("请输入选择编号")
                    continue

                choice_num = int(choice)
                if 1 <= choice_num <= len(shops):
                    selected_shop = shops[choice_num - 1]
                    print(f"\n已选择: {selected_shop['name']}")
                    return selected_shop
                else:
                    print(f"请输入 1-{len(shops)} 之间的数字")
            except ValueError:
                print("请输入有效的数字")
            except KeyboardInterrupt:
                print("\n用户取消选择")
                return None


# ----------------- 店铺信息获取 -----------------
def get_shop_details(url):
    """获取店铺详细信息"""
    try:
        # 发送请求
        resp = requests.get(url, headers=pc_headers, timeout=10)
        resp.raise_for_status()
        print(f"\n店铺页面状态码: {resp.status_code}")

        # 解析HTML
        tree = html.fromstring(resp.text)

        # 使用XPath提取信息
        shop_name = tree.xpath('//span[@class="shopName wx-text"]/text()')
        shop_name = shop_name[0].strip() if shop_name else "未找到"
        print(f"店铺名称: {shop_name}")

        # 提取评分
        rating_1 = tree.xpath('//div[@class="star-container star-45 wx-view"]/div[6]/text()')
        rating_2 = tree.xpath('//span[@class="scoreText wx-text"]/text()')
        rating_1 = rating_1[0].strip() if rating_1 else "未找到"
        rating_2 = rating_2[0].strip() if rating_2 else "未找到"
        print(f'评分: {rating_1} {rating_2}')

        # 提取评论数量
        review_count = tree.xpath('//span[@class="reviews wx-text"]/text()')
        review_count = review_count[0].strip() if review_count else "未找到"
        print(f'评论数量: {review_count}')

        # 提取人均价格
        avg_price = tree.xpath('//span[@class="price wx-text"]/text()')
        avg_price = avg_price[0].strip() if avg_price else "未找到"
        print(f'人均价格: {avg_price}')

        # 提取地址
        address = tree.xpath('//span[@class="addressText wx-text"]/text()')
        address = address[0].strip() if address else "未找到"
        address_2 = tree.xpath('//span[@class="desc-addr-txt wx-text"]/text()')
        address_2 = address_2[0].strip() if address_2 else "未找到"
        print(f'地址: {address} -- {address_2}')

        # 提取电话号码
        data = resp.text
        phone_matches = re.findall(r'"phoneNos":\s*\["?(\d+)"?\]', data)
        phone = phone_matches[0] if phone_matches else "未找到"
        print(f'电话号码: {phone}')

        # 提取榜单信息
        Ranking_list = tree.xpath('//div[@class="rank-text wx-view"]/text()')
        Ranking_list = Ranking_list[0].strip() if Ranking_list else "未找到"
        print(f'榜单: {Ranking_list}')

        # 提取营业时间
        raw = tree.xpath('//div[@class="left-service wx-view"]//text()')
        business_hours = ' '.join(t.strip() for t in raw if t.strip())
        print(f'营业状态与时间: {business_hours or "未找到"}')

        # 返回提取的所有信息
        return {
            'shop_name': shop_name,
            'rating_1': rating_1,
            'rating_2': rating_2,
            'review_count': review_count,
            'avg_price': avg_price,
            'address': f"{address} -- {address_2}",
            'phone': phone,
            'ranking': Ranking_list,
            'business_hours': business_hours or "未找到"
        }

    except requests.RequestException as e:
        print(f"请求错误: {e}")
        return None
    except Exception as e:
        print(f"解析错误: {e}")
        return None


# ----------------- 评论爬取 -----------------
def fetch_comments(shop_id, start=0):
    """
    抓取一页评论
    :shop_id: 店铺ID
    :start: 分页偏移
    :return: (is_end, 当前页评论list[dict])
    """
    try:
        query_id = gen_query_id()
        url = (
            "https://mapi.dianping.com/mapi/review/outsidesiftedreviewlist.bin?"
            f"optimus_code=10&"
            f"optimus_partner=76&"
            f"optimus_risk_level=71&"
            f"reqsource=4&"
            f"filterid=800&"
            f"merge=1&"
            f"needfilter=1&"
            f"queryid={query_id}&"
            f"referid={shop_id}&"
            f"refertype=0&"
            f"start={start}&"
            f"multifilterids=%7B%22filterIds%22%3A%5B800%5D%7D&"
            f"yodaReady=h5&"
            f"csecplatform=4&"
            f"csecversion=4.1.1"
        )

        resp = requests.get(url, headers=m_headers, timeout=10)
        resp.raise_for_status()

        try:
            data = resp.json()
        except Exception:
            print("接口返回不是JSON格式")
            return True, []

        if not isinstance(data, dict):
            print("接口返回异常数据结构")
            return True, []

        review_list = data.get("list")
        if not isinstance(review_list, list):
            print("未获取到评论数据,可能被风控或无数据")
            return True, []

        is_end = data.get("isEnd", True)

        comments = []
        for item in review_list:
            if not isinstance(item, dict):
                continue

            feed_user = item.get("feedUser")
            if not isinstance(feed_user, dict):
                continue

            username = feed_user.get("userName")
            if not username or username == "商家回应":
                continue

            content = item.get("content", "")
            score_list = item.get("feedScoreList") or []

            score_text = ""
            if isinstance(score_list, list):
                score_text = " ".join(
                    s.get("text", "") for s in score_list if isinstance(s, dict)
                )

            comments.append({
                "username": username,
                "content": content.replace("\n", ""),
                "score_text": score_text,
                "shop_id": shop_id
            })

        return is_end, comments

    except requests.RequestException as e:
        print(f"请求异常: {e}")
        return True, []



def save_to_csv(comments, filename="dianping_comments.csv"):
    """
    将评论数据保存到CSV文件
    """
    # 如果文件不存在,创建文件并写入表头
    file_exists = os.path.isfile(filename)

    with open(filename, 'a', newline='', encoding='utf-8-sig') as csvfile:
        fieldnames = ['username', 'content', 'score_text', 'shop_id']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

        if not file_exists:
            writer.writeheader()

        for comment in comments:
            writer.writerow(comment)

    print(f"已保存 {len(comments)} 条评论到 {filename}")


def crawl_all_comments(shop_id, shop_name):
    """
    爬取指定店铺的所有评论
    """
    start_index = 0
    all_comments = []
    page_count = 0

    print(f"\n开始抓取 '{shop_name}' 的评论...")

    while True:
        result = fetch_comments(shop_id, start=start_index)

        if not result:
            print("接口返回空结果,停止抓取")
            break

        is_end, comments = result

        if comments:
            all_comments.extend(comments)
            save_to_csv(comments)
            page_count += 1
            print(f"第 {page_count} 页抓取完成,共 {len(comments)} 条评论")

        if is_end:
            print(f"所有评论抓取完成!共 {page_count} 页,总计 {len(all_comments)} 条评论")
            break

        start_index += 14
        time.sleep(random.uniform(1, 2))

    return all_comments, page_count



# ----------------- 主程序 -----------------
def main():
    # 第一步:搜索店铺
    name = input('请输入店铺名称: ').strip()
    if not name:
        print("店铺名称不能为空")
        return

    # 获取店铺列表
    shops = get_shop_list(name)

    # 让用户选择店铺
    selected_shop = select_shop(shops)

    if selected_shop:
        print(f"\n正在获取店铺详细信息...")
        print(f"店铺URL: {selected_shop['url']}")
        print(f"店铺ID: {selected_shop['id']}")

        # 第二步:获取选定店铺的详细信息
        shop_details = get_shop_details(selected_shop['url'])

        if shop_details:
            print("\n" + "=" * 50)
            print("店铺信息获取完成!")
            print("=" * 50)

            # 第三步:爬取所有评论
            print("\n开始爬取评论数据...")
            all_comments, page_count = crawl_all_comments(selected_shop['id'], selected_shop['name'])

            # 最终统计
            print(f"\n最终统计:")
            print(f"店铺名称: {selected_shop['name']}")
            print(f"总页数: {page_count}")
            print(f"总评论数: {len(all_comments)}")
            print(f"数据已保存到 dianping_comments.csv")

    else:
        print("未选择任何店铺")


if __name__ == "__main__":
    main()

使用说明

  1. 环境准备

    • 安装Python 3.x
    • 安装所需库:pip install requests lxml
  2. 配置修改

    • 登录大众点评,更新cookie信息
    • 配置代理IP(如果需要)
    • 调整请求头中的User-Agent
  3. 运行程序

    • 执行主程序
    • 输入目标店铺名称
    • 从搜索结果中选择具体店铺
    • 程序自动获取店铺信息和评论数据

技术要点

  1. 反爬虫策略

    • 使用真实的请求头信息
    • 配置cookie模拟登录状态
    • 使用代理IP轮换
    • 添加随机延时
  2. 数据解析

    • 使用XPath定位HTML元素
    • 使用正则表达式提取特定数据
    • 处理JSON格式的API响应
  3. 错误处理

    • 网络请求异常处理
    • 数据解析异常处理
    • 文件操作异常处理

运行结果

在这里插入图片描述在这里插入图片描述

在这里插入图片描述

注意事项

  1. 遵守法律法规:爬取数据时请遵守网站的使用条款和相关法律法规
  2. 控制请求频率:避免对目标网站造成过大压力
  3. 数据使用:仅限学习和研究使用,不得用于商业用途

总结

本文提供了一个完整的大众点评数据爬取解决方案,涵盖了从店铺搜索到评论获取的全流程。通过模块化的设计和完善的错误处理,确保了程序的稳定性和可用性。读者可以根据自己的需求对代码进行修改和扩展。


声明:本文仅供技术学习交流,请遵守相关法律法规和网站的使用条款,合理使用爬虫技术。

Logo

DAMO开发者矩阵,由阿里巴巴达摩院和中国互联网协会联合发起,致力于探讨最前沿的技术趋势与应用成果,搭建高质量的交流与分享平台,推动技术创新与产业应用链接,围绕“人工智能与新型计算”构建开放共享的开发者生态。

更多推荐