项目四 基于框架实现动态网页数据采集与存储

# 语法格式一：使用 scrapy.Field()
# 导入scrapy模块
import scrapy
# 定义包含scrapy.Item参数的类
class ScrapynameItem(scrapy.Item):
    # define the fields for your item here like:
    # 自定义字段
    name = scrapy.Field()
    # 通过提示
    pass

# 语法格式二：引入 Item 和 Field
# 导入scrapy的Item参数和Field方法
from scrapy import Item, Field
# 定义包含scrapy.Item参数的类
class ScrapynameItem(Item):
    # 自定义字段
    name = Field()
    # 通过提示
    pass

# MySpider.py 中使用字段
import scrapy
# 导入 ScrapynameItem 类
from ScrapyName.items import ScrapynameItem

class MyspiderSpider(scrapy.Spider):
    def parse(self, response):
        item = ScrapynameItem()
        item["name"] = "值"

# xpath() 方法应用
import scrapy
# 从 scrapy.selector 中导入 Selector
from scrapy.selector import Selector

class MyspiderSpider(scrapy.Spider):
    name = 'MySpider'
    allowed_domains = ['域名列表']
    start_urls = ['网址列表']

    def parse(self, response):
        # 构造 Selector 实例
        sel = Selector(response)
        # 解析 HTML
        xpath = sel.xpath('表达式')
        pass

# css() 方法应用
import scrapy
# 从 scrapy.selector 中导入 Selector
from scrapy.selector import Selector

class MyspiderSpider(scrapy.Spider):
    name = 'MySpider'
    allowed_domains = ['域名列表']
    start_urls = ['网址列表']

    def parse(self, response):
        # 构造 Selector 实例
        sel = Selector(response)
        # 解析 HTML
        css = sel.css('表达式')
        pass

scrapy crawl 爬虫文件名称 path

ITEM_PIPELINES = {
    'ScrapyProject.pipelines.ScrapyprojectPipeline': 300,
}

# items.py —— 定义要采集的字段
import scrapy

class NewsItem(scrapy.Item):
    title = scrapy.Field()      # 标题
    link  = scrapy.Field()      # 链接
    time  = scrapy.Field()      # 发布时间

# spiders/news.py —— 解析规则
import scrapy
from demo.items import NewsItem

class NewsSpider(scrapy.Spider):
    name = "news"
    start_urls = ["https://www.example.com/news/index.html"]

    def parse(self, response):
        # response.xpath() 直接使用，无需额外导入
        for li in response.xpath('//ul[@class="news"]/li'):
            item = NewsItem()
            item["title"] = li.xpath("a/text()").get()
            item["link"]  = li.xpath("a/@href").get()
            item["time"]  = li.xpath("span/text()").get()
            yield item

# 运行爬虫并输出 JSON 文件
[root@master demo]# scrapy crawl news -o news.json
[root@master demo]# cat news.json
[
  {"title": "学校召开教学改革会议", "link": "/news/1.html", "time": "2023-02-10"},
  {"title": "运动会圆满落幕", "link": "/news/2.html", "time": "2023-02-11"},
  {"title": "新图书馆正式开放", "link": "/news/3.html", "time": "2023-02-12"}
]

# 也可以输出 CSV / JSON Lines 格式
[root@master demo]# scrapy crawl news -o news.csv
[root@master demo]# scrapy crawl news -o news.jl

# pipelines.py
import sqlite3

class NewsPipeline:
    def open_spider(self, spider):
        # 爬虫启动时建立数据库连接
        self.conn = sqlite3.connect("news.db")
        self.conn.execute("CREATE TABLE IF NOT EXISTS news("
                          "title TEXT, link TEXT PRIMARY KEY, time TEXT)")

    def process_item(self, item, spider):
        # 1. 去重：link 为主键，重复时数据库会拒绝写入
        # 2. 清洗：空标题替换为"未知"
        if not item.get("title"):
            item["title"] = "未知"
        self.conn.execute(
            "INSERT OR IGNORE INTO news(title, link, time) VALUES(?,?,?)",
            (item["title"], item["link"], item["time"]))
        self.conn.commit()
        return item

    def close_spider(self, spider):
        self.conn.close()

# settings.py —— 开启管道
ITEM_PIPELINES = {
    "demo.pipelines.NewsPipeline": 300,
}