author: 专注Python实战分享爬虫与数据分析干货title: Python爬虫实战⑭Scrapy实战——新闻网站全站爬取update: 2026-04-26tags: Python,Scrapy,爬虫实战,新闻网站,全站爬取,分布式作者专注Python实战分享爬虫与数据分析干货更新时间2026年4月适合人群已掌握Scrapy、想完成完整项目的开发者前言做一个能上线的爬虫项目前面学了Scrapy的各个组件但都是零散知识。今天我们把这些知识串起来做一个完整项目爬取新闻网站的全站数据。项目目标爬取多个新闻网站的标题、正文、时间、来源自动发现新文章并跟进数据清洗后存入MongoDB完整的日志和异常处理分布式部署准备一、项目规划1.1 选择目标网站我们以一个模拟的新闻网站为例子。实际项目中选择robots.txt允许爬取的新闻站点。1.2 数据模型设计# items.pyimportscrapyclassNewsItem(scrapy.Item):新闻数据模型titlescrapy.Field()# 标题contentscrapy.Field()# 正文内容publish_timescrapy.Field()# 发布时间authorscrapy.Field()# 作者/来源categoryscrapy.Field()# 分类tagsscrapy.Field()# 标签urlscrapy.Field()# 原文链接sourcescrapy.Field()# 来源网站crawl_timescrapy.Field()# 爬取时间imagesscrapy.Field()# 图片列表classArticleItem(scrapy.Item):文章摘要数据模型article_idscrapy.Field()# 文章ID从URL提取titlescrapy.Field()summaryscrapy.Field()# 摘要publish_timescrapy.Field()urlscrapy.Field()thumbnailscrapy.Field()# 缩略图categoryscrapy.Field()二、编写爬虫2.1 列表页Spider# spiders/news_spider.pyimportscrapyfromscrapy.linkextractorsimportLinkExtractorfromscrapy.spidersimportCrawlSpider,Rulefromnews_project.itemsimportNewsItem,ArticleItemfromdatetimeimportdatetimeimportreclassNewsListSpider(CrawlSpider):新闻列表爬虫自动发现文章链接namenews_listallowed_domains[news.example.com]start_urls[https://news.example.com/,https://news.example.com/tech/,https://news.example.com/business/,https://news.example.com/culture/,]rules(# 规则1列表页翻页Rule(LinkExtractor(allowr/page/\d,restrict_css.pagination, .page-nav),followTrue,),# 规则2文章详情页Rule(LinkExtractor(allowr/article/\d\.html,restrict_css.article-list, .news-list, main),callbackparse_article,followFalse,),)custom_settings{DOWNLOAD_DELAY:2,CONCURRENT_REQUESTS_PER_DOMAIN:2,}defparse_article(self,response):解析文章详情页itemArticleItem()# 文章IDmatchre.search(r/article/(\d)\.html,response.url)item[article_id]match.group(1)ifmatchelse# 标题titleresponse.css(h1.article-title::text).get()ifnottitle:titleresponse.css(h1::text).get()item[title]title.strip()iftitleelse# 摘要summaryresponse.css(meta[namedescription]::attr(content)).get()ifnotsummary:summaryresponse.css(.article-summary::text).get()item[summary]summary.strip()ifsummaryelse# 发布时间time_strresponse.css(time.publish-time::attr(datetime)).get()ifnottime_str:time_strresponse.css(span.time::text).get()item[publish_time]self._parse_time(time_str)# 分类categoryresponse.css(.article-category a::text).get()ifnotcategory:categoryresponse.css(nav.breadcrumb span::text).get()item[category]category.strip()ifcategoryelse# 缩略图thumbnailresponse.css(meta[propertyog:image]::attr(content)).get()ifnotthumbnail:thumbnailresponse.css(.article-cover img::attr(src)).get()item[thumbnail]response.urljoin(thumbnail)ifthumbnailelse# 标签tagsresponse.css(.article-tags a::text).getall()item[tags][t.strip()fortintagsift.strip()]# 来源网站item[source]示例新闻网站# 原始URLitem[url]response.urlyielditem# 同时请求完整内容如果需要的话yieldscrapy.Request(response.url,callbackself.parse_full_content,meta{item:item},dont_filterTrue,)defparse_full_content(self,response):解析文章完整内容正文itemresponse.meta[item]# 提取正文段落paragraphsresponse.css(.article-content p::text).getall()ifnotparagraphs:paragraphsresponse.css(article p::text).getall()ifnotparagraphs:paragraphsresponse.css(.content p::text).getall()item[content]\n.join(p.strip()forpinparagraphsifp.strip())item[crawl_time]datetime.now().isoformat()yielditemdef_parse_time(self,time_str):解析时间字符串ifnottime_str:return# 常见格式patterns[(r\d{4}-\d{2}-\d{2},%Y-%m-%d),(r\d{4}/\d{2}/\d{2},%Y/%m/%d),(r\d{4}年\d{1,2}月\d{1,2}日,%Y年%m月%d日),]forpattern,fmtinpatterns:matchre.search(pattern,time_str)ifmatch:try:returndatetime.strptime(match.group(),fmt).isoformat()exceptValueError:continuereturntime_str2.2 详情页Spider# spiders/news_detail_spider.pyimportscrapyfromnews_project.itemsimportNewsItemclassNewsDetailSpider(scrapy.Spider):新闻详情爬虫根据URL列表抓取详情namenews_detaildef__init__(self,url_fileNone,*args,**kwargs):super().__init__(*args,**kwargs)self.url_fileurl_filedefstart_requests(self):从文件读取URL列表ifself.url_file:withopen(self.url_file,r,encodingutf-8)asf:urls[line.strip()forlineinfifline.strip()]forurlinurls:yieldscrapy.Request(url,callbackself.parse)else:self.logger.warning(未提供URL文件跳过)defparse(self,response):解析新闻详情itemNewsItem()item[url]response.url item[title]response.css(h1::text).get()oritem[content] .join(p.strip()forpinresponse.css(.article p::text).getall())item[publish_time]response.css(time::attr(datetime)).get()oritem[source]response.css(.source::text).get()oryielditem三、Pipeline编写# pipelines.pyimportpymongofromdatetimeimportdatetimefromitemadapterimportItemAdapterimporthashlibclassNewsValidationPipeline:数据验证defprocess_item(self,item,spider):adapterItemAdapter(item)# 必填字段检查ifnotadapter.get(title):spider.logger.warning(f缺少标题跳过:{adapter.get(url)})returnNoneifnotadapter.get(url):returnNone# 生成文章IDifnotadapter.get(article_id):urladapter.get(url,)adapter[article_id]hashlib.md5(url.encode()).hexdigest()[:16]# 清洗内容contentadapter.get(content,)ifcontent:adapter[content]content.strip()# 限制内容长度iflen(adapter[content])50000:adapter[content]adapter[content][:50000]returnitemclassMongoDBStoragePipeline:MongoDB存储def__init__(self,mongo_uri,mongo_db):self.mongo_urimongo_uri self.mongo_dbmongo_db self.clientNoneself.dbNoneclassmethoddeffrom_crawler(cls,crawler):returncls(mongo_uricrawler.settings.get(MONGO_URI,mongodb://localhost:27017),mongo_dbcrawler.settings.get(MONGO_DATABASE,news_crawler),)defopen_spider(self,spider):self.clientpymongo.MongoClient(self.mongo_uri)self.dbself.client[self.mongo_db]spider.logger.info(f连接MongoDB:{self.mongo_db})defprocess_item(self,item,spider):adapterItemAdapter(item)docdict(adapter)# 按Spider名称分集合collection_namef{spider.name}_items# 去重插入try:self.db[collection_name].update_one({article_id:doc.get(article_id,)},{$set:doc},upsertTrue,)spider.logger.debug(f存储:{doc.get(title,)[:30]})exceptExceptionase:spider.logger.error(f存储失败:{e})returnitemdefclose_spider(self,spider):self.client.close()spider.logger.info(MongoDB连接关闭)classDuplicatesFilterPipeline:URL去重Pipelinedef__init__(self):self.seen_urlsset()defopen_spider(self,spider):self.seen_urls.clear()defprocess_item(self,item,spider):adapterItemAdapter(item)urladapter.get(url,)ifurlinself.seen_urls:spider.logger.debug(f重复URL跳过:{url})returnNoneself.seen_urls.add(url)returnitem四、Settings配置# settings.pyBOT_NAMEnews_projectSPIDER_MODULES[news_project.spiders]NEWSPIDER_MODULEnews_project.spidersROBOTSTXT_OBEYTrueDEFAULT_REQUEST_HEADERS{User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36,Accept:text/html,application/xhtmlxml,application/xml;q0.9,*/*;q0.8,Accept-Language:zh-CN,zh;q0.9,en;q0.8,}DOWNLOAD_DELAY2CONCURRENT_REQUESTS_PER_DOMAIN2# 自动限速AUTOTHROTTLE_ENABLEDTrueAUTOTHROTTLE_START_DELAY1AUTOTHROTTLE_MAX_DELAY10# MongoDB配置MONGO_URImongodb://localhost:27017MONGO_DATABASEnews_crawler# Pipeline配置ITEM_PIPELINES{news_project.pipelines.DuplicatesFilterPipeline:100,news_project.pipelines.NewsValidationPipeline:200,news_project.pipelines.MongoDBStoragePipeline:300,}# 日志LOG_FILEnews_crawler.logLOG_LEVELINFO五、运行与部署# 本地运行cdnews_project scrapy crawl news_list-sJOBDIRcrawls/news_list# -s JOBDIR保存爬虫状态中断后可继续# scrapy crawl news_list -s JOBDIRcrawls/news_list# 导出数据scrapy crawl news_list-onews.json-sJOBDIRcrawls/news_list# Linux后台运行nohupnohupscrapy crawl news_list-sJOBDIRcrawls/news_listcrawler.log21# 查看日志tail-fcrawler.log# 查看爬虫状态scrapy list六、监控与维护# monitor.py — 爬虫健康监控importpymongofromdatetimeimportdatetime,timedeltadefcheck_crawler_health():检查爬虫运行状态clientpymongo.MongoClient(mongodb://localhost:27017)dbclient[news_crawler]collections[news_list_items]forcol_nameincollections:coldb[col_name]nowdatetime.now()one_hour_agonow-timedelta(hours1)one_day_agonow-timedelta(days1)hour_countcol.count_documents({crawl_time:{$gte:one_hour_ago.isoformat()}})day_countcol.count_documents({crawl_time:{$gte:one_day_ago.isoformat()}})print(f\n爬虫:{col_name})print(f 最近1小时:{hour_count}条)print(f 最近24小时:{day_count}条)ifhour_count0:print( ⚠️ 警告: 过去1小时无新数据可能爬虫已停止)client.close()if__name____main__:check_crawler_health()七、知识卡配置说明CrawlSpider Rules自动发现链接无需手动翻页LinkExtractor restrict_css在指定区域内提取链接followTrue跟进链接继续爬取followFalse不跟进只解析当前页JOBDIR保存爬虫状态中断可续ROBOTSTXT_OBEY遵守robots.txtMONGO_URIMongoDB连接地址upsertTrue不存在则插入存在则更新八、课后作业必做题用Scrapy爬取任意新闻网站的文章列表实现MongoDB存储Pipeline配置自动限速和请求延迟选做题实现爬虫健康监控脚本部署到Linux服务器并设置定时任务有问题欢迎评论区留言大家一起讨论标签Python | Scrapy | 爬虫实战 | 新闻网站 | 全站爬取 | MongoDB