# coding=utf-8
__author__ = 'scaleworld'
import requests
import re
import sysreload(sys)
sys.setdefaultencoding("utf-8")class Spider:def __init__(self):print '开始爬取极客学院课程信息。。。'# 获取源代码def getSource(self, url):html = requests.get(url)return html.text# 获取每个课程块信息def getLessons(self, source):lessons = re.findall('deg="0" >(.*?)', source, re.S)return lessons# 获取课程信息，如课程名称、课程介绍、课程时间、课程等级、学习人数def getLessonInfo(self, lesson):info = {}info['title'] = re.search('
(.*?)', lesson, re.S).group(2).strip()info['desc'] = re.search('
(.*?)', lesson, re.S).group(1).strip()timeandlevel = re.findall('(.*?)', lesson, re.S)info['time'] = timeandlevel[0].strip().replace("\n", "").replace("    ", "")info['level'] = timeandlevel[1].strip()info['learnNumber'] = re.search('"learn-number">(.*?)', lesson, re.S).group(1).strip()return info# 保存课程信息到文件LessionInfos.txtdef saveLessionInfos(self, lessonInfos):# 'w'：只写，会覆盖之前写入的内容# 也可以用'a'：追加到文件末尾# 如果文件不存在，则自动创建文件f = open('LessionInfos.txt', 'w')i = 0for each in lessonInfos:i += 1f.writelines('第' + str(i) + '个课程：\n')f.writelines('title:' + each['title'] + '\n')f.writelines('desc:' + each['desc'] + '\n')f.writelines('time:' + each['time'] + '\n')f.writelines('level:' + each['level'] + '\n')f.writelines('learnNumber:' + each['learnNumber'] + '\n\n')f.close()if __name__ == '__main__':# 定义课程信息数组lessonInfos = []# 课程信息页面urlurl = 'http://www.jikexueyuan.com/course/'# 实例化爬虫spider = Spider()# 取[1,21)及1到20页的课程信息for i in range(1, 21):# 构建分页URLpageUrl = url + '?pageNum=' + str(i)print '正在处理页面：' + pageUrlsource = spider.getSource(pageUrl)lessons = spider.getLessons(source)for lesson in lessons:lessonInfo = spider.getLessonInfo(lesson)lessonInfos.append(lessonInfo)# print 'title:'+lessonInfo.get('title')# print 'desc:'+lessonInfo.get('desc')# print 'time:'+lessonInfo.get('time')# print 'level:'+lessonInfo.get('level')# print 'learnNumber:'+lessonInfo.get('learnNumber')print '已处理' + str(lessons.__len__()) + '个课程信息。'print '极客学院课程信息爬取完毕，正在保存课程信息。。。'spider.saveLessionInfos(lessonInfos)print '极客学院课程信息保存完毕。'