数据采集实验

1.1 题目

熟练掌握 Selenium 查找HTML元素、爬取Ajax网页数据、等待HTML元素等内容。
使用Selenium框架爬取京东商城某类商品信息及图片。
候选网站：http://www.jd.com

1.2 思路

1.2.1 发送请求

• 引入驱动配置

   chrome_options = Options()
   chrome_options.add_argument('--headless')
   chrome_options.add_argument('--disable-gpu')
   self.driver = webdriver.Chrome(chrome_options=chrome_options)

• 启动驱动发起访问

url = "http://www.jd.com"
spider = MySpider()
while True:
    print("1.爬取")
    print("2.显示")
    print("3.退出")
    s = input("请选择(1,2,3):")
    if s == "1":
        spider.executeSpider(url, "手机")
        continue
    elif s == "2":
        spider.showDB()
        continue
    elif s == "3":
        break

1.2.2 获取页面信息

• 普通数据抓取

lis =self.driver.find_elements_by_xpath("//div[@id='J_goodsList']//li[@class='gl-item']")
for li in lis:
    try:
        src1 = li.find_element_by_xpath(".//div[@class='p-img']//a//img").get_attribute("src")
    except:
        src1 = ""
    try:
        src2 = li.find_element_by_xpath(".//div[@class='p-img']//a//img").get_attribute("data-lazy-img")
    except:
        src2 = ""
    try:
        price = li.find_element_by_xpath(".//div[@class='p-price']//i").text
    except:
        price = "0"
    try:
        note = li.find_element_by_xpath(".//div[@class='p-name p-name-type-2']//em").text
        mark = note.split(" ")[0]
        mark = mark.replace("爱心东东\n", "")
        mark = mark.replace(",", "")
        note = note.replace("爱心东东\n", "")
        note = note.replace(",", "")
    except:
        note = ""
        mark = ""

• 图片链接获取并保存

if src1:
    src1 = urllib.request.urljoin(self.driver.current_url, src1)
    p = src1.rfind(".")
    mFile = no + src1[p:]
elif src2:
    src2 = urllib.request.urljoin(self.driver.current_url, src2)
    p = src2.rfind(".")
    mFile = no + src2[p:]
if src1 or src2:
    T = threading.Thread(target=self.download, args=(src1, src2, mFile))
    T.setDaemon(False)
    T.start()
    self.threads.append(T)
else:
    mFile = ""

1.2.3 保存数据

• 保存在数据库中

1. 建表

        try:
            self.conn = sqlite3.connect("phones.db")
            self.cursor = self.conn.cursor()
            try:
                # 如果有表就删除
                self.cursor.execute("drop table phones")
            except Exception as err:
                print(err)
            try:
                #  建立新的表
                sql = "create  table  phones  (mNo  varchar(32) primary key, mMark varchar(256),mPrice varchar(32),mNote varchar(1024),mFile varchar(256))"
                self.cursor.execute(sql)
            except:
                pass
        except Exception as err:
            print(err)

2. 插入数据

    def insertDB(self, mNo, mMark, mPrice, mNote, mFile):
        try:
            sql = "insert into phones (mNo,mMark,mPrice,mNote,mFile) values (?,?,?,?,?)"
            self.cursor.execute(sql, (mNo, mMark, mPrice, mNote, mFile))
        except Exception as err:
            print(err)

• 保存图片

1. 创建文件夹

        try:
            if not os.path.exists(MySpider.imagePath):
                os.mkdir(MySpider.imagePath)
            images = os.listdir(MySpider.imagePath)
            for img in images:
                s = os.path.join(MySpider.imagePath, img)
                os.remove(s)
        except Exception as err:
            print(err)

2. 保存