济南建公司网站短视频搜索优化
以下是使用Python编写的针对淘宝、天猫、京东详情页的爬虫实例。请注意，这些实例仅供参考，实际使用时可能需要根据网站结构的变化进行调整，并且需要遵守各平台的爬虫协议和法律法规。
淘宝详情页爬虫实例
-
环境准备：
- Python 3.x
- Selenium库
- ChromeDriver（或对应浏览器的WebDriver）
-
代码实现：
from selenium import webdriver | |
from selenium.webdriver.common.by import By | |
from selenium.webdriver.common.keys import Keys | |
import time | |
import csv | |
# 初始化WebDriver | |
driver = webdriver.Chrome() | |
# 打开淘宝并搜索商品 | |
driver.get('https://www.taobao.com/') | |
driver.maximize_window() # 最大化浏览器窗口 | |
driver.implicitly_wait(10) # 设置隐式等待时间 | |
# 搜索商品（这里以“手机”为例） | |
search_keyword = '手机' | |
driver.find_element(By.XPATH, '//*[@id="q"]').send_keys(search_keyword) | |
driver.find_element(By.XPATH, '//*[@id="J_TSearchForm"]/div[1]/button').click() | |
# 等待搜索结果加载完成 | |
time.sleep(5) | |
# 解析搜索结果页面并提取商品详情页链接 | |
product_links = [] | |
for item in driver.find_elements(By.XPATH, '//div[@class="grid g-clearfix"]/div/div'): | |
detail_url = item.find_element(By.XPATH, './/div[@class="pic"]/a').get_attribute('href') | |
product_links.append(detail_url) | |
# 遍历商品详情页链接并提取所需信息 | |
with open('taobao_products.csv', 'w', newline='', encoding='utf-8') as csvfile: | |
fieldnames = ['title', 'price', 'seller', 'location', 'detail_url'] | |
writer = csv.DictWriter(csvfile, fieldnames=fieldnames) | |
writer.writeheader() | |
for link in product_links: | |
driver.get(link) | |
time.sleep(3) # 等待详情页加载完成 | |
title = driver.find_element(By.XPATH, '//*[@id="J_DetailHeader"]/div[1]/h1').text | |
price = driver.find_element(By.XPATH, '//*[@id="J_StrPrice"]/em').text | |
seller = driver.find_element(By.XPATH, '//*[@id="J_OtherOptions"]/div[1]/p[1]/a').text | |
location = driver.find_element(By.XPATH, '//*[@id="J_OtherOptions"]/div[1]/p[2]/span').text | |
writer.writerow({ | |
'title': title, | |
'price': price, | |
'seller': seller, | |
'location': location, | |
'detail_url': link | |
}) | |
# 关闭WebDriver | |
driver.quit() |
天猫详情页爬虫实例
- 环境准备：与淘宝相同。
- 代码实现（以搜索“羽毛球”为例）：
from selenium import webdriver | |
from selenium.webdriver.common.by import By | |
from selenium.webdriver.support.ui import WebDriverWait | |
from selenium.webdriver.support import expected_conditions as EC | |
import time | |
import csv | |
# 初始化WebDriver | |
driver = webdriver.Chrome() | |
# 打开天猫并搜索商品 | |
driver.get('https://list.tmall.com/') | |
driver.maximize_window() | |
driver.implicitly_wait(10) | |
# 搜索商品（这里以“羽毛球”为例） | |
search_keyword = '羽毛球' | |
driver.get(f'https://list.tmall.com/search_product.htm?q={search_keyword}') | |
# 等待搜索结果加载完成 | |
wait = WebDriverWait(driver, 10) | |
page_total_element = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.tm-pagination .ui-page-item.ui-page-item-last em'))) | |
page_total = page_total_element.text | |
# 解析搜索结果页面并提取商品信息 | |
product_info = [] | |
for page in range(1, int(page_total) + 1): | |
try: | |
# 如果是非第一页，则进行翻页操作 | |
if page > 1: | |
input_element = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.ui-page > div.ui-page-wrap > b.ui-page-skip > form > input.ui-page-skipTo'))) | |
input_element.clear() | |
input_element.send_keys(page) | |
submit_button = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.ui-page > div.ui-page-wrap > b.ui-page-skip > form > button.ui-btn-s'))) | |
submit_button.click() | |
time.sleep(2) # 等待页面加载 | |
# 提取当前页的商品信息 | |
goods = driver.find_elements(By.CSS_SELECTOR, '#J_ItemList .product') | |
for good in goods: | |
title = good.find_element(By.CSS_SELECTOR, '.productTitle').text | |
price = good.find_element(By.CSS_SELECTOR, '.productPrice').text.replace('¥', '') | |
detail_url = good.find_element(By.CSS_SELECTOR, '.productImg').get_attribute('href') | |
product_info.append({ | |
'title': title, | |
'price': price, | |
'detail_url': detail_url | |
}) | |
except Exception as e: | |
print(f"Error on page {page}: {e}") | |
# 将商品信息写入CSV文件 | |
with open('tmall_products.csv', 'w', newline='', encoding='utf-8') as csvfile: | |
fieldnames = ['title', 'price', 'detail_url'] | |
writer = csv.DictWriter(csvfile, fieldnames=fieldnames) | |
writer.writeheader() | |
for product in product_info: | |
writer.writerow(product) | |
# 关闭WebDriver | |
driver.quit() |
京东详情页爬虫实例
- 环境准备：与淘宝相同。
- 代码实现（以搜索“手机”为例，并提取详情页图片）：
from selenium import webdriver | |
from selenium.webdriver.common.by import By | |
from selenium.webdriver.support.ui import WebDriverWait | |
from selenium.webdriver.support import expected_conditions as EC | |
import time | |
import os | |
import requests | |
# 初始化WebDriver | |
driver = webdriver.Chrome() | |
# 打开京东并搜索商品 | |
driver.get('https://search.jd.com/') | |
driver.maximize_window() | |
driver.implicitly_wait(10) | |
# 搜索商品（这里以“手机”为例） | |
search_keyword = '手机' | |
driver.find_element(By.XPATH, '//*[@id="key"]').send_keys(search_keyword) | |
driver.find_element(By.XPATH, '//*[@id="search"]/div/button').click() | |
# 等待搜索结果加载完成 | |
wait = WebDriverWait(driver, 10) | |
# 提取商品详情页链接并进入详情页提取图片 | |
product_links = [] | |
for item in driver.find_elements(By.CSS_SELECTOR, '.gl-item'): | |
detail_url = item.find_element(By.CSS_SELECTOR, '.p-name em a').get_attribute('href') | |
product_links.append(detail_url) | |
headers = { | |
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' | |
} | |
for link in product_links: | |
driver.get(link) | |
time.sleep(3) # 等待详情页加载完成 | |
# 提取图片链接并下载 | |
image_urls = [] | |
try: | |
images = driver.find_elements(By.CSS_SELECTOR, '.sku-gallery img') | |
for img in images: | |
image_urls.append(img.get_attribute('src')) | |
except Exception as e: | |
print(f"Error extracting images from {link}: {e}") | |
continue | |
image_dir = f'./jd_images/{link.split("/")[-1]}' | |
if not os.path.exists(image_dir): | |
os.makedirs(image_dir) |