python自动化提取网页数据（Google浏览器版）
·
利用selenium库配合谷歌驱动实现自动化，pandas库保存网页数据，实现方式如下：
from bs4 import BeautifulSoup
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import re
import pandas as pd
from openpyxl.styles import Border,Side,Font,Alignment,PatternFill
import os
def write_file(v_df,out_put_flie,md,sheent_name):
with pd.ExcelWriter(out_put_flie, engine='openpyxl', mode=md) as writer:
v_df.to_excel(writer, sheet_name=sheent_name, index=False) # 输出总体文件
ws = writer.sheets[sheent_name] #
# 加边框
thin = Side(style='thin', color='000000')
border = Border(top=thin, right=thin, bottom=thin, left=thin)
# 给所有表格加边框
for row in ws.iter_rows(min_row=1, max_row=ws.max_row, min_col=1, max_col=ws.max_column):
for cell in row:
cell.border = border
# 自适应列宽处理
for col in ws.columns:
col_center = col[0].column_letter
max_len = max(len(str(cell.value)) for cell in col)
ws.column_dimensions[col_center].width = max_len + 6
def acess_new_website(out_put_flie,yp_bh):
if os.path.exists(out_put_flie):
md="a"
#os.remove(out_put_flie)
else:
md="w"
login_url = "目标登录页面"
USERNAME = "USERNAME"
PASSWORD = "PASSWORD"
chrome_options=Options()
chrome_options.binary_location=r"C:\chrome.exe" #指定谷歌浏览器程序路径
chrome_options.add_argument('--enable-javascript')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('disable-dev-shm-usage')
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option('useAutomationExtension', False)
#关闭密码保存弹窗
chrome_options.add_experimental_option("prefs",{
"credentials_enable_service":False,
"profile.password_manager_enabled":False
})
driver =webdriver.Chrome(executable_path=r"E:\chromedriver.exe",options=chrome_options) #executable_path 指定谷歌浏览器驱动程序
driver.maximize_window()
wait = WebDriverWait(driver, 15)
try:
driver.get(login_url)
time.sleep(2)
driver.find_element(By.NAME, "username").send_keys(USERNAME)
driver.find_element(By.NAME, "password").send_keys(PASSWORD)
time.sleep(3)
driver.find_element(By.NAME, "password").submit()
time.sleep(4)
menu1 = wait.until(EC.element_to_be_clickable((By.XPATH, "//*[text()='一级菜单']"))) #一级菜单按钮
menu1.click()
time.sleep(5)
menu2 = wait.until(EC.element_to_be_clickable((By.XPATH, "//*[text()='二级菜单']"))) #二级菜单按钮
menu2.click()
time.sleep(5)
wait.until(EC.presence_of_element_located((By.XPATH, '//div[.//label[contains(text(),"筛选字段")]]'))) #筛选字段
ypbh_str = "//div[@label='筛选字段']//input[@placeholder='请输入']"
input_ypbh = wait.until(EC.element_to_be_clickable((By.XPATH, ypbh_str)))
input_ypbh.clear()
input_ypbh.send_keys(yp_bh)
time.sleep(2)
cx_str = '//label[.//button/span[normalize-space()="查询"]]/button' #查询按钮
search_btn = wait.until(EC.element_to_be_clickable((By.XPATH, cx_str)))
search_btn.click()
time.sleep(6)
# 点击详情专用
# 先定位并勾选单行单选框
click_radio_js = '''
var r = document.querySelector(".el-table__body-wrapper tbody tr:first-child .el-radio input");
if(r){
r.focus();
r.dispatchEvent(new MouseEvent('click',{bubbles: true}));
console.log("单选框点击成功");
} else {
console.log("没找到单选框，需要手动点");
}
'''
driver.execute_script(click_radio_js)
time.sleep(3)
# 点击查看按钮
ck_str = '//label[.//button/span[normalize-space()="查看"]]/button'
search_btn = wait.until(EC.element_to_be_clickable((By.XPATH, ck_str)))
search_btn.click()
time.sleep(4)
menu2 = wait.until(EC.element_to_be_clickable((By.XPATH, "//*[text()='一级菜单']"))) #按钮链接一级菜单
menu2.click()
time.sleep(8)
# 打印JS值，显示字段值
driver.execute_script('''
var arr = document.querySelectorAll(".el-input__inner,.el-textarea__inner");
for(var i=0;i<arr.length;i++){
var d = arr[i];
//处理下拉框，读取展示文字
var selectWrap = d.closest(".el-select");
if(selectWrap){
//优先取input_value
var showVal =d.value || selectWrap.querySelector(".el-input__inner").value || "";
d.setAttribute("value",showVal);
}else{
d.setAttribute("value",d.value);
}
}
''') # var showVal =d.value || selectWrap.querySelector(".el-input__inner").value || "";
page_cx = driver.page_source
soup_cx = BeautifulSoup(page_cx, "html.parser")
#print(soup_cx)
# # 查找小项标题字段
cx_all_html = soup_cx.select(
'div[data-v-name="tsscCollapseItem"]')
valid_block = [cx for cx in cx_all_html ] # 剔除基本信息页签
df_pivot_new = pd.DataFrame()
df2_pivot_new = pd.DataFrame()
total_pivot_df =pd.DataFrame()
input_col_dict={}
for idx,item in enumerate(valid_block):
input_col_list = []
title_col = item.find("span",
class_="el-tooltip item header-title") # 获取板块标题 #find("span",class_="el-tooltip item header-title") #获取标题
#print(title_col)
single_table =item.select(".myTable")
#展示型字段
for num,table in enumerate(single_table):
filed_names = []
table_header = table.select('.el-table__header-wrapper .el-table__header .cell') # 获取字段
for cell in table_header:
raw =cell.get_text(strip=True)
if raw not in ["用户名称","用户编号",'']:
filed_names.append(raw)
#filed_names = [cell.get_text(strip=True) for cell in table_header if cell.get_text(strip=True) not in ["用户名称","用户编号",'']] # 表字段列表
#查找字段值
filed_names_len = len(filed_names)
cell_list = table.select('.el-table__body-wrapper .el-table__body .cell')
if cell_list:
# 有数据则正常取数
data_col = [cell.get_text(strip=True) for cell in cell_list] # 获取表数据
else: #无数据时，则按照空列表填充
data_col = [""] *filed_names_len
# 一维列表转二维列表
if filed_names_len > 0:
data_col_new = [data_col[i:i + filed_names_len] for i in range(0, len(data_col), filed_names_len)]
else:
data_col_new = []
if len(filed_names) > 0 and len(data_col_new) > 0:
df2 = pd.DataFrame(data_col_new, columns=filed_names)
else:
df2 = pd.DataFrame([], columns=filed_names)
df2_pivot = df2.T
df2_pivot.reset_index(inplace=True)
# 无值时新增列
if df2_pivot.shape[1] == 0:
df2_pivot[0] = ""
df2_pivot.rename(columns={"index": "新字段", 0: "新字段值"}, inplace=True)
if df2_pivot.shape[1] >= 3:
df2_pivot.rename(columns={1: "新字段值2"}, inplace=True)
df2_pivot.insert(0, "新增字段1", "字段1值")
df2_pivot.insert(1, "新增字段2", f"字段2值-{title_col.get_text(strip=True)}")
#df2_pivot.insert(0, "序号", f"{idx}-{num}")
suffix_list = [f"{idx}-{num}-{i}" for i in range(len(df2_pivot))]
df2_pivot.insert(0, "序号", suffix_list)
#df2_pivot["序号"]=suffix_list
df2_pivot_new = pd.concat([df2_pivot_new, df2_pivot], ignore_index=True)
# 查找输入型字段
xq_lable_data = item.select(".el-form-item[prop]")
filed_names_input = [item.select_one(".el-form-item__label").get_text(strip=True) for item in xq_lable_data]
for item in xq_lable_data:
lab = item.select_one(".el-form-item__label") # 字段中文名称
inp = item.select_one(".el-input__inner") or item.select_one(".el-textarea__inner")
if inp:
var = inp.get("value", "").strip() # 字段值
input_col_dict[lab.get_text(strip=True)] = var
print(f"生成 {title_col.get_text(strip=True)} 标题完毕")
#只保留本次字段和值
new_col_dict ={k:input_col_dict[k] for k in input_col_dict if k in filed_names_input}
input_col_list.append(new_col_dict)
if len(input_col_list)>0:
df = pd.DataFrame(input_col_list)
df_pivot = df.T
df_pivot.reset_index(inplace=True)
df_pivot.rename(columns={"index": "新字段", 0: "新字段值"}, inplace=True)
df_pivot.insert(0, "新增字段1", "字段1值")
df_pivot.insert(1, f"新增字段2", f"字段2值-{title_col.get_text(strip=True)}")
df_pivot["新字段值2"] = ""
df_pivot.insert(0, "序号", str(idx))
df_pivot_new = pd.concat([df_pivot_new, df_pivot], ignore_index=True)
total_pivot_df = pd.concat([df_pivot_new, df2_pivot_new], ignore_index=True)
total_pivot_df["临时序号"] = total_pivot_df["序号"].str.split("-").apply(lambda x:tuple(int(i) for i in x))
df2_pivot_new2 = total_pivot_df.sort_values(by="临时序号")
df2_pivot_new2.drop(columns=["临时序号"],inplace=True)
write_file(df2_pivot_new2, out_put_flie, 'w', 'Sheet1')
finally:
driver.quit()
#pass
#入口函数
yp_zl="分类" #
yp_bh ="输入查询字段值"#
out_put_flie = r"E:\临时文件\los_output\客户-" + yp_zl + ".xlsx"
acess_new_website(out_put_flie,yp_bh)
这里是“一人公司”的成长家园。我们提供从产品曝光、技术变现到法律财税的全栈内容，并连接云服务、办公空间等稀缺资源，助你专注创造，无忧运营。
更多推荐
6
0- 0
已为社区贡献1条内容
所有评论(0)