91 lines
3.6 KiB
Python
91 lines
3.6 KiB
Python
# -*- coding: utf-8 -*-
|
||
import os
|
||
import pandas as pd
|
||
import io
|
||
|
||
def load_and_profile_data(file_paths: list) -> str:
|
||
"""
|
||
加载数据并生成数据画像
|
||
|
||
Args:
|
||
file_paths: 文件路径列表
|
||
|
||
Returns:
|
||
包含数据画像的Markdown字符串
|
||
"""
|
||
profile_summary = "# 数据画像报告 (Data Profile)\n\n"
|
||
|
||
if not file_paths:
|
||
return profile_summary + "未提供数据文件。"
|
||
|
||
for file_path in file_paths:
|
||
file_name = os.path.basename(file_path)
|
||
profile_summary += f"## 文件: {file_name}\n\n"
|
||
|
||
if not os.path.exists(file_path):
|
||
profile_summary += f"⚠️ 文件不存在: {file_path}\n\n"
|
||
continue
|
||
|
||
try:
|
||
# 根据扩展名选择加载方式
|
||
ext = os.path.splitext(file_path)[1].lower()
|
||
if ext == '.csv':
|
||
# 尝试多种编码
|
||
try:
|
||
df = pd.read_csv(file_path, encoding='utf-8')
|
||
except UnicodeDecodeError:
|
||
try:
|
||
df = pd.read_csv(file_path, encoding='gbk')
|
||
except Exception:
|
||
df = pd.read_csv(file_path, encoding='latin1')
|
||
elif ext in ['.xlsx', '.xls']:
|
||
df = pd.read_excel(file_path)
|
||
else:
|
||
profile_summary += f"⚠️ 不支持的文件格式: {ext}\n\n"
|
||
continue
|
||
|
||
# 基础信息
|
||
rows, cols = df.shape
|
||
profile_summary += f"- **维度**: {rows} 行 x {cols} 列\n"
|
||
profile_summary += f"- **列名**: `{', '.join(df.columns)}`\n\n"
|
||
|
||
profile_summary += "### 列详细分布:\n"
|
||
|
||
# 遍历分析每列
|
||
for col in df.columns:
|
||
dtype = df[col].dtype
|
||
null_count = df[col].isnull().sum()
|
||
null_ratio = (null_count / rows) * 100
|
||
|
||
profile_summary += f"#### {col} ({dtype})\n"
|
||
if null_count > 0:
|
||
profile_summary += f"- ⚠️ 空值: {null_count} ({null_ratio:.1f}%)\n"
|
||
|
||
# 数值列分析
|
||
if pd.api.types.is_numeric_dtype(dtype):
|
||
desc = df[col].describe()
|
||
profile_summary += f"- 统计: Min={desc['min']:.2f}, Max={desc['max']:.2f}, Mean={desc['mean']:.2f}\n"
|
||
|
||
# 文本/分类列分析
|
||
elif pd.api.types.is_object_dtype(dtype) or pd.api.types.is_categorical_dtype(dtype):
|
||
unique_count = df[col].nunique()
|
||
profile_summary += f"- 唯一值数量: {unique_count}\n"
|
||
|
||
# 如果唯一值较少（<50）或者看起来是分类数据，显示Top分布
|
||
# 这对识别“高频问题”至关重要
|
||
if unique_count > 0:
|
||
top_n = df[col].value_counts().head(5)
|
||
top_items_str = ", ".join([f"{k}({v})" for k, v in top_n.items()])
|
||
profile_summary += f"- **TOP 5 高频值**: {top_items_str}\n"
|
||
|
||
# 时间列分析
|
||
elif pd.api.types.is_datetime64_any_dtype(dtype):
|
||
profile_summary += f"- 范围: {df[col].min()} 至 {df[col].max()}\n"
|
||
|
||
profile_summary += "\n"
|
||
|
||
except Exception as e:
|
||
profile_summary += f"❌ 读取或分析文件失败: {str(e)}\n\n"
|
||
|
||
return profile_summary
|