zhaojie/iov_data_analysis_agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

清理表情

Browse Source
This commit is contained in:
Zhaojie
2026-01-31 18:00:05 +08:00
parent 674f48c74b
commit 5eb13324c2
15 changed files with 394 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
utils/data_loader.py
View File

@@ -34,7 +34,7 @@ def load_and_profile_data(file_paths: list) -> str:
profile_summary += f"## 文件: {file_name}\n\n"
if not os.path.exists(file_path):
profile_summary += f"⚠️ 文件不存在: {file_path}\n\n"
profile_summary += f"[WARN] 文件不存在: {file_path}\n\n"
continue
try:
@@ -52,7 +52,7 @@ def load_and_profile_data(file_paths: list) -> str:
elif ext in ['.xlsx', '.xls']:
df = pd.read_excel(file_path)
else:
profile_summary += f"⚠️ 不支持的文件格式: {ext}\n\n"
profile_summary += f"[WARN] 不支持的文件格式: {ext}\n\n"
continue
# 基础信息
@@ -70,7 +70,7 @@ def load_and_profile_data(file_paths: list) -> str:
profile_summary += f"#### {col} ({dtype})\n"
if null_count > 0:
profile_summary += f"- ⚠️ 空值: {null_count} ({null_ratio:.1f}%)\n"
profile_summary += f"- [WARN] 空值: {null_count} ({null_ratio:.1f}%)\n"
# 数值列分析
if pd.api.types.is_numeric_dtype(dtype):
@@ -96,7 +96,7 @@ def load_and_profile_data(file_paths: list) -> str:
profile_summary += "\n"
except Exception as e:
profile_summary += f"❌ 读取或分析文件失败: {str(e)}\n\n"
profile_summary += f"[ERROR] 读取或分析文件失败: {str(e)}\n\n"
return profile_summary
@@ -141,7 +141,7 @@ def load_data_chunked(file_path: str, chunksize: Optional[int] = None) -> Iterat
except UnicodeDecodeError:
continue
except Exception as e:
print(f"❌ 读取CSV文件失败: {e}")
print(f"[ERROR] 读取CSV文件失败: {e}")
break
elif ext in ['.xlsx', '.xls']:
# Excel文件不支持chunksize，直接读取
@@ -151,7 +151,7 @@ def load_data_chunked(file_path: str, chunksize: Optional[int] = None) -> Iterat
for i in range(0, len(df), chunksize):
yield df.iloc[i:i+chunksize]
except Exception as e:
print(f"❌ 读取Excel文件失败: {e}")
print(f"[ERROR] 读取Excel文件失败: {e}")
def load_data_with_cache(file_path: str, force_reload: bool = False) -> Optional[pd.DataFrame]:
@@ -166,7 +166,7 @@ def load_data_with_cache(file_path: str, force_reload: bool = False) -> Optional
DataFrame或None
"""
if not os.path.exists(file_path):
print(f"⚠️ 文件不存在: {file_path}")
print(f"[WARN] 文件不存在: {file_path}")
return None
# 检查文件大小
@@ -174,7 +174,7 @@ def load_data_with_cache(file_path: str, force_reload: bool = False) -> Optional
# 对于大文件，建议使用流式处理
if file_size_mb > app_config.max_file_size_mb:
print(f"⚠️ 文件过大 ({file_size_mb:.1f}MB)，建议使用 load_data_chunked() 流式处理")
print(f"[WARN] 文件过大 ({file_size_mb:.1f}MB)，建议使用 load_data_chunked() 流式处理")
# 生成缓存键
cache_key = get_file_hash(file_path)
@@ -183,7 +183,7 @@ def load_data_with_cache(file_path: str, force_reload: bool = False) -> Optional
if not force_reload and app_config.data_cache_enabled:
cached_data = data_cache.get(cache_key)
if cached_data is not None:
print(f"💾 从缓存加载数据: {os.path.basename(file_path)}")
print(f"[CACHE] 从缓存加载数据: {os.path.basename(file_path)}")
return cached_data
# 加载数据
@@ -202,16 +202,16 @@ def load_data_with_cache(file_path: str, force_reload: bool = False) -> Optional
elif ext in ['.xlsx', '.xls']:
df = pd.read_excel(file_path)
else:
print(f"⚠️ 不支持的文件格式: {ext}")
print(f"[WARN] 不支持的文件格式: {ext}")
return None
# 缓存数据
if df is not None and app_config.data_cache_enabled:
data_cache.set(cache_key, df)
print(f"✅ 数据已缓存: {os.path.basename(file_path)}")
print(f"[OK] 数据已缓存: {os.path.basename(file_path)}")
return df
except Exception as e:
print(f"❌ 加载数据失败: {e}")
print(f"[ERROR] 加载数据失败: {e}")
return None