mirror of
https://github.com/mindverse/Second-Me.git
synced 2026-10-24 15:41:06 +00:00
f5bb0dad59
* Add code for data filtering llm judge * Ignore log file created on root (mainly for synthetic_data_generation.log) * Fix metadata API compatibility issues by commenting out metadata tags in LLM API calls - Commented out metadata.tags parameters in all LLM API calls across the codebase - This fixes compatibility issues with custom LLM providers that don't support metadata - Affects shades generation, topics generation, wiki generation, bio QA, and question generation - Preserves the original code structure for future re-enabling if needed * feat: add data filtering pipeline with Ollama integration - Add MergedDataJudge class for intelligent data filtering using Ollama Gemma - Integrate automatic Ollama CLI installation into project setup process - Add DATA_FILTERING step to training pipeline with concurrent processing - Include testing for MergedDataJudge in its local main() function - Add Ollama dependency to pyproject.toml * feat: add automatic Ollama model cleanup after data filtering * Add logging for outputting data filtering parameters * fix: adjust error handling for MergedDataJudge: - Keep original merged.json unchanged when any error occurs - Exit filtering process immediately on errors instead of continuing with defaults - Ensure training pipeline continues safely even if data filtering fails * Add frontend for data filtering pipeline * resolve data filtering quality_level error by commenting out problematic fields, change TrainProcessService back to original class definition * fix: quote unquoted shade icons to prevent JSON parsing errors * Fixed wiki_res.json missing due to no database connection at wiki/base.py module import * Added scoring reasoning as part of the merged data * fix: filter ANSI escape sequences from Ollama logs in data filtering step * fix: Add data filtering steps to cloud training to resolve KeyError - Added 'Data Filtering' step to cloud training progress holder - Added data filtering step execution in cloud training service - Added data filtering parameters to cloud training routes - Updated frontend to send data filtering parameters - Fixed missing except clause in cloud training service This resolves the KeyError: 'data_filtering' when switching from cloud to local training.
63 lines
1.6 KiB
TOML
63 lines
1.6 KiB
TOML
[tool.poetry]
|
|
name = "lpm_kernel"
|
|
version = "0.1.0"
|
|
description = ""
|
|
authors = ["Kevin <kevin@mindverse.ai>"]
|
|
readme = "README.md"
|
|
|
|
[tool.poetry.dependencies]
|
|
python = ">=3.12,<3.13"
|
|
flask = "^3.0.0"
|
|
flask-sock = "^0.6.0" # Downgraded to stable version
|
|
sqlalchemy = {extras = ["asyncio"], version = "^2.0.0"}
|
|
tiktoken = "^0.8.0"
|
|
aiomysql = "^0.2.0"
|
|
python-dotenv = "^1.0.0"
|
|
aiohttp = "^3.9.1"
|
|
asyncio = "^3.4.3"
|
|
async-timeout = "^4.0.0"
|
|
pytesseract = "^0.3.10"
|
|
pdfplumber = "^0.10.3"
|
|
flask-pydantic = "^0.11.0"
|
|
pydantic = "^2.5.2"
|
|
pymupdf = "^1.23.7"
|
|
scikit-learn = "^1.3.0"
|
|
requests = "^2.31.0"
|
|
numpy = "^1.26.0"
|
|
openai = "^1.50.0"
|
|
langchain = "0.3.13"
|
|
chromadb = "^0.4.22"
|
|
sentencepiece = "^0.2.0"
|
|
httpx = {extras = ["socks"], version = "^0.28.1"}
|
|
python-socks = "^2.4.0"
|
|
jsonlines = "^4.0.0"
|
|
charset-normalizer = "^3.4.1"
|
|
modelscope = "^1.25.0"
|
|
sentence-transformers = "^2.6.0"
|
|
levenshtein = "0.27.1"
|
|
|
|
# Development environment dependencies
|
|
# Use 'poetry install --with dev' to install development dependencies
|
|
ollama = "^0.5.1"
|
|
[tool.poetry.group.dev.dependencies]
|
|
pytest = "7.4.4"
|
|
ruff = "0.1.15"
|
|
pandas = "2.2.3"
|
|
fnllm = {extras = ["azure", "openai"], version = "0.1.2"}
|
|
transformers = "4.51.0"
|
|
torch = "2.5.1"
|
|
peft = "0.14.0"
|
|
trl = "0.13.0"
|
|
gguf = "^0.17.0"
|
|
datasets = "3.3.2"
|
|
jiter = "0.8.2"
|
|
|
|
# Documentation environment dependencies
|
|
# Use 'poetry install --with docs' to install documentation dependencies
|
|
[tool.poetry.group.docs.dependencies]
|
|
mkdocs-material = "9.5.3"
|
|
mkdocs = "1.5.3"
|
|
|
|
[build-system]
|
|
requires = ["poetry-core>=1.0.0"]
|
|
build-backend = "poetry.core.masonry.api"
|