feat(skills): add environment-management skill
Conda + uv hybrid environment setup for ML projects.
This commit is contained in:
parent
849cfe5409
commit
d376cc113a
224
.opencode/skills/environment-management/SKILL.md
Normal file
224
.opencode/skills/environment-management/SKILL.md
Normal file
@ -0,0 +1,224 @@
|
||||
---
|
||||
name: environment-management
|
||||
description: Use when setting up Python environment for ML/DL paper replication using Conda + uv
|
||||
---
|
||||
|
||||
# Environment Management (Conda + uv)
|
||||
|
||||
## Overview
|
||||
|
||||
Hybrid approach using Conda for system-level dependencies and uv for project isolation.
|
||||
|
||||
**Announce at start:** "I'm using the environment-management skill for Conda + uv setup."
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────┐
|
||||
│ Conda (System Base) │
|
||||
│ - Python interpreter │
|
||||
│ - CUDA toolkit │
|
||||
│ - System-level C++ libraries │
|
||||
└─────────────────────────────────────────┘
|
||||
│
|
||||
│ provides Python
|
||||
▼
|
||||
┌─────────────────────────────────────────┐
|
||||
│ uv (Project Isolation) │
|
||||
│ - Per-project .venv │
|
||||
│ - Fast dependency resolution │
|
||||
│ - Reproducible installs │
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Setup Commands
|
||||
|
||||
### Step 1: Conda Base Environment
|
||||
|
||||
Check if base exists:
|
||||
```bash
|
||||
conda env list | grep ai_base
|
||||
```
|
||||
|
||||
Create if needed:
|
||||
```bash
|
||||
# Linux/Mac
|
||||
conda create -n ai_base python=3.10 cuda-toolkit=11.8 -y
|
||||
|
||||
# Windows (CUDA from NVIDIA, not conda)
|
||||
conda create -n ai_base python=3.10 -y
|
||||
```
|
||||
|
||||
### Step 2: Project Environment
|
||||
|
||||
```bash
|
||||
cd workspace/{paper_name}
|
||||
|
||||
# Get Conda Python path
|
||||
# Linux/Mac:
|
||||
PYTHON_PATH=$(conda run -n ai_base which python)
|
||||
|
||||
# Windows:
|
||||
# PYTHON_PATH=$(conda run -n ai_base python -c "import sys; print(sys.executable)")
|
||||
|
||||
# Create uv venv
|
||||
uv venv --python $PYTHON_PATH
|
||||
```
|
||||
|
||||
### Step 3: Activate and Install
|
||||
|
||||
```bash
|
||||
# Linux/Mac
|
||||
source .venv/bin/activate
|
||||
|
||||
# Windows
|
||||
.venv\Scripts\activate
|
||||
|
||||
# Install dependencies
|
||||
uv pip install -e ".[dev]"
|
||||
```
|
||||
|
||||
## pyproject.toml Template
|
||||
|
||||
```toml
|
||||
[project]
|
||||
name = "{paper_name}"
|
||||
version = "0.1.0"
|
||||
description = "Replication of {paper_title}"
|
||||
requires-python = ">=3.10"
|
||||
|
||||
dependencies = [
|
||||
# Core ML
|
||||
"torch>=2.0.0",
|
||||
"numpy>=1.24.0",
|
||||
|
||||
# Visualization
|
||||
"matplotlib>=3.7.0",
|
||||
"seaborn>=0.12.0",
|
||||
|
||||
# Utilities
|
||||
"tqdm>=4.65.0",
|
||||
"pyyaml>=6.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=7.0.0",
|
||||
"pytest-cov>=4.0.0",
|
||||
"black>=23.0.0",
|
||||
"ruff>=0.0.260",
|
||||
]
|
||||
|
||||
# Add based on paper requirements
|
||||
vision = [
|
||||
"torchvision>=0.15.0",
|
||||
"pillow>=9.5.0",
|
||||
]
|
||||
|
||||
nlp = [
|
||||
"transformers>=4.30.0",
|
||||
"tokenizers>=0.13.0",
|
||||
"datasets>=2.12.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
python_files = ["test_*.py"]
|
||||
addopts = "-v --tb=short"
|
||||
|
||||
[tool.black]
|
||||
line-length = 88
|
||||
target-version = ["py310"]
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 88
|
||||
select = ["E", "F", "I", "N", "W"]
|
||||
```
|
||||
|
||||
## PyTorch + CUDA Compatibility
|
||||
|
||||
| CUDA Version | PyTorch Version | Install Command |
|
||||
|--------------|-----------------|-----------------|
|
||||
| 11.8 | 2.0+ | `uv pip install torch --index-url https://download.pytorch.org/whl/cu118` |
|
||||
| 12.1 | 2.1+ | `uv pip install torch --index-url https://download.pytorch.org/whl/cu121` |
|
||||
| CPU only | Any | `uv pip install torch --index-url https://download.pytorch.org/whl/cpu` |
|
||||
|
||||
## Environment Verification
|
||||
|
||||
```bash
|
||||
# Check Python
|
||||
python --version
|
||||
|
||||
# Check PyTorch
|
||||
python -c "import torch; print(f'PyTorch {torch.__version__}')"
|
||||
|
||||
# Check CUDA
|
||||
python -c "import torch; print(f'CUDA available: {torch.cuda.is_available()}')"
|
||||
python -c "import torch; print(f'CUDA version: {torch.version.cuda}')"
|
||||
|
||||
# Check GPU
|
||||
python -c "import torch; print(f'GPU: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"N/A\"}')"
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### CUDA Not Found
|
||||
|
||||
```bash
|
||||
# Check NVIDIA driver
|
||||
nvidia-smi
|
||||
|
||||
# Reinstall PyTorch with correct CUDA
|
||||
uv pip install torch --index-url https://download.pytorch.org/whl/cu118 --force-reinstall
|
||||
```
|
||||
|
||||
### Dependency Conflicts
|
||||
|
||||
```bash
|
||||
# Clear cache and reinstall
|
||||
uv cache clean
|
||||
uv pip install -e ".[dev]" --force-reinstall
|
||||
```
|
||||
|
||||
### Permission Errors (Windows)
|
||||
|
||||
```powershell
|
||||
# Run as Administrator or:
|
||||
Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **One environment per paper**: Don't mix dependencies
|
||||
2. **Pin versions in pyproject.toml**: For reproducibility
|
||||
3. **Use dev dependencies**: Keep test tools separate
|
||||
4. **Document CUDA version**: In README.md
|
||||
5. **Commit pyproject.toml**: Not .venv/
|
||||
|
||||
## Quick Reference
|
||||
|
||||
```bash
|
||||
# Full setup sequence (Linux/Mac)
|
||||
conda activate ai_base || conda create -n ai_base python=3.10 cuda-toolkit=11.8 -y && conda activate ai_base
|
||||
cd workspace/{paper_name}
|
||||
uv venv --python $(which python)
|
||||
source .venv/bin/activate
|
||||
uv pip install -e ".[dev]"
|
||||
pytest tests/ -v
|
||||
```
|
||||
|
||||
```powershell
|
||||
# Full setup sequence (Windows)
|
||||
conda activate ai_base
|
||||
# If not exists: conda create -n ai_base python=3.10 -y && conda activate ai_base
|
||||
cd workspace\{paper_name}
|
||||
$env:PYTHON_PATH = (conda run -n ai_base python -c "import sys; print(sys.executable)")
|
||||
uv venv --python $env:PYTHON_PATH
|
||||
.venv\Scripts\activate
|
||||
uv pip install -e ".[dev]"
|
||||
pytest tests\ -v
|
||||
```
|
||||
Loading…
Reference in New Issue
Block a user