From 1b1464e278b59d0765f0c7c19b07748068d7a955 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 22:50:55 +0000 Subject: [PATCH] Create AI chat agent for VS Code with Docker service management Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- .env.example | 48 ++ .github/copilot-instructions.md | 253 +++++++++ .gitignore | 76 +++ README.md | 379 ++++++++++++- docker-compose/README.md | 228 ++++++++ docker-compose/development.yml | 187 +++++++ docker-compose/infrastructure.yml | 107 ++++ docker-compose/media.yml | 171 ++++++ docker-compose/monitoring.yml | 173 ++++++ docs/docker-guidelines.md | 858 ++++++++++++++++++++++++++++++ docs/quick-reference.md | 394 ++++++++++++++ 11 files changed, 2873 insertions(+), 1 deletion(-) create mode 100644 .env.example create mode 100644 .github/copilot-instructions.md create mode 100644 .gitignore create mode 100644 docker-compose/README.md create mode 100644 docker-compose/development.yml create mode 100644 docker-compose/infrastructure.yml create mode 100644 docker-compose/media.yml create mode 100644 docker-compose/monitoring.yml create mode 100644 docs/docker-guidelines.md create mode 100644 docs/quick-reference.md diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..ee0642c --- /dev/null +++ b/.env.example @@ -0,0 +1,48 @@ +# Environment Variables Template +# Copy this file to .env and fill in your values +# NEVER commit .env to git! + +# User and Group IDs (get with: id -u and id -g) +PUID=1000 +PGID=1000 + +# Timezone (list: timedatectl list-timezones) +TZ=America/New_York + +# Server IP address +SERVER_IP=192.168.1.100 + +# Directory Paths +USERDIR=/home/username/homelab +MEDIADIR=/mnt/media +DOWNLOADDIR=/mnt/downloads +PROJECTDIR=/home/username/projects + +# Plex Configuration +PLEX_CLAIM=claim-xxxxxxxxxx + +# Monitoring Passwords +GRAFANA_ADMIN_PASSWORD=changeme + +# Code Server Passwords +CODE_SERVER_PASSWORD=changeme +CODE_SERVER_SUDO_PASSWORD=changeme + +# Database Credentials +POSTGRES_USER=postgres +POSTGRES_PASSWORD=changeme +POSTGRES_DB=homelab + +PGADMIN_EMAIL=admin@example.com +PGADMIN_PASSWORD=changeme + +# Jupyter Token +JUPYTER_TOKEN=changeme + +# Pi-hole +PIHOLE_PASSWORD=changeme + +# Watchtower Notifications (optional) +# WATCHTOWER_NOTIFICATION_URL= + +# Add your own variables below diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..1186bc5 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,253 @@ +# AI Homelab Management Assistant + +You are an AI assistant specialized in managing Docker-based homelab infrastructure. Your role is to help users create, modify, and manage Docker services while maintaining consistency across the entire server stack. + +## Core Principles + +### 1. Docker Compose First +- **ALWAYS** use Docker Compose stacks for persistent services +- Only use `docker run` for temporary containers (e.g., testing nvidia-container-toolkit functionality) +- Maintain all services in organized docker-compose.yml files + +### 2. Consistency is Key +- Keep consistent naming conventions across all compose files +- Use the same network naming patterns +- Maintain uniform volume mount structures +- Apply consistent environment variable patterns + +### 3. Stack-Aware Changes +- Before making changes, consider the impact on the entire server stack +- Check for service dependencies (networks, volumes, other services) +- Ensure changes don't break existing integrations +- Validate that port assignments don't conflict + +## Creating a New Docker Service + +When creating a new service, follow these steps: + +1. **Assess the Stack** + - Review existing services and their configurations + - Check for available ports + - Identify shared networks and volumes + - Note any dependent services + +2. **Choose the Right Location** + - Place related services in the same compose file + - Use separate compose files for different functional areas (e.g., monitoring, media, development) + - Keep the file structure organized by category + +3. **Service Definition Template** + ```yaml + services: + service-name: + image: image:tag # Always pin versions for stability + container_name: service-name # Use descriptive, consistent names + restart: unless-stopped # Standard restart policy + networks: + - homelab-network # Use shared networks + ports: + - "host_port:container_port" # Document port purpose + volumes: + - ./config/service-name:/config # Config in local directory + - service-data:/data # Named volumes for persistent data + environment: + - PUID=1000 # Standard user/group IDs + - PGID=1000 + - TZ=America/New_York # Consistent timezone + labels: + - "homelab.category=category-name" # For organization + - "homelab.description=Service description" + + volumes: + service-data: + driver: local + + networks: + homelab-network: + external: true # Or define once in main compose + ``` + +4. **Configuration Best Practices** + - Pin image versions (avoid `:latest` in production) + - Use environment variables for configuration + - Store sensitive data in `.env` files (never commit these!) + - Use named volumes for data that should persist + - Bind mount config directories for easy access + +5. **Documentation** + - Add comments explaining non-obvious configurations + - Document port mappings and their purposes + - Note any special requirements or dependencies + +## Editing an Existing Service + +When modifying a service: + +1. **Review Current Configuration** + - Read the entire service definition + - Check for dependencies (links, depends_on, networks) + - Note any volumes or data that might be affected + +2. **Plan the Change** + - Identify what needs to change + - Consider backward compatibility + - Plan for data migration if needed + +3. **Make Minimal Changes** + - Change only what's necessary + - Maintain existing patterns and conventions + - Keep the same structure unless there's a good reason to change it + +4. **Validate the Change** + - Check YAML syntax + - Verify port availability + - Ensure network connectivity + - Test the service starts correctly + +5. **Update Documentation** + - Update comments if behavior changes + - Revise README files if user interaction changes + +## Common Operations + +### Testing a New Image +```bash +# Use docker run for quick tests, then convert to compose +docker run --rm -it \ + --name test-container \ + image:tag \ + command +``` + +### Checking NVIDIA GPU Access +```bash +# Temporary test container for GPU +docker run --rm --gpus all nvidia/cuda:12.0.0-base-ubuntu22.04 nvidia-smi +``` + +### Deploying a Stack +```bash +# Start all services in a compose file +docker compose -f docker-compose.yml up -d + +# Start specific services +docker compose -f docker-compose.yml up -d service-name +``` + +### Updating a Service +```bash +# Pull latest image (if version updated) +docker compose -f docker-compose.yml pull service-name + +# Recreate the service +docker compose -f docker-compose.yml up -d service-name +``` + +### Checking Logs +```bash +# View logs for a service +docker compose -f docker-compose.yml logs -f service-name +``` + +## Network Management + +### Standard Network Setup +- Use a shared bridge network for inter-service communication +- Name it consistently (e.g., `homelab-network`) +- Define it once in a main compose file or create it manually + +### Network Isolation +- Use separate networks for different security zones +- Keep databases on internal networks only +- Expose only necessary services to external networks + +## Volume Management + +### Volume Strategy +- **Named volumes**: For data that should persist but doesn't need direct access +- **Bind mounts**: For configs you want to edit directly +- **tmpfs**: For temporary data that should not persist + +### Backup Considerations +- Keep important data in well-defined volumes +- Document backup procedures for each service +- Use consistent paths for easier backup automation + +## Environment Variables + +### Standard Variables +```yaml +environment: + - PUID=1000 # User ID for file permissions + - PGID=1000 # Group ID for file permissions + - TZ=America/New_York # Timezone + - UMASK=022 # File creation mask +``` + +### Sensitive Data +- Store secrets in `.env` files +- Reference them in compose: `${VARIABLE_NAME}` +- Never commit `.env` files to git +- Provide `.env.example` templates + +## Troubleshooting + +### Service Won't Start +1. Check logs: `docker compose logs service-name` +2. Verify configuration syntax +3. Check for port conflicts +4. Verify volume mounts exist +5. Check network connectivity + +### Permission Issues +1. Verify PUID/PGID match host user +2. Check directory permissions +3. Verify volume ownership + +### Network Issues +1. Verify network exists: `docker network ls` +2. Check if services are on same network +3. Use service names for DNS resolution +4. Check firewall rules + +## File Organization + +``` +/home/user/homelab/ +├── docker-compose/ +│ ├── media.yml # Media server services +│ ├── monitoring.yml # Monitoring stack +│ ├── development.yml # Dev tools +│ └── infrastructure.yml # Core services +├── config/ +│ ├── service1/ +│ ├── service2/ +│ └── ... +├── data/ # Bind mount data +│ └── ... +├── .env # Global secrets (gitignored) +└── README.md # Stack documentation +``` + +## Safety Checks + +Before deploying any changes: +- [ ] YAML syntax is valid +- [ ] Ports don't conflict with existing services +- [ ] Networks exist or are defined +- [ ] Volume paths are correct +- [ ] Environment variables are set +- [ ] No secrets in compose files +- [ ] Service dependencies are met +- [ ] Backup of current configuration exists + +## Remember + +- **Think before you act**: Consider the entire stack +- **Be consistent**: Follow established patterns +- **Document everything**: Future you will thank you +- **Test safely**: Use temporary containers first +- **Back up first**: Always have a rollback plan +- **Security matters**: Keep secrets secret, update regularly + +When a user asks you to create or modify a Docker service, follow these guidelines carefully, ask clarifying questions if needed, and always prioritize the stability and consistency of the entire homelab infrastructure. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7afc29a --- /dev/null +++ b/.gitignore @@ -0,0 +1,76 @@ +# Environment variables and secrets +.env +*.env +!.env.example + +# Config directories with sensitive data +config/*/secrets/ +config/*/*.key +config/*/*.pem +config/*/*.crt +config/*/db/ + +# Backup files +*.backup +*.bak +backups/ + +# OS files +.DS_Store +Thumbs.db +*.swp +*.swo +*~ + +# Editor files +.vscode/ +.idea/ +*.sublime-* + +# Logs +*.log +logs/ + +# Temporary files +tmp/ +temp/ +*.tmp + +# Docker volumes (if locally mounted) +volumes/ + +# Documentation builds +docs/_build/ +docs/.doctrees/ + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +venv/ +env/ + +# Node +node_modules/ +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Database files +*.sqlite +*.db + +# Certificates and keys +*.pem +*.key +*.crt +*.cer +*.p12 +*.pfx + +# Monitoring data (if stored locally) +prometheus-data/ +grafana-data/ +loki-data/ diff --git a/README.md b/README.md index cc6fde9..46852ea 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,379 @@ # AI-Homelab -AI Powered Homelab administration + +AI-Powered Homelab Administration with GitHub Copilot + +## Overview + +This repository provides a structured approach to managing a homelab infrastructure using Docker Compose, with integrated AI assistance through GitHub Copilot. The AI assistant is specifically trained to help you create, modify, and manage Docker services while maintaining consistency across your entire server stack. + +## Features + +- **AI-Powered Management**: GitHub Copilot integration with specialized instructions for Docker service management +- **Docker Compose First**: All persistent services defined in organized compose files +- **Consistent Structure**: Standardized naming conventions and patterns across all services +- **Stack-Aware Changes**: AI considers the entire infrastructure when making changes +- **Comprehensive Documentation**: Detailed guidelines and examples for all operations +- **Example Services**: Ready-to-use compose files for common homelab services + +## Quick Start + +### Prerequisites + +- Docker Engine 24.0+ installed +- Docker Compose V2 +- Git +- VS Code with GitHub Copilot extension (for AI assistance) + +### Initial Setup + +1. **Clone the repository:** + ```bash + git clone https://github.com/kelinfoxy/AI-Homelab.git + cd AI-Homelab + ``` + +2. **Create environment file:** + ```bash + cp .env.example .env + # Edit .env with your values + nano .env + ``` + +3. **Create the main network:** + ```bash + docker network create homelab-network + ``` + +4. **Create config directories:** + ```bash + mkdir -p config + ``` + +5. **Start your first service:** + ```bash + # Example: Start Portainer for container management + docker compose -f docker-compose/infrastructure.yml up -d portainer + ``` + +6. **Access the service:** + Open `http://your-server-ip:9000` in your browser + +## Repository Structure + +``` +AI-Homelab/ +├── .github/ +│ └── copilot-instructions.md # AI assistant guidelines +├── docker-compose/ +│ ├── infrastructure.yml # Core services (proxy, DNS, Portainer) +│ ├── media.yml # Media services (Plex, Sonarr, Radarr) +│ ├── monitoring.yml # Observability (Prometheus, Grafana) +│ ├── development.yml # Dev tools (code-server, databases) +│ └── README.md # Docker Compose documentation +├── docs/ +│ └── docker-guidelines.md # Comprehensive Docker guidelines +├── config/ # Service configurations (gitignored) +├── .env.example # Environment variable template +├── .gitignore # Git ignore patterns +└── README.md # This file +``` + +## Using the AI Assistant + +### In VS Code + +1. **Install GitHub Copilot** extension in VS Code +2. **Open this repository** in VS Code +3. **Start Copilot Chat** and ask questions like: + - "Help me add a new media service to my homelab" + - "Create a docker-compose file for Home Assistant" + - "How do I configure GPU support for Plex?" + - "Check my compose file for port conflicts" + +The AI assistant automatically follows the guidelines in `.github/copilot-instructions.md` to: +- Maintain consistency with existing services +- Use Docker Compose for all persistent services +- Consider the entire stack when making changes +- Follow naming conventions and best practices + +### Example Interactions + +**Creating a new service:** +``` +You: "I want to add Home Assistant to my homelab" + +Copilot: [Analyzes existing stack, checks for conflicts, creates compose configuration] +- Checks available ports +- Uses consistent naming +- Connects to appropriate networks +- Follows established patterns +``` + +**Modifying a service:** +``` +You: "Add GPU support to my Plex container" + +Copilot: [Reviews current Plex configuration and updates it] +- Checks if NVIDIA runtime is available +- Updates device mappings +- Adds required environment variables +- Maintains existing configuration +``` + +## Available Service Stacks + +### Infrastructure (`infrastructure.yml`) +- **Nginx Proxy Manager**: Web-based reverse proxy with SSL +- **Pi-hole**: Network-wide ad blocking and DNS +- **Portainer**: Docker container management UI +- **Watchtower**: Automatic container updates + +### Media (`media.yml`) +- **Plex**: Media streaming server +- **Jellyfin**: Open-source media server alternative +- **Sonarr**: TV show automation +- **Radarr**: Movie automation +- **Prowlarr**: Indexer manager +- **qBittorrent**: Torrent client + +### Monitoring (`monitoring.yml`) +- **Prometheus**: Metrics collection +- **Grafana**: Metrics visualization +- **Node Exporter**: Host metrics +- **cAdvisor**: Container metrics +- **Uptime Kuma**: Service uptime monitoring +- **Loki**: Log aggregation +- **Promtail**: Log shipping + +### Development (`development.yml`) +- **Code Server**: VS Code in browser +- **GitLab CE**: Self-hosted Git repository +- **PostgreSQL**: SQL database +- **Redis**: In-memory data store +- **pgAdmin**: PostgreSQL UI +- **Jupyter Lab**: Interactive notebooks +- **Node-RED**: Visual automation + +## Common Operations + +### Starting Services + +Start all services in a compose file: +```bash +docker compose -f docker-compose/infrastructure.yml up -d +``` + +Start specific services: +```bash +docker compose -f docker-compose/media.yml up -d plex sonarr radarr +``` + +### Stopping Services + +Stop all services: +```bash +docker compose -f docker-compose/infrastructure.yml down +``` + +Stop specific service: +```bash +docker compose -f docker-compose/media.yml stop plex +``` + +### Viewing Logs + +Follow logs for a service: +```bash +docker compose -f docker-compose/media.yml logs -f plex +``` + +View last 100 lines: +```bash +docker compose -f docker-compose/media.yml logs --tail=100 plex +``` + +### Updating Services + +Pull latest images: +```bash +docker compose -f docker-compose/media.yml pull +``` + +Update specific service: +```bash +docker compose -f docker-compose/media.yml pull plex +docker compose -f docker-compose/media.yml up -d plex +``` + +### Testing with Docker Run + +Test NVIDIA GPU support: +```bash +docker run --rm --gpus all nvidia/cuda:12.0.0-base-ubuntu22.04 nvidia-smi +``` + +Test a new image: +```bash +docker run --rm -it alpine:latest /bin/sh +``` + +## Network Architecture + +All services connect to networks for inter-service communication: + +- **homelab-network**: Main network for all services +- **media-network**: Isolated network for media stack +- **monitoring-network**: Network for observability stack +- **database-network**: Isolated network for databases + +Create networks manually: +```bash +docker network create homelab-network +docker network create media-network +docker network create monitoring-network +docker network create database-network +``` + +## Documentation + +### Comprehensive Guides + +- **[Docker Guidelines](docs/docker-guidelines.md)**: Complete guide to Docker service management +- **[Docker Compose README](docker-compose/README.md)**: Compose-specific documentation +- **[Copilot Instructions](.github/copilot-instructions.md)**: AI assistant guidelines + +### Key Principles + +1. **Docker Compose First**: Always use compose for persistent services +2. **Docker Run for Testing**: Only use `docker run` for temporary containers +3. **Consistency**: Follow established patterns and naming conventions +4. **Stack Awareness**: Consider dependencies and interactions +5. **Documentation**: Comment complex configurations +6. **Security**: Keep secrets in `.env` files, never commit them + +## Configuration Management + +### Environment Variables + +All services use variables from `.env`: +- `PUID`/`PGID`: User/group IDs for file permissions +- `TZ`: Timezone for all services +- `SERVER_IP`: Your server's IP address +- Service-specific credentials and paths + +### Config Files + +Service configurations are stored in `config/service-name/`: +``` +config/ +├── plex/ # Plex configuration +├── sonarr/ # Sonarr configuration +├── grafana/ # Grafana dashboards +└── ... +``` + +**Note**: Config directories are gitignored to prevent committing sensitive data. + +## Security Best Practices + +1. **Pin Image Versions**: Never use `:latest` in production +2. **Use Environment Variables**: Store secrets in `.env` (gitignored) +3. **Run as Non-Root**: Set PUID/PGID to match your user +4. **Limit Exposure**: Bind ports to localhost when possible +5. **Regular Updates**: Keep images updated via Watchtower +6. **Scan Images**: Use `docker scan` to check for vulnerabilities + +## Troubleshooting + +### Service Won't Start + +1. Check logs: `docker compose -f file.yml logs service-name` +2. Validate config: `docker compose -f file.yml config` +3. Check port conflicts: `sudo netstat -tlnp | grep PORT` +4. Verify network exists: `docker network ls` + +### Permission Issues + +1. Check PUID/PGID match your user: `id -u` and `id -g` +2. Fix ownership: `sudo chown -R 1000:1000 ./config/service-name` + +### Network Issues + +1. Verify network exists: `docker network inspect homelab-network` +2. Test connectivity: `docker compose exec service1 ping service2` + +### Getting Help + +- Review the [Docker Guidelines](docs/docker-guidelines.md) +- Ask GitHub Copilot in VS Code +- Check service-specific documentation +- Review Docker logs for error messages + +## Backup Strategy + +### What to Backup + +1. **Docker Compose files** (version controlled in git) +2. **Config directories**: `./config/*` +3. **Named volumes**: `docker volume ls` +4. **Environment file**: `.env` (securely, not in git) + +### Backup Named Volumes + +```bash +# Backup a volume +docker run --rm \ + -v volume-name:/data \ + -v $(pwd)/backups:/backup \ + busybox tar czf /backup/volume-backup.tar.gz /data +``` + +### Restore Named Volumes + +```bash +# Restore a volume +docker run --rm \ + -v volume-name:/data \ + -v $(pwd)/backups:/backup \ + busybox tar xzf /backup/volume-backup.tar.gz -C / +``` + +## Contributing + +Contributions are welcome! Please: + +1. Fork the repository +2. Create a feature branch +3. Follow existing patterns and conventions +4. Test your changes +5. Submit a pull request + +## License + +This project is provided as-is for personal homelab use. + +## Acknowledgments + +- Docker and Docker Compose communities +- LinuxServer.io for excellent container images +- GitHub Copilot for AI assistance capabilities +- All the open-source projects used in example compose files + +## Getting Started Checklist + +- [ ] Install Docker and Docker Compose +- [ ] Clone this repository +- [ ] Copy `.env.example` to `.env` and configure +- [ ] Create `homelab-network`: `docker network create homelab-network` +- [ ] Start infrastructure services: `docker compose -f docker-compose/infrastructure.yml up -d` +- [ ] Access Portainer at `http://server-ip:9000` +- [ ] Install VS Code with GitHub Copilot extension +- [ ] Open repository in VS Code and start using AI assistance +- [ ] Add more services as needed using AI guidance + +## Support + +For issues, questions, or suggestions: +- Open an issue on GitHub +- Consult the comprehensive [documentation](docs/docker-guidelines.md) +- Use GitHub Copilot in VS Code for real-time assistance diff --git a/docker-compose/README.md b/docker-compose/README.md new file mode 100644 index 0000000..b21bf61 --- /dev/null +++ b/docker-compose/README.md @@ -0,0 +1,228 @@ +# Docker Compose Stacks + +This directory contains Docker Compose files for managing your homelab services. Each file is organized by functional area to maintain clarity and organization. + +## Structure + +``` +docker-compose/ +├── infrastructure.yml # Core services (reverse proxy, DNS, etc.) +├── media.yml # Media server services (Plex, Jellyfin, etc.) +├── monitoring.yml # Observability stack (Prometheus, Grafana, etc.) +├── development.yml # Development tools and services +└── README.md # This file +``` + +## Usage + +### Starting Services + +Start all services in a compose file: +```bash +docker compose -f docker-compose/infrastructure.yml up -d +``` + +Start a specific service: +```bash +docker compose -f docker-compose/media.yml up -d plex +``` + +Start multiple compose files together: +```bash +docker compose -f docker-compose/infrastructure.yml -f docker-compose/media.yml up -d +``` + +### Stopping Services + +Stop all services in a compose file: +```bash +docker compose -f docker-compose/infrastructure.yml down +``` + +Stop a specific service: +```bash +docker compose -f docker-compose/media.yml stop plex +``` + +### Viewing Status + +Check running services: +```bash +docker compose -f docker-compose/media.yml ps +``` + +View logs: +```bash +docker compose -f docker-compose/media.yml logs -f plex +``` + +### Updating Services + +Pull latest images: +```bash +docker compose -f docker-compose/media.yml pull +``` + +Update a specific service: +```bash +docker compose -f docker-compose/media.yml pull plex +docker compose -f docker-compose/media.yml up -d plex +``` + +## Networks + +All services connect to a shared bridge network called `homelab-network`. Create it once: + +```bash +docker network create homelab-network +``` + +Some services may use additional networks for security isolation: +- `monitoring-network` - For monitoring stack +- `database-network` - For database isolation +- `media-network` - For media services + +Create them as needed: +```bash +docker network create monitoring-network +docker network create database-network +docker network create media-network +``` + +## Environment Variables + +Create a `.env` file in the root of your homelab directory with common variables: + +```bash +# .env +PUID=1000 +PGID=1000 +TZ=America/New_York +USERDIR=/home/username/homelab +DATADIR=/mnt/data +``` + +Never commit `.env` files to git! Use `.env.example` as a template instead. + +## Best Practices + +1. **Pin Versions**: Always specify image versions (e.g., `nginx:1.25.3` not `nginx:latest`) +2. **Use Labels**: Add labels for organization and documentation +3. **Health Checks**: Define health checks for critical services +4. **Resource Limits**: Set memory and CPU limits for resource-intensive services +5. **Logging**: Configure log rotation to prevent disk space issues +6. **Restart Policies**: Use `unless-stopped` for most services +7. **Comments**: Document non-obvious configurations + +## Template + +When creating a new service, use this template: + +```yaml +services: + service-name: + image: vendor/image:version + container_name: service-name + restart: unless-stopped + networks: + - homelab-network + ports: + - "host_port:container_port" + volumes: + - ./config/service-name:/config + - service-data:/data + environment: + - PUID=${PUID} + - PGID=${PGID} + - TZ=${TZ} + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:port/health"] + interval: 30s + timeout: 10s + retries: 3 + labels: + - "homelab.category=category" + - "homelab.description=Service description" + +volumes: + service-data: + driver: local + +networks: + homelab-network: + external: true +``` + +## Troubleshooting + +### Service won't start +1. Check logs: `docker compose -f file.yml logs service-name` +2. Validate config: `docker compose -f file.yml config` +3. Check for port conflicts: `sudo netstat -tlnp | grep PORT` +4. Verify volumes exist and have correct permissions + +### Permission errors +1. Ensure PUID and PGID match your user: `id -u` and `id -g` +2. Fix directory ownership: `sudo chown -R 1000:1000 ./config/service-name` + +### Network issues +1. Verify network exists: `docker network ls` +2. Check service is connected: `docker network inspect homelab-network` +3. Test connectivity: `docker compose exec service1 ping service2` + +## Migration from Docker Run + +If you have services running via `docker run`, migrate them to compose: + +1. Get current configuration: + ```bash + docker inspect container-name > container-config.json + ``` + +2. Convert to compose format (extract image, ports, volumes, environment) + +3. Test the compose configuration + +4. Stop old container: + ```bash + docker stop container-name + docker rm container-name + ``` + +5. Start with compose: + ```bash + docker compose -f file.yml up -d + ``` + +## Backup Strategy + +Regular backups are essential: + +```bash +# Backup compose files (already in git) +git add docker-compose/*.yml +git commit -m "Update compose configurations" + +# Backup volumes +docker run --rm \ + -v volume-name:/data \ + -v $(pwd)/backups:/backup \ + busybox tar czf /backup/volume-name-$(date +%Y%m%d).tar.gz /data + +# Backup config directories +tar czf backups/config-$(date +%Y%m%d).tar.gz config/ +``` + +## Getting Help + +- Check the [Docker Guidelines](../docs/docker-guidelines.md) for detailed documentation +- Review the [GitHub Copilot Instructions](../.github/copilot-instructions.md) for AI assistance +- Consult service-specific documentation in `config/service-name/README.md` + +## Examples + +See the example compose files in this directory: +- `infrastructure.yml` - Essential services like reverse proxy +- `media.yml` - Media server stack +- `monitoring.yml` - Observability and monitoring +- `development.yml` - Development environments and tools diff --git a/docker-compose/development.yml b/docker-compose/development.yml new file mode 100644 index 0000000..75a5110 --- /dev/null +++ b/docker-compose/development.yml @@ -0,0 +1,187 @@ +# Development Services +# Tools and services for development work + +services: + # Code Server - VS Code in the browser + # Access at: http://server-ip:8443 + code-server: + image: lscr.io/linuxserver/code-server:4.20.0 + container_name: code-server + restart: unless-stopped + networks: + - homelab-network + ports: + - "8443:8443" + volumes: + - ./config/code-server:/config + - ${PROJECTDIR:-/home/user/projects}:/projects + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ:-America/New_York} + - PASSWORD=${CODE_SERVER_PASSWORD:-changeme} + - SUDO_PASSWORD=${CODE_SERVER_SUDO_PASSWORD:-changeme} + labels: + - "homelab.category=development" + - "homelab.description=VS Code in browser for remote development" + + # GitLab CE - Self-hosted Git repository manager + # Access at: http://server-ip:8929 + # Note: Requires significant resources (4GB+ RAM recommended) + gitlab: + image: gitlab/gitlab-ce:16.7.0-ce.0 + container_name: gitlab + restart: unless-stopped + networks: + - homelab-network + ports: + - "8929:80" # Web UI + - "2222:22" # SSH + volumes: + - ./config/gitlab/config:/etc/gitlab + - gitlab-logs:/var/log/gitlab + - gitlab-data:/var/opt/gitlab + environment: + GITLAB_OMNIBUS_CONFIG: | + external_url 'http://${SERVER_IP}:8929' + gitlab_rails['gitlab_shell_ssh_port'] = 2222 + gitlab_rails['time_zone'] = '${TZ:-America/New_York}' + shm_size: '256m' + deploy: + resources: + limits: + memory: 4G + reservations: + memory: 2G + labels: + - "homelab.category=development" + - "homelab.description=Self-hosted Git repository manager" + + # PostgreSQL - Database for development + # Access at: localhost:5432 from other containers + postgres: + image: postgres:16.1-alpine + container_name: postgres-dev + restart: unless-stopped + networks: + - database-network + - homelab-network + ports: + - "5432:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + - ./config/postgres/init:/docker-entrypoint-initdb.d + environment: + - POSTGRES_USER=${POSTGRES_USER:-postgres} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-changeme} + - POSTGRES_DB=${POSTGRES_DB:-homelab} + - PGDATA=/var/lib/postgresql/data/pgdata + labels: + - "homelab.category=development" + - "homelab.description=PostgreSQL database for development" + + # Redis - In-memory data store + # Access at: localhost:6379 from other containers + redis: + image: redis:7.2.3-alpine + container_name: redis-dev + restart: unless-stopped + networks: + - database-network + - homelab-network + ports: + - "6379:6379" + volumes: + - redis-data:/data + - ./config/redis/redis.conf:/usr/local/etc/redis/redis.conf + command: redis-server /usr/local/etc/redis/redis.conf --appendonly yes + labels: + - "homelab.category=development" + - "homelab.description=Redis in-memory data store" + + # pgAdmin - PostgreSQL management UI + # Access at: http://server-ip:5050 + pgadmin: + image: dpage/pgadmin4:8.2 + container_name: pgadmin + restart: unless-stopped + networks: + - database-network + - homelab-network + ports: + - "5050:80" + volumes: + - pgadmin-data:/var/lib/pgadmin + environment: + - PGADMIN_DEFAULT_EMAIL=${PGADMIN_EMAIL:-admin@example.com} + - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_PASSWORD:-changeme} + - PGADMIN_CONFIG_SERVER_MODE=False + depends_on: + - postgres + labels: + - "homelab.category=development" + - "homelab.description=PostgreSQL administration UI" + + # Jupyter Lab - Interactive computing notebooks + # Access at: http://server-ip:8888 + # Token displayed in logs on first start + jupyter: + image: jupyter/scipy-notebook:2023-12-25 + container_name: jupyter + restart: unless-stopped + networks: + - homelab-network + ports: + - "8888:8888" + volumes: + - ./config/jupyter:/home/jovyan/work + environment: + - JUPYTER_ENABLE_LAB=yes + - GRANT_SUDO=yes + user: root + command: start-notebook.sh --NotebookApp.token='${JUPYTER_TOKEN:-changeme}' + # Uncomment for GPU support (NVIDIA) + # runtime: nvidia + # environment: + # - NVIDIA_VISIBLE_DEVICES=all + labels: + - "homelab.category=development" + - "homelab.description=Jupyter Lab for data science and ML" + + # Node-RED - Visual programming for automation + # Access at: http://server-ip:1880 + nodered: + image: nodered/node-red:3.1.3 + container_name: nodered + restart: unless-stopped + networks: + - homelab-network + ports: + - "1880:1880" + volumes: + - nodered-data:/data + environment: + - TZ=${TZ:-America/New_York} + labels: + - "homelab.category=development" + - "homelab.description=Visual automation and workflow tool" + +volumes: + gitlab-logs: + driver: local + gitlab-data: + driver: local + postgres-data: + driver: local + redis-data: + driver: local + pgadmin-data: + driver: local + nodered-data: + driver: local + +networks: + database-network: + driver: bridge + homelab-network: + external: true diff --git a/docker-compose/infrastructure.yml b/docker-compose/infrastructure.yml new file mode 100644 index 0000000..6c7c617 --- /dev/null +++ b/docker-compose/infrastructure.yml @@ -0,0 +1,107 @@ +# Infrastructure Services +# Core services that other services depend on + +services: + # Nginx Proxy Manager - Web-based reverse proxy management + # Access at: http://server-ip:81 + # Default credentials: admin@example.com / changeme + nginx-proxy-manager: + image: jc21/nginx-proxy-manager:2.10.4 + container_name: nginx-proxy-manager + restart: unless-stopped + networks: + - homelab-network + ports: + - "80:80" # HTTP + - "443:443" # HTTPS + - "81:81" # Admin UI + volumes: + - ./config/nginx-proxy-manager/data:/data + - ./config/nginx-proxy-manager/letsencrypt:/etc/letsencrypt + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:81"] + interval: 30s + timeout: 10s + retries: 3 + labels: + - "homelab.category=infrastructure" + - "homelab.description=Reverse proxy with Let's Encrypt support" + + # Pi-hole - Network-wide ad blocker and DNS server + # Access at: http://server-ip:8080/admin + pihole: + image: pihole/pihole:2024.01.0 + container_name: pihole + restart: unless-stopped + networks: + - homelab-network + ports: + - "53:53/tcp" # DNS TCP + - "53:53/udp" # DNS UDP + - "8080:80/tcp" # Web interface + volumes: + - ./config/pihole/etc-pihole:/etc/pihole + - ./config/pihole/etc-dnsmasq.d:/etc/dnsmasq.d + environment: + - TZ=${TZ:-America/New_York} + - WEBPASSWORD=${PIHOLE_PASSWORD:-changeme} + - FTLCONF_LOCAL_IPV4=${SERVER_IP} + dns: + - 127.0.0.1 + - 1.1.1.1 + cap_add: + - NET_ADMIN + labels: + - "homelab.category=infrastructure" + - "homelab.description=Network-wide ad blocking and DNS" + + # Portainer - Docker management UI + # Access at: http://server-ip:9000 + portainer: + image: portainer/portainer-ce:2.19.4 + container_name: portainer + restart: unless-stopped + networks: + - homelab-network + ports: + - "9000:9000" + - "9443:9443" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - portainer-data:/data + security_opt: + - no-new-privileges:true + labels: + - "homelab.category=infrastructure" + - "homelab.description=Docker container management UI" + + # Watchtower - Automatic container updates + # Runs silently in background, no UI + watchtower: + image: containrrr/watchtower:1.7.1 + container_name: watchtower + restart: unless-stopped + networks: + - homelab-network + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + - WATCHTOWER_CLEANUP=true + - WATCHTOWER_INCLUDE_RESTARTING=true + - WATCHTOWER_SCHEDULE=0 0 4 * * * # 4 AM daily + - WATCHTOWER_NOTIFICATIONS=shoutrrr + - WATCHTOWER_NOTIFICATION_URL=${WATCHTOWER_NOTIFICATION_URL} + labels: + - "homelab.category=infrastructure" + - "homelab.description=Automatic Docker container updates" + +volumes: + portainer-data: + driver: local + +networks: + homelab-network: + external: true diff --git a/docker-compose/media.yml b/docker-compose/media.yml new file mode 100644 index 0000000..22f83d7 --- /dev/null +++ b/docker-compose/media.yml @@ -0,0 +1,171 @@ +# Media Services +# Services for media management and streaming + +services: + # Plex Media Server - Media streaming platform + # Access at: http://server-ip:32400/web + plex: + image: plexinc/pms-docker:1.40.0.7998-f68041501 + container_name: plex + restart: unless-stopped + networks: + - media-network + - homelab-network + ports: + - "32400:32400" # Plex web UI + volumes: + - ./config/plex:/config + - ${MEDIADIR:-/media}:/media:ro + - plex-transcode:/transcode + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ:-America/New_York} + - PLEX_CLAIM=${PLEX_CLAIM} + - ADVERTISE_IP=http://${SERVER_IP}:32400/ + devices: + # Uncomment for hardware transcoding (Intel QuickSync) + # - /dev/dri:/dev/dri + # Uncomment for NVIDIA GPU transcoding + # - /dev/nvidia0:/dev/nvidia0 + # - /dev/nvidiactl:/dev/nvidiactl + # - /dev/nvidia-modeset:/dev/nvidia-modeset + # - /dev/nvidia-uvm:/dev/nvidia-uvm + # - /dev/nvidia-uvm-tools:/dev/nvidia-uvm-tools + # Uncomment for NVIDIA GPU support + # runtime: nvidia + # environment: + # - NVIDIA_VISIBLE_DEVICES=all + # - NVIDIA_DRIVER_CAPABILITIES=compute,video,utility + labels: + - "homelab.category=media" + - "homelab.description=Plex media streaming server" + + # Jellyfin - Free alternative to Plex + # Access at: http://server-ip:8096 + jellyfin: + image: jellyfin/jellyfin:10.8.13 + container_name: jellyfin + restart: unless-stopped + networks: + - media-network + - homelab-network + ports: + - "8096:8096" + volumes: + - ./config/jellyfin:/config + - ./config/jellyfin/cache:/cache + - ${MEDIADIR:-/media}:/media:ro + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ:-America/New_York} + devices: + # Uncomment for hardware transcoding + # - /dev/dri:/dev/dri + labels: + - "homelab.category=media" + - "homelab.description=Open-source media streaming server" + + # Sonarr - TV show automation + # Access at: http://server-ip:8989 + sonarr: + image: lscr.io/linuxserver/sonarr:4.0.0 + container_name: sonarr + restart: unless-stopped + networks: + - media-network + - homelab-network + ports: + - "8989:8989" + volumes: + - ./config/sonarr:/config + - ${MEDIADIR:-/media}:/media + - ${DOWNLOADDIR:-/downloads}:/downloads + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ:-America/New_York} + labels: + - "homelab.category=media" + - "homelab.description=TV show management and automation" + + # Radarr - Movie automation + # Access at: http://server-ip:7878 + radarr: + image: lscr.io/linuxserver/radarr:5.2.6 + container_name: radarr + restart: unless-stopped + networks: + - media-network + - homelab-network + ports: + - "7878:7878" + volumes: + - ./config/radarr:/config + - ${MEDIADIR:-/media}:/media + - ${DOWNLOADDIR:-/downloads}:/downloads + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ:-America/New_York} + labels: + - "homelab.category=media" + - "homelab.description=Movie management and automation" + + # Prowlarr - Indexer manager + # Access at: http://server-ip:9696 + prowlarr: + image: lscr.io/linuxserver/prowlarr:1.11.4 + container_name: prowlarr + restart: unless-stopped + networks: + - media-network + - homelab-network + ports: + - "9696:9696" + volumes: + - ./config/prowlarr:/config + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ:-America/New_York} + labels: + - "homelab.category=media" + - "homelab.description=Indexer manager for Sonarr/Radarr" + + # qBittorrent - Torrent client + # Access at: http://server-ip:8081 + # Default credentials: admin / adminadmin + qbittorrent: + image: lscr.io/linuxserver/qbittorrent:4.6.2 + container_name: qbittorrent + restart: unless-stopped + networks: + - media-network + - homelab-network + ports: + - "8081:8081" # Web UI + - "6881:6881" # Torrent port TCP + - "6881:6881/udp" # Torrent port UDP + volumes: + - ./config/qbittorrent:/config + - ${DOWNLOADDIR:-/downloads}:/downloads + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ:-America/New_York} + - WEBUI_PORT=8081 + labels: + - "homelab.category=media" + - "homelab.description=Torrent download client" + +volumes: + plex-transcode: + driver: local + +networks: + media-network: + driver: bridge + homelab-network: + external: true diff --git a/docker-compose/monitoring.yml b/docker-compose/monitoring.yml new file mode 100644 index 0000000..a4d67a3 --- /dev/null +++ b/docker-compose/monitoring.yml @@ -0,0 +1,173 @@ +# Monitoring and Observability Services +# Services for monitoring your homelab infrastructure + +services: + # Prometheus - Metrics collection and storage + # Access at: http://server-ip:9090 + prometheus: + image: prom/prometheus:v2.48.1 + container_name: prometheus + restart: unless-stopped + networks: + - monitoring-network + - homelab-network + ports: + - "9090:9090" + volumes: + - ./config/prometheus:/etc/prometheus + - prometheus-data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=30d' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--web.enable-lifecycle' + user: "${PUID:-1000}:${PGID:-1000}" + labels: + - "homelab.category=monitoring" + - "homelab.description=Metrics collection and time-series database" + + # Grafana - Metrics visualization + # Access at: http://server-ip:3000 + # Default credentials: admin / admin (change on first login) + grafana: + image: grafana/grafana:10.2.3 + container_name: grafana + restart: unless-stopped + networks: + - monitoring-network + - homelab-network + ports: + - "3000:3000" + volumes: + - grafana-data:/var/lib/grafana + - ./config/grafana/provisioning:/etc/grafana/provisioning + environment: + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin} + - GF_USERS_ALLOW_SIGN_UP=false + - GF_SERVER_ROOT_URL=http://${SERVER_IP}:3000 + - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource,grafana-piechart-panel + user: "${PUID:-1000}:${PGID:-1000}" + depends_on: + - prometheus + labels: + - "homelab.category=monitoring" + - "homelab.description=Metrics visualization and dashboards" + + # Node Exporter - Host metrics exporter + # Metrics at: http://server-ip:9100/metrics + node-exporter: + image: prom/node-exporter:v1.7.0 + container_name: node-exporter + restart: unless-stopped + networks: + - monitoring-network + ports: + - "9100:9100" + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.rootfs=/rootfs' + - '--path.sysfs=/host/sys' + - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' + labels: + - "homelab.category=monitoring" + - "homelab.description=Hardware and OS metrics exporter" + + # cAdvisor - Container metrics exporter + # Access at: http://server-ip:8082 + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.47.2 + container_name: cadvisor + restart: unless-stopped + networks: + - monitoring-network + ports: + - "8082:8080" + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker:/var/lib/docker:ro + - /dev/disk:/dev/disk:ro + privileged: true + devices: + - /dev/kmsg + labels: + - "homelab.category=monitoring" + - "homelab.description=Container metrics and performance monitoring" + + # Uptime Kuma - Uptime monitoring + # Access at: http://server-ip:3001 + uptime-kuma: + image: louislam/uptime-kuma:1.23.11 + container_name: uptime-kuma + restart: unless-stopped + networks: + - monitoring-network + - homelab-network + ports: + - "3001:3001" + volumes: + - uptime-kuma-data:/app/data + labels: + - "homelab.category=monitoring" + - "homelab.description=Service uptime monitoring and alerts" + + # Loki - Log aggregation + # Access at: http://server-ip:3100 + loki: + image: grafana/loki:2.9.3 + container_name: loki + restart: unless-stopped + networks: + - monitoring-network + ports: + - "3100:3100" + volumes: + - ./config/loki:/etc/loki + - loki-data:/loki + command: -config.file=/etc/loki/loki-config.yml + user: "${PUID:-1000}:${PGID:-1000}" + labels: + - "homelab.category=monitoring" + - "homelab.description=Log aggregation system" + + # Promtail - Log shipper for Loki + # Ships Docker container logs to Loki + promtail: + image: grafana/promtail:2.9.3 + container_name: promtail + restart: unless-stopped + networks: + - monitoring-network + volumes: + - ./config/promtail:/etc/promtail + - /var/log:/var/log:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + command: -config.file=/etc/promtail/promtail-config.yml + depends_on: + - loki + labels: + - "homelab.category=monitoring" + - "homelab.description=Log collector for Loki" + +volumes: + prometheus-data: + driver: local + grafana-data: + driver: local + uptime-kuma-data: + driver: local + loki-data: + driver: local + +networks: + monitoring-network: + driver: bridge + homelab-network: + external: true diff --git a/docs/docker-guidelines.md b/docs/docker-guidelines.md new file mode 100644 index 0000000..2d7e681 --- /dev/null +++ b/docs/docker-guidelines.md @@ -0,0 +1,858 @@ +# Docker Service Management Guidelines + +## Overview + +This document provides comprehensive guidelines for managing Docker services in your AI-powered homelab. These guidelines ensure consistency, maintainability, and reliability across your entire infrastructure. + +## Table of Contents + +1. [Philosophy](#philosophy) +2. [Docker Compose vs Docker Run](#docker-compose-vs-docker-run) +3. [Service Creation Guidelines](#service-creation-guidelines) +4. [Service Modification Guidelines](#service-modification-guidelines) +5. [Naming Conventions](#naming-conventions) +6. [Network Architecture](#network-architecture) +7. [Volume Management](#volume-management) +8. [Security Best Practices](#security-best-practices) +9. [Monitoring and Logging](#monitoring-and-logging) +10. [Troubleshooting](#troubleshooting) + +## Philosophy + +### Core Principles + +1. **Infrastructure as Code**: All services must be defined in Docker Compose files +2. **Reproducibility**: Any service should be rebuildable from compose files +3. **Documentation**: Every non-obvious configuration must be commented +4. **Consistency**: Use the same patterns across all services +5. **Safety First**: Always test changes in isolation before deploying + +### The Stack Mindset + +Think of your homelab as an interconnected stack where: +- Services depend on networks +- Networks connect services +- Volumes persist data +- Changes ripple through the system + +Always ask: "How does this change affect other services?" + +## Docker Compose vs Docker Run + +### Docker Compose: For Everything Persistent + +Use Docker Compose for: +- All production services +- Services that need to restart automatically +- Multi-container applications +- Services with complex configurations +- Anything you want to keep long-term + +**Example:** +```yaml +# docker-compose/plex.yml +services: + plex: + image: plexinc/pms-docker:1.40.0.7775-456fbaf97 + container_name: plex + restart: unless-stopped + networks: + - media-network + ports: + - "32400:32400" + volumes: + - ./config/plex:/config + - /media:/media:ro + environment: + - PUID=1000 + - PGID=1000 + - TZ=America/New_York +``` + +### Docker Run: For Temporary Operations Only + +Use `docker run` for: +- Testing new images +- One-off commands +- Debugging +- Verification tasks (like GPU testing) + +**Examples:** +```bash +# Test if NVIDIA GPU is accessible +docker run --rm --gpus all nvidia/cuda:12.0.0-base-ubuntu22.04 nvidia-smi + +# Quick test of a new image +docker run --rm -it alpine:latest /bin/sh + +# One-off database backup +docker run --rm -v mydata:/data busybox tar czf /backup/data.tar.gz /data +``` + +## Service Creation Guidelines + +### Step-by-Step Process + +#### 1. Planning Phase + +**Before writing any YAML:** + +- [ ] What problem does this service solve? +- [ ] Does a similar service already exist? +- [ ] What are the dependencies? +- [ ] What ports are needed? +- [ ] What data needs to persist? +- [ ] What environment variables are required? +- [ ] What networks should it connect to? +- [ ] Are there any security considerations? + +#### 2. Research Phase + +- Read the official image documentation +- Check example configurations +- Review resource requirements +- Understand health check requirements +- Note any special permissions needed + +#### 3. Implementation Phase + +**Start with a minimal configuration:** + +```yaml +services: + service-name: + image: vendor/image:specific-version + container_name: service-name + restart: unless-stopped +``` + +**Add networks:** +```yaml + networks: + - homelab-network +``` + +**Add ports (if externally accessible):** +```yaml + ports: + - "8080:8080" # Web UI +``` + +**Add volumes:** +```yaml + volumes: + - ./config/service-name:/config + - service-data:/data +``` + +**Add environment variables:** +```yaml + environment: + - PUID=1000 + - PGID=1000 + - TZ=${TIMEZONE} +``` + +**Add health checks (if applicable):** +```yaml + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s +``` + +#### 4. Testing Phase + +```bash +# Validate syntax +docker compose -f docker-compose/service.yml config + +# Start in foreground to see logs +docker compose -f docker-compose/service.yml up + +# If successful, restart in background +docker compose -f docker-compose/service.yml down +docker compose -f docker-compose/service.yml up -d +``` + +#### 5. Documentation Phase + +Add comments to your compose file: +```yaml +services: + sonarr: + image: lscr.io/linuxserver/sonarr:4.0.0 + container_name: sonarr + # Sonarr - TV Show management and automation + # Web UI: http://server-ip:8989 + # Connects to: Prowlarr (indexers), qBittorrent (downloads) + restart: unless-stopped +``` + +Update your main README or service-specific README with: +- Service purpose +- Access URLs +- Default credentials (if any) +- Configuration notes +- Backup instructions + +## Service Modification Guidelines + +### Before Modifying + +1. **Back up current configuration:** + ```bash + cp docker-compose/service.yml docker-compose/service.yml.backup + ``` + +2. **Document why you're making the change** + - Create a comment in the compose file + - Note in your changelog or docs + +3. **Understand the current state:** + ```bash + # Check if service is running + docker compose -f docker-compose/service.yml ps + + # Review current configuration + docker compose -f docker-compose/service.yml config + + # Check logs for any existing issues + docker compose -f docker-compose/service.yml logs --tail=50 + ``` + +### Making the Change + +1. **Edit the compose file** + - Make minimal, targeted changes + - Keep existing structure when possible + - Add comments for new configurations + +2. **Validate syntax:** + ```bash + docker compose -f docker-compose/service.yml config + ``` + +3. **Apply the change:** + ```bash + # Pull new image if version changed + docker compose -f docker-compose/service.yml pull + + # Recreate the service + docker compose -f docker-compose/service.yml up -d + ``` + +4. **Verify the change:** + ```bash + # Check service is running + docker compose -f docker-compose/service.yml ps + + # Watch logs for errors + docker compose -f docker-compose/service.yml logs -f + + # Test functionality + curl http://localhost:port/health + ``` + +### Rollback Plan + +If something goes wrong: +```bash +# Stop the service +docker compose -f docker-compose/service.yml down + +# Restore backup +mv docker-compose/service.yml.backup docker-compose/service.yml + +# Restart with old configuration +docker compose -f docker-compose/service.yml up -d +``` + +## Naming Conventions + +### Service Names + +Use lowercase with hyphens: +- ✅ `plex-media-server` +- ✅ `home-assistant` +- ❌ `PlexMediaServer` +- ❌ `home_assistant` + +### Container Names + +Match service names or be descriptive: +```yaml +services: + plex: + container_name: plex # Simple match + + database: + container_name: media-database # Descriptive +``` + +### Network Names + +Use purpose-based naming: +- `homelab-network` - Main network +- `media-network` - Media services +- `monitoring-network` - Observability stack +- `isolated-network` - Untrusted services + +### Volume Names + +Use `service-purpose` pattern: +```yaml +volumes: + plex-config: + plex-metadata: + database-data: + nginx-certs: +``` + +### File Names + +Organize by function: +- `docker-compose/media.yml` - Media services (Plex, Jellyfin, etc.) +- `docker-compose/monitoring.yml` - Monitoring stack +- `docker-compose/infrastructure.yml` - Core services (DNS, reverse proxy) +- `docker-compose/development.yml` - Dev tools + +## Network Architecture + +### Network Types + +1. **Bridge Networks** (Most Common) + ```yaml + networks: + homelab-network: + driver: bridge + ipam: + config: + - subnet: 172.20.0.0/16 + ``` + +2. **Host Network** (When Performance Critical) + ```yaml + services: + performance-critical: + network_mode: host + ``` + +3. **Overlay Networks** (For Swarm/Multi-host) + ```yaml + networks: + swarm-network: + driver: overlay + ``` + +### Network Design Patterns + +#### Pattern 1: Single Shared Network +Simplest approach for small homelabs: +```yaml +networks: + homelab-network: + external: true +``` + +Create once manually: +```bash +docker network create homelab-network +``` + +#### Pattern 2: Segmented Networks +Better security through isolation: +```yaml +networks: + frontend-network: # Web-facing services + backend-network: # Databases, internal services + monitoring-network: # Observability +``` + +#### Pattern 3: Service-Specific Networks +Each service group has its own network: +```yaml +services: + web: + networks: + - frontend + - backend + + database: + networks: + - backend # Not exposed to frontend +``` + +### Network Security + +- Place databases on internal networks only +- Use separate networks for untrusted services +- Expose minimal ports to the host +- Use reverse proxies for web services + +## Volume Management + +### Volume Types + +#### Named Volumes (Managed by Docker) +```yaml +volumes: + database-data: + driver: local +``` + +**Use for:** +- Database files +- Application data +- Anything Docker should manage + +**Advantages:** +- Docker handles permissions +- Easy to backup/restore +- Portable across systems + +#### Bind Mounts (Direct Host Paths) +```yaml +volumes: + - ./config/app:/config + - /media:/media:ro +``` + +**Use for:** +- Configuration files you edit directly +- Large media libraries +- Shared data with host + +**Advantages:** +- Direct file access +- Easy to edit +- Can share with host applications + +#### tmpfs Mounts (RAM) +```yaml +tmpfs: + - /tmp +``` + +**Use for:** +- Temporary data +- Cache that doesn't need persistence +- Sensitive data that shouldn't touch disk + +### Volume Best Practices + +1. **Consistent Paths:** + ```yaml + volumes: + - ./config/service:/config # Always use /config inside container + - service-data:/data # Always use /data for application data + ``` + +2. **Read-Only When Possible:** + ```yaml + volumes: + - /media:/media:ro # Media library is read-only + ``` + +3. **Separate Config from Data:** + ```yaml + volumes: + - ./config/plex:/config # Editable configuration + - plex-metadata:/metadata # Application-managed data + ``` + +4. **Backup Strategy:** + ```bash + # Backup named volume + docker run --rm \ + -v plex-metadata:/data \ + -v $(pwd)/backups:/backup \ + busybox tar czf /backup/plex-metadata.tar.gz /data + ``` + +## Security Best Practices + +### 1. Image Security + +**Pin Specific Versions:** +```yaml +# ✅ Good - Specific version +image: nginx:1.25.3-alpine + +# ❌ Bad - Latest tag +image: nginx:latest +``` + +**Use Official or Trusted Images:** +- Official Docker images +- LinuxServer.io (lscr.io) +- Trusted vendors + +**Scan Images:** +```bash +docker scan vendor/image:tag +``` + +### 2. Secret Management + +**Never Commit Secrets:** +```yaml +# .env file (gitignored) +DB_PASSWORD=super-secret-password +API_KEY=sk-1234567890 + +# docker-compose.yml +environment: + - DB_PASSWORD=${DB_PASSWORD} + - API_KEY=${API_KEY} +``` + +**Provide Templates:** +```bash +# .env.example (committed) +DB_PASSWORD=changeme +API_KEY=your-api-key-here +``` + +### 3. User Permissions + +**Run as Non-Root:** +```yaml +environment: + - PUID=1000 # Your user ID + - PGID=1000 # Your group ID +``` + +**Check Current User:** +```bash +id -u # Gets your UID +id -g # Gets your GID +``` + +### 4. Network Security + +**Minimal Exposure:** +```yaml +# ✅ Good - Only expose what's needed +ports: + - "127.0.0.1:8080:8080" # Only accessible from localhost + +# ❌ Bad - Exposed to all interfaces +ports: + - "8080:8080" +``` + +**Use Reverse Proxy:** +```yaml +# Don't expose services directly +# Use Nginx/Traefik to proxy with SSL +``` + +### 5. Resource Limits + +**Prevent Resource Exhaustion:** +```yaml +deploy: + resources: + limits: + cpus: '2' + memory: 4G + reservations: + cpus: '0.5' + memory: 1G +``` + +## Monitoring and Logging + +### Logging Configuration + +**Standard Logging:** +```yaml +logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" +``` + +**Centralized Logging:** +```yaml +logging: + driver: "syslog" + options: + syslog-address: "tcp://192.168.1.100:514" +``` + +### Health Checks + +**HTTP Health Check:** +```yaml +healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 30s + timeout: 10s + retries: 3 +``` + +**TCP Health Check:** +```yaml +healthcheck: + test: ["CMD-SHELL", "nc -z localhost 5432 || exit 1"] + interval: 30s + timeout: 5s + retries: 3 +``` + +**Custom Script:** +```yaml +healthcheck: + test: ["CMD", "/healthcheck.sh"] + interval: 30s + timeout: 10s + retries: 3 +``` + +### Monitoring Stack Example + +```yaml +# docker-compose/monitoring.yml +services: + prometheus: + image: prom/prometheus:v2.48.0 + container_name: prometheus + restart: unless-stopped + volumes: + - ./config/prometheus:/etc/prometheus + - prometheus-data:/prometheus + ports: + - "9090:9090" + networks: + - monitoring-network + + grafana: + image: grafana/grafana:10.2.2 + container_name: grafana + restart: unless-stopped + volumes: + - grafana-data:/var/lib/grafana + ports: + - "3000:3000" + networks: + - monitoring-network + depends_on: + - prometheus + +volumes: + prometheus-data: + grafana-data: + +networks: + monitoring-network: + driver: bridge +``` + +## Troubleshooting + +### Common Issues + +#### Service Won't Start + +**1. Check logs:** +```bash +docker compose -f docker-compose/service.yml logs +``` + +**2. Validate configuration:** +```bash +docker compose -f docker-compose/service.yml config +``` + +**3. Check for port conflicts:** +```bash +# See what's using a port +sudo netstat -tlnp | grep :8080 +``` + +**4. Verify image exists:** +```bash +docker images | grep service-name +``` + +#### Permission Errors + +**1. Check PUID/PGID:** +```bash +# Your user ID +id -u + +# Your group ID +id -g +``` + +**2. Fix directory permissions:** +```bash +sudo chown -R 1000:1000 ./config/service-name +``` + +**3. Check volume permissions:** +```bash +docker compose -f docker-compose/service.yml exec service-name ls -la /config +``` + +#### Network Connectivity Issues + +**1. Verify network exists:** +```bash +docker network ls +docker network inspect homelab-network +``` + +**2. Check if services are on same network:** +```bash +docker network inspect homelab-network | grep Name +``` + +**3. Test connectivity:** +```bash +docker compose -f docker-compose/service.yml exec service1 ping service2 +``` + +#### Container Keeps Restarting + +**1. Watch logs:** +```bash +docker compose -f docker-compose/service.yml logs -f +``` + +**2. Check health status:** +```bash +docker compose -f docker-compose/service.yml ps +``` + +**3. Inspect container:** +```bash +docker inspect container-name +``` + +### Debugging Commands + +```bash +# Enter running container +docker compose -f docker-compose/service.yml exec service-name /bin/sh + +# View full container configuration +docker inspect container-name + +# See resource usage +docker stats container-name + +# View recent events +docker events --since 10m + +# Check disk space +docker system df +``` + +### Recovery Procedures + +#### Service Corrupted + +```bash +# Stop service +docker compose -f docker-compose/service.yml down + +# Remove container and volumes (backup first!) +docker compose -f docker-compose/service.yml down -v + +# Recreate from scratch +docker compose -f docker-compose/service.yml up -d +``` + +#### Network Issues + +```bash +# Remove and recreate network +docker network rm homelab-network +docker network create homelab-network + +# Restart services +docker compose -f docker-compose/*.yml up -d +``` + +#### Full System Reset (Nuclear Option) + +```bash +# ⚠️ WARNING: This removes everything! +# Backup first! + +# Stop all containers +docker stop $(docker ps -aq) + +# Remove all containers +docker rm $(docker ps -aq) + +# Remove all volumes (careful!) +docker volume rm $(docker volume ls -q) + +# Remove all networks (except defaults) +docker network prune -f + +# Rebuild from compose files +docker compose -f docker-compose/*.yml up -d +``` + +## Maintenance + +### Regular Tasks + +**Weekly:** +- Review logs for errors +- Check disk space: `docker system df` +- Update security patches on images + +**Monthly:** +- Update images to latest versions +- Review and prune unused resources +- Backup volumes +- Review and optimize compose files + +**Quarterly:** +- Full stack review +- Documentation update +- Performance optimization +- Security audit + +### Update Procedure + +```bash +# 1. Backup current state +docker compose -f docker-compose/service.yml config > backup/service-config.yml + +# 2. Update image version in compose file +# Edit docker-compose/service.yml + +# 3. Pull new image +docker compose -f docker-compose/service.yml pull + +# 4. Recreate service +docker compose -f docker-compose/service.yml up -d + +# 5. Verify +docker compose -f docker-compose/service.yml logs -f + +# 6. Test functionality +# Access service and verify it works +``` + +## Conclusion + +Following these guidelines ensures: +- Consistent infrastructure +- Easy troubleshooting +- Reproducible deployments +- Maintainable system +- Better security + +Remember: **Infrastructure as Code** means treating your Docker Compose files as critical documentation. Keep them clean, commented, and version-controlled. diff --git a/docs/quick-reference.md b/docs/quick-reference.md new file mode 100644 index 0000000..a77b1b1 --- /dev/null +++ b/docs/quick-reference.md @@ -0,0 +1,394 @@ +# Quick Reference Guide + +## Common Commands + +### Service Management + +```bash +# Start all services in a file +docker compose -f docker-compose/file.yml up -d + +# Start specific service +docker compose -f docker-compose/file.yml up -d service-name + +# Stop all services +docker compose -f docker-compose/file.yml down + +# Stop specific service +docker compose -f docker-compose/file.yml stop service-name + +# Restart service +docker compose -f docker-compose/file.yml restart service-name + +# Remove service and volumes +docker compose -f docker-compose/file.yml down -v +``` + +### Monitoring + +```bash +# View logs +docker compose -f docker-compose/file.yml logs -f service-name + +# Check service status +docker compose -f docker-compose/file.yml ps + +# View resource usage +docker stats + +# Inspect service +docker inspect container-name +``` + +### Updates + +```bash +# Pull latest images +docker compose -f docker-compose/file.yml pull + +# Pull and update specific service +docker compose -f docker-compose/file.yml pull service-name +docker compose -f docker-compose/file.yml up -d service-name +``` + +### Network Management + +```bash +# List networks +docker network ls + +# Inspect network +docker network inspect homelab-network + +# Create network +docker network create network-name + +# Remove network +docker network rm network-name +``` + +### Volume Management + +```bash +# List volumes +docker volume ls + +# Inspect volume +docker volume inspect volume-name + +# Remove volume +docker volume rm volume-name + +# Backup volume +docker run --rm -v volume-name:/data -v $(pwd)/backups:/backup \ + busybox tar czf /backup/backup.tar.gz /data + +# Restore volume +docker run --rm -v volume-name:/data -v $(pwd)/backups:/backup \ + busybox tar xzf /backup/backup.tar.gz -C / +``` + +### System Maintenance + +```bash +# View disk usage +docker system df + +# Clean up unused resources +docker system prune + +# Clean up everything (careful!) +docker system prune -a --volumes + +# Remove unused images +docker image prune + +# Remove unused volumes +docker volume prune + +# Remove unused networks +docker network prune +``` + +## Port Reference + +### Infrastructure Services +- **80**: Nginx Proxy Manager (HTTP) +- **443**: Nginx Proxy Manager (HTTPS) +- **81**: Nginx Proxy Manager (Admin) +- **53**: Pi-hole (DNS) +- **8080**: Pi-hole (Web UI) +- **9000**: Portainer +- **9443**: Portainer (HTTPS) + +### Media Services +- **32400**: Plex +- **8096**: Jellyfin +- **8989**: Sonarr +- **7878**: Radarr +- **9696**: Prowlarr +- **8081**: qBittorrent + +### Monitoring Services +- **9090**: Prometheus +- **3000**: Grafana +- **9100**: Node Exporter +- **8082**: cAdvisor +- **3001**: Uptime Kuma +- **3100**: Loki + +### Development Services +- **8443**: Code Server +- **8929**: GitLab +- **2222**: GitLab SSH +- **5432**: PostgreSQL +- **6379**: Redis +- **5050**: pgAdmin +- **8888**: Jupyter Lab +- **1880**: Node-RED + +## Environment Variables Quick Reference + +```bash +# User/Group +PUID=1000 # Your user ID (get with: id -u) +PGID=1000 # Your group ID (get with: id -g) + +# General +TZ=America/New_York # Your timezone +SERVER_IP=192.168.1.100 # Server IP address + +# Paths +USERDIR=/home/username/homelab +MEDIADIR=/mnt/media +DOWNLOADDIR=/mnt/downloads +PROJECTDIR=/home/username/projects +``` + +## Network Setup + +```bash +# Create all networks at once +docker network create homelab-network +docker network create media-network +docker network create monitoring-network +docker network create database-network +``` + +## Service URLs + +After starting services, access them at: + +``` +Infrastructure: +http://SERVER_IP:81 - Nginx Proxy Manager +http://SERVER_IP:8080 - Pi-hole +http://SERVER_IP:9000 - Portainer + +Media: +http://SERVER_IP:32400/web - Plex +http://SERVER_IP:8096 - Jellyfin +http://SERVER_IP:8989 - Sonarr +http://SERVER_IP:7878 - Radarr +http://SERVER_IP:9696 - Prowlarr +http://SERVER_IP:8081 - qBittorrent + +Monitoring: +http://SERVER_IP:9090 - Prometheus +http://SERVER_IP:3000 - Grafana +http://SERVER_IP:3001 - Uptime Kuma + +Development: +http://SERVER_IP:8443 - Code Server +http://SERVER_IP:8929 - GitLab +http://SERVER_IP:5050 - pgAdmin +http://SERVER_IP:8888 - Jupyter Lab +http://SERVER_IP:1880 - Node-RED +``` + +## Troubleshooting Quick Fixes + +### Service won't start +```bash +# 1. Check logs +docker compose -f docker-compose/file.yml logs service-name + +# 2. Validate configuration +docker compose -f docker-compose/file.yml config + +# 3. Check what's using the port +sudo netstat -tlnp | grep PORT_NUMBER +``` + +### Permission errors +```bash +# Check your IDs +id -u # Should match PUID in .env +id -g # Should match PGID in .env + +# Fix ownership +sudo chown -R 1000:1000 ./config/service-name +``` + +### Network issues +```bash +# Check network exists +docker network inspect homelab-network + +# Recreate network +docker network rm homelab-network +docker network create homelab-network +docker compose -f docker-compose/file.yml up -d +``` + +### Container keeps restarting +```bash +# Watch logs in real-time +docker compose -f docker-compose/file.yml logs -f service-name + +# Check resource usage +docker stats container-name + +# Inspect container +docker inspect container-name | less +``` + +## Testing GPU Support (NVIDIA) + +```bash +# Test if nvidia-container-toolkit works +docker run --rm --gpus all nvidia/cuda:12.0.0-base-ubuntu22.04 nvidia-smi + +# If successful, you should see your GPU info +``` + +## Backup Commands + +```bash +# Backup all config directories +tar czf backup-config-$(date +%Y%m%d).tar.gz config/ + +# Backup a specific volume +docker run --rm \ + -v volume-name:/data \ + -v $(pwd)/backups:/backup \ + busybox tar czf /backup/volume-name-$(date +%Y%m%d).tar.gz /data + +# Backup .env file (store securely!) +cp .env .env.backup +``` + +## Health Checks + +```bash +# Check all container health status +docker ps --format "table {{.Names}}\t{{.Status}}" + +# Check specific service health +docker inspect --format='{{json .State.Health}}' container-name | jq +``` + +## Resource Limits + +Add to service definition if needed: + +```yaml +deploy: + resources: + limits: + cpus: '2' + memory: 4G + reservations: + cpus: '0.5' + memory: 1G +``` + +## Common Patterns + +### Add a new service +1. Choose the appropriate compose file +2. Add service definition following existing patterns +3. Use environment variables from .env +4. Connect to homelab-network +5. Pin specific image version +6. Add labels for organization +7. Test: `docker compose -f file.yml config` +8. Deploy: `docker compose -f file.yml up -d service-name` + +### Update a service version +1. Edit compose file with new version +2. Pull new image: `docker compose -f file.yml pull service-name` +3. Recreate: `docker compose -f file.yml up -d service-name` +4. Check logs: `docker compose -f file.yml logs -f service-name` + +### Remove a service +1. Stop service: `docker compose -f file.yml stop service-name` +2. Remove service: `docker compose -f file.yml rm service-name` +3. Remove from compose file +4. Optional: Remove volumes: `docker volume rm volume-name` +5. Optional: Remove config: `rm -rf config/service-name` + +## AI Assistant Usage in VS Code + +### Ask for help: +- "Add Jellyfin to my media stack" +- "Configure GPU for Plex" +- "Create monitoring dashboard setup" +- "Help me troubleshoot port conflicts" +- "Generate a compose file for Home Assistant" + +### The AI will: +- Check existing services +- Follow naming conventions +- Avoid port conflicts +- Use proper network configuration +- Include health checks +- Add documentation comments +- Suggest related services + +## Quick Deployment + +### Minimal setup +```bash +# 1. Clone and configure +git clone https://github.com/kelinfoxy/AI-Homelab.git +cd AI-Homelab +cp .env.example .env +nano .env # Edit values + +# 2. Create network +docker network create homelab-network + +# 3. Start Portainer (for container management) +docker compose -f docker-compose/infrastructure.yml up -d portainer + +# 4. Access at http://SERVER_IP:9000 +``` + +### Full stack deployment +```bash +# After minimal setup, deploy everything: +docker compose -f docker-compose/infrastructure.yml up -d +docker compose -f docker-compose/media.yml up -d +docker compose -f docker-compose/monitoring.yml up -d +docker compose -f docker-compose/development.yml up -d +``` + +## Maintenance Schedule + +### Daily (automated) +- Watchtower checks for updates at 4 AM + +### Weekly +- Review logs: `docker compose -f docker-compose/*.yml logs --tail=100` +- Check disk space: `docker system df` + +### Monthly +- Update pinned versions in compose files +- Backup volumes and configs +- Review security updates + +### Quarterly +- Full system audit +- Documentation review +- Performance optimization