From 0c78b3f03851d4a31d33b9e4e44810c608b9b0af Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 22:44:55 +0000 Subject: [PATCH 01/16] Initial plan From 1b1464e278b59d0765f0c7c19b07748068d7a955 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 22:50:55 +0000 Subject: [PATCH 02/16] Create AI chat agent for VS Code with Docker service management Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- .env.example | 48 ++ .github/copilot-instructions.md | 253 +++++++++ .gitignore | 76 +++ README.md | 379 ++++++++++++- docker-compose/README.md | 228 ++++++++ docker-compose/development.yml | 187 +++++++ docker-compose/infrastructure.yml | 107 ++++ docker-compose/media.yml | 171 ++++++ docker-compose/monitoring.yml | 173 ++++++ docs/docker-guidelines.md | 858 ++++++++++++++++++++++++++++++ docs/quick-reference.md | 394 ++++++++++++++ 11 files changed, 2873 insertions(+), 1 deletion(-) create mode 100644 .env.example create mode 100644 .github/copilot-instructions.md create mode 100644 .gitignore create mode 100644 docker-compose/README.md create mode 100644 docker-compose/development.yml create mode 100644 docker-compose/infrastructure.yml create mode 100644 docker-compose/media.yml create mode 100644 docker-compose/monitoring.yml create mode 100644 docs/docker-guidelines.md create mode 100644 docs/quick-reference.md diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..ee0642c --- /dev/null +++ b/.env.example @@ -0,0 +1,48 @@ +# Environment Variables Template +# Copy this file to .env and fill in your values +# NEVER commit .env to git! + +# User and Group IDs (get with: id -u and id -g) +PUID=1000 +PGID=1000 + +# Timezone (list: timedatectl list-timezones) +TZ=America/New_York + +# Server IP address +SERVER_IP=192.168.1.100 + +# Directory Paths +USERDIR=/home/username/homelab +MEDIADIR=/mnt/media +DOWNLOADDIR=/mnt/downloads +PROJECTDIR=/home/username/projects + +# Plex Configuration +PLEX_CLAIM=claim-xxxxxxxxxx + +# Monitoring Passwords +GRAFANA_ADMIN_PASSWORD=changeme + +# Code Server Passwords +CODE_SERVER_PASSWORD=changeme +CODE_SERVER_SUDO_PASSWORD=changeme + +# Database Credentials +POSTGRES_USER=postgres +POSTGRES_PASSWORD=changeme +POSTGRES_DB=homelab + +PGADMIN_EMAIL=admin@example.com +PGADMIN_PASSWORD=changeme + +# Jupyter Token +JUPYTER_TOKEN=changeme + +# Pi-hole +PIHOLE_PASSWORD=changeme + +# Watchtower Notifications (optional) +# WATCHTOWER_NOTIFICATION_URL= + +# Add your own variables below diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..1186bc5 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,253 @@ +# AI Homelab Management Assistant + +You are an AI assistant specialized in managing Docker-based homelab infrastructure. Your role is to help users create, modify, and manage Docker services while maintaining consistency across the entire server stack. + +## Core Principles + +### 1. Docker Compose First +- **ALWAYS** use Docker Compose stacks for persistent services +- Only use `docker run` for temporary containers (e.g., testing nvidia-container-toolkit functionality) +- Maintain all services in organized docker-compose.yml files + +### 2. Consistency is Key +- Keep consistent naming conventions across all compose files +- Use the same network naming patterns +- Maintain uniform volume mount structures +- Apply consistent environment variable patterns + +### 3. Stack-Aware Changes +- Before making changes, consider the impact on the entire server stack +- Check for service dependencies (networks, volumes, other services) +- Ensure changes don't break existing integrations +- Validate that port assignments don't conflict + +## Creating a New Docker Service + +When creating a new service, follow these steps: + +1. **Assess the Stack** + - Review existing services and their configurations + - Check for available ports + - Identify shared networks and volumes + - Note any dependent services + +2. **Choose the Right Location** + - Place related services in the same compose file + - Use separate compose files for different functional areas (e.g., monitoring, media, development) + - Keep the file structure organized by category + +3. **Service Definition Template** + ```yaml + services: + service-name: + image: image:tag # Always pin versions for stability + container_name: service-name # Use descriptive, consistent names + restart: unless-stopped # Standard restart policy + networks: + - homelab-network # Use shared networks + ports: + - "host_port:container_port" # Document port purpose + volumes: + - ./config/service-name:/config # Config in local directory + - service-data:/data # Named volumes for persistent data + environment: + - PUID=1000 # Standard user/group IDs + - PGID=1000 + - TZ=America/New_York # Consistent timezone + labels: + - "homelab.category=category-name" # For organization + - "homelab.description=Service description" + + volumes: + service-data: + driver: local + + networks: + homelab-network: + external: true # Or define once in main compose + ``` + +4. **Configuration Best Practices** + - Pin image versions (avoid `:latest` in production) + - Use environment variables for configuration + - Store sensitive data in `.env` files (never commit these!) + - Use named volumes for data that should persist + - Bind mount config directories for easy access + +5. **Documentation** + - Add comments explaining non-obvious configurations + - Document port mappings and their purposes + - Note any special requirements or dependencies + +## Editing an Existing Service + +When modifying a service: + +1. **Review Current Configuration** + - Read the entire service definition + - Check for dependencies (links, depends_on, networks) + - Note any volumes or data that might be affected + +2. **Plan the Change** + - Identify what needs to change + - Consider backward compatibility + - Plan for data migration if needed + +3. **Make Minimal Changes** + - Change only what's necessary + - Maintain existing patterns and conventions + - Keep the same structure unless there's a good reason to change it + +4. **Validate the Change** + - Check YAML syntax + - Verify port availability + - Ensure network connectivity + - Test the service starts correctly + +5. **Update Documentation** + - Update comments if behavior changes + - Revise README files if user interaction changes + +## Common Operations + +### Testing a New Image +```bash +# Use docker run for quick tests, then convert to compose +docker run --rm -it \ + --name test-container \ + image:tag \ + command +``` + +### Checking NVIDIA GPU Access +```bash +# Temporary test container for GPU +docker run --rm --gpus all nvidia/cuda:12.0.0-base-ubuntu22.04 nvidia-smi +``` + +### Deploying a Stack +```bash +# Start all services in a compose file +docker compose -f docker-compose.yml up -d + +# Start specific services +docker compose -f docker-compose.yml up -d service-name +``` + +### Updating a Service +```bash +# Pull latest image (if version updated) +docker compose -f docker-compose.yml pull service-name + +# Recreate the service +docker compose -f docker-compose.yml up -d service-name +``` + +### Checking Logs +```bash +# View logs for a service +docker compose -f docker-compose.yml logs -f service-name +``` + +## Network Management + +### Standard Network Setup +- Use a shared bridge network for inter-service communication +- Name it consistently (e.g., `homelab-network`) +- Define it once in a main compose file or create it manually + +### Network Isolation +- Use separate networks for different security zones +- Keep databases on internal networks only +- Expose only necessary services to external networks + +## Volume Management + +### Volume Strategy +- **Named volumes**: For data that should persist but doesn't need direct access +- **Bind mounts**: For configs you want to edit directly +- **tmpfs**: For temporary data that should not persist + +### Backup Considerations +- Keep important data in well-defined volumes +- Document backup procedures for each service +- Use consistent paths for easier backup automation + +## Environment Variables + +### Standard Variables +```yaml +environment: + - PUID=1000 # User ID for file permissions + - PGID=1000 # Group ID for file permissions + - TZ=America/New_York # Timezone + - UMASK=022 # File creation mask +``` + +### Sensitive Data +- Store secrets in `.env` files +- Reference them in compose: `${VARIABLE_NAME}` +- Never commit `.env` files to git +- Provide `.env.example` templates + +## Troubleshooting + +### Service Won't Start +1. Check logs: `docker compose logs service-name` +2. Verify configuration syntax +3. Check for port conflicts +4. Verify volume mounts exist +5. Check network connectivity + +### Permission Issues +1. Verify PUID/PGID match host user +2. Check directory permissions +3. Verify volume ownership + +### Network Issues +1. Verify network exists: `docker network ls` +2. Check if services are on same network +3. Use service names for DNS resolution +4. Check firewall rules + +## File Organization + +``` +/home/user/homelab/ +├── docker-compose/ +│ ├── media.yml # Media server services +│ ├── monitoring.yml # Monitoring stack +│ ├── development.yml # Dev tools +│ └── infrastructure.yml # Core services +├── config/ +│ ├── service1/ +│ ├── service2/ +│ └── ... +├── data/ # Bind mount data +│ └── ... +├── .env # Global secrets (gitignored) +└── README.md # Stack documentation +``` + +## Safety Checks + +Before deploying any changes: +- [ ] YAML syntax is valid +- [ ] Ports don't conflict with existing services +- [ ] Networks exist or are defined +- [ ] Volume paths are correct +- [ ] Environment variables are set +- [ ] No secrets in compose files +- [ ] Service dependencies are met +- [ ] Backup of current configuration exists + +## Remember + +- **Think before you act**: Consider the entire stack +- **Be consistent**: Follow established patterns +- **Document everything**: Future you will thank you +- **Test safely**: Use temporary containers first +- **Back up first**: Always have a rollback plan +- **Security matters**: Keep secrets secret, update regularly + +When a user asks you to create or modify a Docker service, follow these guidelines carefully, ask clarifying questions if needed, and always prioritize the stability and consistency of the entire homelab infrastructure. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7afc29a --- /dev/null +++ b/.gitignore @@ -0,0 +1,76 @@ +# Environment variables and secrets +.env +*.env +!.env.example + +# Config directories with sensitive data +config/*/secrets/ +config/*/*.key +config/*/*.pem +config/*/*.crt +config/*/db/ + +# Backup files +*.backup +*.bak +backups/ + +# OS files +.DS_Store +Thumbs.db +*.swp +*.swo +*~ + +# Editor files +.vscode/ +.idea/ +*.sublime-* + +# Logs +*.log +logs/ + +# Temporary files +tmp/ +temp/ +*.tmp + +# Docker volumes (if locally mounted) +volumes/ + +# Documentation builds +docs/_build/ +docs/.doctrees/ + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +venv/ +env/ + +# Node +node_modules/ +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Database files +*.sqlite +*.db + +# Certificates and keys +*.pem +*.key +*.crt +*.cer +*.p12 +*.pfx + +# Monitoring data (if stored locally) +prometheus-data/ +grafana-data/ +loki-data/ diff --git a/README.md b/README.md index cc6fde9..46852ea 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,379 @@ # AI-Homelab -AI Powered Homelab administration + +AI-Powered Homelab Administration with GitHub Copilot + +## Overview + +This repository provides a structured approach to managing a homelab infrastructure using Docker Compose, with integrated AI assistance through GitHub Copilot. The AI assistant is specifically trained to help you create, modify, and manage Docker services while maintaining consistency across your entire server stack. + +## Features + +- **AI-Powered Management**: GitHub Copilot integration with specialized instructions for Docker service management +- **Docker Compose First**: All persistent services defined in organized compose files +- **Consistent Structure**: Standardized naming conventions and patterns across all services +- **Stack-Aware Changes**: AI considers the entire infrastructure when making changes +- **Comprehensive Documentation**: Detailed guidelines and examples for all operations +- **Example Services**: Ready-to-use compose files for common homelab services + +## Quick Start + +### Prerequisites + +- Docker Engine 24.0+ installed +- Docker Compose V2 +- Git +- VS Code with GitHub Copilot extension (for AI assistance) + +### Initial Setup + +1. **Clone the repository:** + ```bash + git clone https://github.com/kelinfoxy/AI-Homelab.git + cd AI-Homelab + ``` + +2. **Create environment file:** + ```bash + cp .env.example .env + # Edit .env with your values + nano .env + ``` + +3. **Create the main network:** + ```bash + docker network create homelab-network + ``` + +4. **Create config directories:** + ```bash + mkdir -p config + ``` + +5. **Start your first service:** + ```bash + # Example: Start Portainer for container management + docker compose -f docker-compose/infrastructure.yml up -d portainer + ``` + +6. **Access the service:** + Open `http://your-server-ip:9000` in your browser + +## Repository Structure + +``` +AI-Homelab/ +├── .github/ +│ └── copilot-instructions.md # AI assistant guidelines +├── docker-compose/ +│ ├── infrastructure.yml # Core services (proxy, DNS, Portainer) +│ ├── media.yml # Media services (Plex, Sonarr, Radarr) +│ ├── monitoring.yml # Observability (Prometheus, Grafana) +│ ├── development.yml # Dev tools (code-server, databases) +│ └── README.md # Docker Compose documentation +├── docs/ +│ └── docker-guidelines.md # Comprehensive Docker guidelines +├── config/ # Service configurations (gitignored) +├── .env.example # Environment variable template +├── .gitignore # Git ignore patterns +└── README.md # This file +``` + +## Using the AI Assistant + +### In VS Code + +1. **Install GitHub Copilot** extension in VS Code +2. **Open this repository** in VS Code +3. **Start Copilot Chat** and ask questions like: + - "Help me add a new media service to my homelab" + - "Create a docker-compose file for Home Assistant" + - "How do I configure GPU support for Plex?" + - "Check my compose file for port conflicts" + +The AI assistant automatically follows the guidelines in `.github/copilot-instructions.md` to: +- Maintain consistency with existing services +- Use Docker Compose for all persistent services +- Consider the entire stack when making changes +- Follow naming conventions and best practices + +### Example Interactions + +**Creating a new service:** +``` +You: "I want to add Home Assistant to my homelab" + +Copilot: [Analyzes existing stack, checks for conflicts, creates compose configuration] +- Checks available ports +- Uses consistent naming +- Connects to appropriate networks +- Follows established patterns +``` + +**Modifying a service:** +``` +You: "Add GPU support to my Plex container" + +Copilot: [Reviews current Plex configuration and updates it] +- Checks if NVIDIA runtime is available +- Updates device mappings +- Adds required environment variables +- Maintains existing configuration +``` + +## Available Service Stacks + +### Infrastructure (`infrastructure.yml`) +- **Nginx Proxy Manager**: Web-based reverse proxy with SSL +- **Pi-hole**: Network-wide ad blocking and DNS +- **Portainer**: Docker container management UI +- **Watchtower**: Automatic container updates + +### Media (`media.yml`) +- **Plex**: Media streaming server +- **Jellyfin**: Open-source media server alternative +- **Sonarr**: TV show automation +- **Radarr**: Movie automation +- **Prowlarr**: Indexer manager +- **qBittorrent**: Torrent client + +### Monitoring (`monitoring.yml`) +- **Prometheus**: Metrics collection +- **Grafana**: Metrics visualization +- **Node Exporter**: Host metrics +- **cAdvisor**: Container metrics +- **Uptime Kuma**: Service uptime monitoring +- **Loki**: Log aggregation +- **Promtail**: Log shipping + +### Development (`development.yml`) +- **Code Server**: VS Code in browser +- **GitLab CE**: Self-hosted Git repository +- **PostgreSQL**: SQL database +- **Redis**: In-memory data store +- **pgAdmin**: PostgreSQL UI +- **Jupyter Lab**: Interactive notebooks +- **Node-RED**: Visual automation + +## Common Operations + +### Starting Services + +Start all services in a compose file: +```bash +docker compose -f docker-compose/infrastructure.yml up -d +``` + +Start specific services: +```bash +docker compose -f docker-compose/media.yml up -d plex sonarr radarr +``` + +### Stopping Services + +Stop all services: +```bash +docker compose -f docker-compose/infrastructure.yml down +``` + +Stop specific service: +```bash +docker compose -f docker-compose/media.yml stop plex +``` + +### Viewing Logs + +Follow logs for a service: +```bash +docker compose -f docker-compose/media.yml logs -f plex +``` + +View last 100 lines: +```bash +docker compose -f docker-compose/media.yml logs --tail=100 plex +``` + +### Updating Services + +Pull latest images: +```bash +docker compose -f docker-compose/media.yml pull +``` + +Update specific service: +```bash +docker compose -f docker-compose/media.yml pull plex +docker compose -f docker-compose/media.yml up -d plex +``` + +### Testing with Docker Run + +Test NVIDIA GPU support: +```bash +docker run --rm --gpus all nvidia/cuda:12.0.0-base-ubuntu22.04 nvidia-smi +``` + +Test a new image: +```bash +docker run --rm -it alpine:latest /bin/sh +``` + +## Network Architecture + +All services connect to networks for inter-service communication: + +- **homelab-network**: Main network for all services +- **media-network**: Isolated network for media stack +- **monitoring-network**: Network for observability stack +- **database-network**: Isolated network for databases + +Create networks manually: +```bash +docker network create homelab-network +docker network create media-network +docker network create monitoring-network +docker network create database-network +``` + +## Documentation + +### Comprehensive Guides + +- **[Docker Guidelines](docs/docker-guidelines.md)**: Complete guide to Docker service management +- **[Docker Compose README](docker-compose/README.md)**: Compose-specific documentation +- **[Copilot Instructions](.github/copilot-instructions.md)**: AI assistant guidelines + +### Key Principles + +1. **Docker Compose First**: Always use compose for persistent services +2. **Docker Run for Testing**: Only use `docker run` for temporary containers +3. **Consistency**: Follow established patterns and naming conventions +4. **Stack Awareness**: Consider dependencies and interactions +5. **Documentation**: Comment complex configurations +6. **Security**: Keep secrets in `.env` files, never commit them + +## Configuration Management + +### Environment Variables + +All services use variables from `.env`: +- `PUID`/`PGID`: User/group IDs for file permissions +- `TZ`: Timezone for all services +- `SERVER_IP`: Your server's IP address +- Service-specific credentials and paths + +### Config Files + +Service configurations are stored in `config/service-name/`: +``` +config/ +├── plex/ # Plex configuration +├── sonarr/ # Sonarr configuration +├── grafana/ # Grafana dashboards +└── ... +``` + +**Note**: Config directories are gitignored to prevent committing sensitive data. + +## Security Best Practices + +1. **Pin Image Versions**: Never use `:latest` in production +2. **Use Environment Variables**: Store secrets in `.env` (gitignored) +3. **Run as Non-Root**: Set PUID/PGID to match your user +4. **Limit Exposure**: Bind ports to localhost when possible +5. **Regular Updates**: Keep images updated via Watchtower +6. **Scan Images**: Use `docker scan` to check for vulnerabilities + +## Troubleshooting + +### Service Won't Start + +1. Check logs: `docker compose -f file.yml logs service-name` +2. Validate config: `docker compose -f file.yml config` +3. Check port conflicts: `sudo netstat -tlnp | grep PORT` +4. Verify network exists: `docker network ls` + +### Permission Issues + +1. Check PUID/PGID match your user: `id -u` and `id -g` +2. Fix ownership: `sudo chown -R 1000:1000 ./config/service-name` + +### Network Issues + +1. Verify network exists: `docker network inspect homelab-network` +2. Test connectivity: `docker compose exec service1 ping service2` + +### Getting Help + +- Review the [Docker Guidelines](docs/docker-guidelines.md) +- Ask GitHub Copilot in VS Code +- Check service-specific documentation +- Review Docker logs for error messages + +## Backup Strategy + +### What to Backup + +1. **Docker Compose files** (version controlled in git) +2. **Config directories**: `./config/*` +3. **Named volumes**: `docker volume ls` +4. **Environment file**: `.env` (securely, not in git) + +### Backup Named Volumes + +```bash +# Backup a volume +docker run --rm \ + -v volume-name:/data \ + -v $(pwd)/backups:/backup \ + busybox tar czf /backup/volume-backup.tar.gz /data +``` + +### Restore Named Volumes + +```bash +# Restore a volume +docker run --rm \ + -v volume-name:/data \ + -v $(pwd)/backups:/backup \ + busybox tar xzf /backup/volume-backup.tar.gz -C / +``` + +## Contributing + +Contributions are welcome! Please: + +1. Fork the repository +2. Create a feature branch +3. Follow existing patterns and conventions +4. Test your changes +5. Submit a pull request + +## License + +This project is provided as-is for personal homelab use. + +## Acknowledgments + +- Docker and Docker Compose communities +- LinuxServer.io for excellent container images +- GitHub Copilot for AI assistance capabilities +- All the open-source projects used in example compose files + +## Getting Started Checklist + +- [ ] Install Docker and Docker Compose +- [ ] Clone this repository +- [ ] Copy `.env.example` to `.env` and configure +- [ ] Create `homelab-network`: `docker network create homelab-network` +- [ ] Start infrastructure services: `docker compose -f docker-compose/infrastructure.yml up -d` +- [ ] Access Portainer at `http://server-ip:9000` +- [ ] Install VS Code with GitHub Copilot extension +- [ ] Open repository in VS Code and start using AI assistance +- [ ] Add more services as needed using AI guidance + +## Support + +For issues, questions, or suggestions: +- Open an issue on GitHub +- Consult the comprehensive [documentation](docs/docker-guidelines.md) +- Use GitHub Copilot in VS Code for real-time assistance diff --git a/docker-compose/README.md b/docker-compose/README.md new file mode 100644 index 0000000..b21bf61 --- /dev/null +++ b/docker-compose/README.md @@ -0,0 +1,228 @@ +# Docker Compose Stacks + +This directory contains Docker Compose files for managing your homelab services. Each file is organized by functional area to maintain clarity and organization. + +## Structure + +``` +docker-compose/ +├── infrastructure.yml # Core services (reverse proxy, DNS, etc.) +├── media.yml # Media server services (Plex, Jellyfin, etc.) +├── monitoring.yml # Observability stack (Prometheus, Grafana, etc.) +├── development.yml # Development tools and services +└── README.md # This file +``` + +## Usage + +### Starting Services + +Start all services in a compose file: +```bash +docker compose -f docker-compose/infrastructure.yml up -d +``` + +Start a specific service: +```bash +docker compose -f docker-compose/media.yml up -d plex +``` + +Start multiple compose files together: +```bash +docker compose -f docker-compose/infrastructure.yml -f docker-compose/media.yml up -d +``` + +### Stopping Services + +Stop all services in a compose file: +```bash +docker compose -f docker-compose/infrastructure.yml down +``` + +Stop a specific service: +```bash +docker compose -f docker-compose/media.yml stop plex +``` + +### Viewing Status + +Check running services: +```bash +docker compose -f docker-compose/media.yml ps +``` + +View logs: +```bash +docker compose -f docker-compose/media.yml logs -f plex +``` + +### Updating Services + +Pull latest images: +```bash +docker compose -f docker-compose/media.yml pull +``` + +Update a specific service: +```bash +docker compose -f docker-compose/media.yml pull plex +docker compose -f docker-compose/media.yml up -d plex +``` + +## Networks + +All services connect to a shared bridge network called `homelab-network`. Create it once: + +```bash +docker network create homelab-network +``` + +Some services may use additional networks for security isolation: +- `monitoring-network` - For monitoring stack +- `database-network` - For database isolation +- `media-network` - For media services + +Create them as needed: +```bash +docker network create monitoring-network +docker network create database-network +docker network create media-network +``` + +## Environment Variables + +Create a `.env` file in the root of your homelab directory with common variables: + +```bash +# .env +PUID=1000 +PGID=1000 +TZ=America/New_York +USERDIR=/home/username/homelab +DATADIR=/mnt/data +``` + +Never commit `.env` files to git! Use `.env.example` as a template instead. + +## Best Practices + +1. **Pin Versions**: Always specify image versions (e.g., `nginx:1.25.3` not `nginx:latest`) +2. **Use Labels**: Add labels for organization and documentation +3. **Health Checks**: Define health checks for critical services +4. **Resource Limits**: Set memory and CPU limits for resource-intensive services +5. **Logging**: Configure log rotation to prevent disk space issues +6. **Restart Policies**: Use `unless-stopped` for most services +7. **Comments**: Document non-obvious configurations + +## Template + +When creating a new service, use this template: + +```yaml +services: + service-name: + image: vendor/image:version + container_name: service-name + restart: unless-stopped + networks: + - homelab-network + ports: + - "host_port:container_port" + volumes: + - ./config/service-name:/config + - service-data:/data + environment: + - PUID=${PUID} + - PGID=${PGID} + - TZ=${TZ} + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:port/health"] + interval: 30s + timeout: 10s + retries: 3 + labels: + - "homelab.category=category" + - "homelab.description=Service description" + +volumes: + service-data: + driver: local + +networks: + homelab-network: + external: true +``` + +## Troubleshooting + +### Service won't start +1. Check logs: `docker compose -f file.yml logs service-name` +2. Validate config: `docker compose -f file.yml config` +3. Check for port conflicts: `sudo netstat -tlnp | grep PORT` +4. Verify volumes exist and have correct permissions + +### Permission errors +1. Ensure PUID and PGID match your user: `id -u` and `id -g` +2. Fix directory ownership: `sudo chown -R 1000:1000 ./config/service-name` + +### Network issues +1. Verify network exists: `docker network ls` +2. Check service is connected: `docker network inspect homelab-network` +3. Test connectivity: `docker compose exec service1 ping service2` + +## Migration from Docker Run + +If you have services running via `docker run`, migrate them to compose: + +1. Get current configuration: + ```bash + docker inspect container-name > container-config.json + ``` + +2. Convert to compose format (extract image, ports, volumes, environment) + +3. Test the compose configuration + +4. Stop old container: + ```bash + docker stop container-name + docker rm container-name + ``` + +5. Start with compose: + ```bash + docker compose -f file.yml up -d + ``` + +## Backup Strategy + +Regular backups are essential: + +```bash +# Backup compose files (already in git) +git add docker-compose/*.yml +git commit -m "Update compose configurations" + +# Backup volumes +docker run --rm \ + -v volume-name:/data \ + -v $(pwd)/backups:/backup \ + busybox tar czf /backup/volume-name-$(date +%Y%m%d).tar.gz /data + +# Backup config directories +tar czf backups/config-$(date +%Y%m%d).tar.gz config/ +``` + +## Getting Help + +- Check the [Docker Guidelines](../docs/docker-guidelines.md) for detailed documentation +- Review the [GitHub Copilot Instructions](../.github/copilot-instructions.md) for AI assistance +- Consult service-specific documentation in `config/service-name/README.md` + +## Examples + +See the example compose files in this directory: +- `infrastructure.yml` - Essential services like reverse proxy +- `media.yml` - Media server stack +- `monitoring.yml` - Observability and monitoring +- `development.yml` - Development environments and tools diff --git a/docker-compose/development.yml b/docker-compose/development.yml new file mode 100644 index 0000000..75a5110 --- /dev/null +++ b/docker-compose/development.yml @@ -0,0 +1,187 @@ +# Development Services +# Tools and services for development work + +services: + # Code Server - VS Code in the browser + # Access at: http://server-ip:8443 + code-server: + image: lscr.io/linuxserver/code-server:4.20.0 + container_name: code-server + restart: unless-stopped + networks: + - homelab-network + ports: + - "8443:8443" + volumes: + - ./config/code-server:/config + - ${PROJECTDIR:-/home/user/projects}:/projects + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ:-America/New_York} + - PASSWORD=${CODE_SERVER_PASSWORD:-changeme} + - SUDO_PASSWORD=${CODE_SERVER_SUDO_PASSWORD:-changeme} + labels: + - "homelab.category=development" + - "homelab.description=VS Code in browser for remote development" + + # GitLab CE - Self-hosted Git repository manager + # Access at: http://server-ip:8929 + # Note: Requires significant resources (4GB+ RAM recommended) + gitlab: + image: gitlab/gitlab-ce:16.7.0-ce.0 + container_name: gitlab + restart: unless-stopped + networks: + - homelab-network + ports: + - "8929:80" # Web UI + - "2222:22" # SSH + volumes: + - ./config/gitlab/config:/etc/gitlab + - gitlab-logs:/var/log/gitlab + - gitlab-data:/var/opt/gitlab + environment: + GITLAB_OMNIBUS_CONFIG: | + external_url 'http://${SERVER_IP}:8929' + gitlab_rails['gitlab_shell_ssh_port'] = 2222 + gitlab_rails['time_zone'] = '${TZ:-America/New_York}' + shm_size: '256m' + deploy: + resources: + limits: + memory: 4G + reservations: + memory: 2G + labels: + - "homelab.category=development" + - "homelab.description=Self-hosted Git repository manager" + + # PostgreSQL - Database for development + # Access at: localhost:5432 from other containers + postgres: + image: postgres:16.1-alpine + container_name: postgres-dev + restart: unless-stopped + networks: + - database-network + - homelab-network + ports: + - "5432:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + - ./config/postgres/init:/docker-entrypoint-initdb.d + environment: + - POSTGRES_USER=${POSTGRES_USER:-postgres} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-changeme} + - POSTGRES_DB=${POSTGRES_DB:-homelab} + - PGDATA=/var/lib/postgresql/data/pgdata + labels: + - "homelab.category=development" + - "homelab.description=PostgreSQL database for development" + + # Redis - In-memory data store + # Access at: localhost:6379 from other containers + redis: + image: redis:7.2.3-alpine + container_name: redis-dev + restart: unless-stopped + networks: + - database-network + - homelab-network + ports: + - "6379:6379" + volumes: + - redis-data:/data + - ./config/redis/redis.conf:/usr/local/etc/redis/redis.conf + command: redis-server /usr/local/etc/redis/redis.conf --appendonly yes + labels: + - "homelab.category=development" + - "homelab.description=Redis in-memory data store" + + # pgAdmin - PostgreSQL management UI + # Access at: http://server-ip:5050 + pgadmin: + image: dpage/pgadmin4:8.2 + container_name: pgadmin + restart: unless-stopped + networks: + - database-network + - homelab-network + ports: + - "5050:80" + volumes: + - pgadmin-data:/var/lib/pgadmin + environment: + - PGADMIN_DEFAULT_EMAIL=${PGADMIN_EMAIL:-admin@example.com} + - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_PASSWORD:-changeme} + - PGADMIN_CONFIG_SERVER_MODE=False + depends_on: + - postgres + labels: + - "homelab.category=development" + - "homelab.description=PostgreSQL administration UI" + + # Jupyter Lab - Interactive computing notebooks + # Access at: http://server-ip:8888 + # Token displayed in logs on first start + jupyter: + image: jupyter/scipy-notebook:2023-12-25 + container_name: jupyter + restart: unless-stopped + networks: + - homelab-network + ports: + - "8888:8888" + volumes: + - ./config/jupyter:/home/jovyan/work + environment: + - JUPYTER_ENABLE_LAB=yes + - GRANT_SUDO=yes + user: root + command: start-notebook.sh --NotebookApp.token='${JUPYTER_TOKEN:-changeme}' + # Uncomment for GPU support (NVIDIA) + # runtime: nvidia + # environment: + # - NVIDIA_VISIBLE_DEVICES=all + labels: + - "homelab.category=development" + - "homelab.description=Jupyter Lab for data science and ML" + + # Node-RED - Visual programming for automation + # Access at: http://server-ip:1880 + nodered: + image: nodered/node-red:3.1.3 + container_name: nodered + restart: unless-stopped + networks: + - homelab-network + ports: + - "1880:1880" + volumes: + - nodered-data:/data + environment: + - TZ=${TZ:-America/New_York} + labels: + - "homelab.category=development" + - "homelab.description=Visual automation and workflow tool" + +volumes: + gitlab-logs: + driver: local + gitlab-data: + driver: local + postgres-data: + driver: local + redis-data: + driver: local + pgadmin-data: + driver: local + nodered-data: + driver: local + +networks: + database-network: + driver: bridge + homelab-network: + external: true diff --git a/docker-compose/infrastructure.yml b/docker-compose/infrastructure.yml new file mode 100644 index 0000000..6c7c617 --- /dev/null +++ b/docker-compose/infrastructure.yml @@ -0,0 +1,107 @@ +# Infrastructure Services +# Core services that other services depend on + +services: + # Nginx Proxy Manager - Web-based reverse proxy management + # Access at: http://server-ip:81 + # Default credentials: admin@example.com / changeme + nginx-proxy-manager: + image: jc21/nginx-proxy-manager:2.10.4 + container_name: nginx-proxy-manager + restart: unless-stopped + networks: + - homelab-network + ports: + - "80:80" # HTTP + - "443:443" # HTTPS + - "81:81" # Admin UI + volumes: + - ./config/nginx-proxy-manager/data:/data + - ./config/nginx-proxy-manager/letsencrypt:/etc/letsencrypt + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:81"] + interval: 30s + timeout: 10s + retries: 3 + labels: + - "homelab.category=infrastructure" + - "homelab.description=Reverse proxy with Let's Encrypt support" + + # Pi-hole - Network-wide ad blocker and DNS server + # Access at: http://server-ip:8080/admin + pihole: + image: pihole/pihole:2024.01.0 + container_name: pihole + restart: unless-stopped + networks: + - homelab-network + ports: + - "53:53/tcp" # DNS TCP + - "53:53/udp" # DNS UDP + - "8080:80/tcp" # Web interface + volumes: + - ./config/pihole/etc-pihole:/etc/pihole + - ./config/pihole/etc-dnsmasq.d:/etc/dnsmasq.d + environment: + - TZ=${TZ:-America/New_York} + - WEBPASSWORD=${PIHOLE_PASSWORD:-changeme} + - FTLCONF_LOCAL_IPV4=${SERVER_IP} + dns: + - 127.0.0.1 + - 1.1.1.1 + cap_add: + - NET_ADMIN + labels: + - "homelab.category=infrastructure" + - "homelab.description=Network-wide ad blocking and DNS" + + # Portainer - Docker management UI + # Access at: http://server-ip:9000 + portainer: + image: portainer/portainer-ce:2.19.4 + container_name: portainer + restart: unless-stopped + networks: + - homelab-network + ports: + - "9000:9000" + - "9443:9443" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - portainer-data:/data + security_opt: + - no-new-privileges:true + labels: + - "homelab.category=infrastructure" + - "homelab.description=Docker container management UI" + + # Watchtower - Automatic container updates + # Runs silently in background, no UI + watchtower: + image: containrrr/watchtower:1.7.1 + container_name: watchtower + restart: unless-stopped + networks: + - homelab-network + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + - WATCHTOWER_CLEANUP=true + - WATCHTOWER_INCLUDE_RESTARTING=true + - WATCHTOWER_SCHEDULE=0 0 4 * * * # 4 AM daily + - WATCHTOWER_NOTIFICATIONS=shoutrrr + - WATCHTOWER_NOTIFICATION_URL=${WATCHTOWER_NOTIFICATION_URL} + labels: + - "homelab.category=infrastructure" + - "homelab.description=Automatic Docker container updates" + +volumes: + portainer-data: + driver: local + +networks: + homelab-network: + external: true diff --git a/docker-compose/media.yml b/docker-compose/media.yml new file mode 100644 index 0000000..22f83d7 --- /dev/null +++ b/docker-compose/media.yml @@ -0,0 +1,171 @@ +# Media Services +# Services for media management and streaming + +services: + # Plex Media Server - Media streaming platform + # Access at: http://server-ip:32400/web + plex: + image: plexinc/pms-docker:1.40.0.7998-f68041501 + container_name: plex + restart: unless-stopped + networks: + - media-network + - homelab-network + ports: + - "32400:32400" # Plex web UI + volumes: + - ./config/plex:/config + - ${MEDIADIR:-/media}:/media:ro + - plex-transcode:/transcode + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ:-America/New_York} + - PLEX_CLAIM=${PLEX_CLAIM} + - ADVERTISE_IP=http://${SERVER_IP}:32400/ + devices: + # Uncomment for hardware transcoding (Intel QuickSync) + # - /dev/dri:/dev/dri + # Uncomment for NVIDIA GPU transcoding + # - /dev/nvidia0:/dev/nvidia0 + # - /dev/nvidiactl:/dev/nvidiactl + # - /dev/nvidia-modeset:/dev/nvidia-modeset + # - /dev/nvidia-uvm:/dev/nvidia-uvm + # - /dev/nvidia-uvm-tools:/dev/nvidia-uvm-tools + # Uncomment for NVIDIA GPU support + # runtime: nvidia + # environment: + # - NVIDIA_VISIBLE_DEVICES=all + # - NVIDIA_DRIVER_CAPABILITIES=compute,video,utility + labels: + - "homelab.category=media" + - "homelab.description=Plex media streaming server" + + # Jellyfin - Free alternative to Plex + # Access at: http://server-ip:8096 + jellyfin: + image: jellyfin/jellyfin:10.8.13 + container_name: jellyfin + restart: unless-stopped + networks: + - media-network + - homelab-network + ports: + - "8096:8096" + volumes: + - ./config/jellyfin:/config + - ./config/jellyfin/cache:/cache + - ${MEDIADIR:-/media}:/media:ro + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ:-America/New_York} + devices: + # Uncomment for hardware transcoding + # - /dev/dri:/dev/dri + labels: + - "homelab.category=media" + - "homelab.description=Open-source media streaming server" + + # Sonarr - TV show automation + # Access at: http://server-ip:8989 + sonarr: + image: lscr.io/linuxserver/sonarr:4.0.0 + container_name: sonarr + restart: unless-stopped + networks: + - media-network + - homelab-network + ports: + - "8989:8989" + volumes: + - ./config/sonarr:/config + - ${MEDIADIR:-/media}:/media + - ${DOWNLOADDIR:-/downloads}:/downloads + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ:-America/New_York} + labels: + - "homelab.category=media" + - "homelab.description=TV show management and automation" + + # Radarr - Movie automation + # Access at: http://server-ip:7878 + radarr: + image: lscr.io/linuxserver/radarr:5.2.6 + container_name: radarr + restart: unless-stopped + networks: + - media-network + - homelab-network + ports: + - "7878:7878" + volumes: + - ./config/radarr:/config + - ${MEDIADIR:-/media}:/media + - ${DOWNLOADDIR:-/downloads}:/downloads + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ:-America/New_York} + labels: + - "homelab.category=media" + - "homelab.description=Movie management and automation" + + # Prowlarr - Indexer manager + # Access at: http://server-ip:9696 + prowlarr: + image: lscr.io/linuxserver/prowlarr:1.11.4 + container_name: prowlarr + restart: unless-stopped + networks: + - media-network + - homelab-network + ports: + - "9696:9696" + volumes: + - ./config/prowlarr:/config + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ:-America/New_York} + labels: + - "homelab.category=media" + - "homelab.description=Indexer manager for Sonarr/Radarr" + + # qBittorrent - Torrent client + # Access at: http://server-ip:8081 + # Default credentials: admin / adminadmin + qbittorrent: + image: lscr.io/linuxserver/qbittorrent:4.6.2 + container_name: qbittorrent + restart: unless-stopped + networks: + - media-network + - homelab-network + ports: + - "8081:8081" # Web UI + - "6881:6881" # Torrent port TCP + - "6881:6881/udp" # Torrent port UDP + volumes: + - ./config/qbittorrent:/config + - ${DOWNLOADDIR:-/downloads}:/downloads + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ:-America/New_York} + - WEBUI_PORT=8081 + labels: + - "homelab.category=media" + - "homelab.description=Torrent download client" + +volumes: + plex-transcode: + driver: local + +networks: + media-network: + driver: bridge + homelab-network: + external: true diff --git a/docker-compose/monitoring.yml b/docker-compose/monitoring.yml new file mode 100644 index 0000000..a4d67a3 --- /dev/null +++ b/docker-compose/monitoring.yml @@ -0,0 +1,173 @@ +# Monitoring and Observability Services +# Services for monitoring your homelab infrastructure + +services: + # Prometheus - Metrics collection and storage + # Access at: http://server-ip:9090 + prometheus: + image: prom/prometheus:v2.48.1 + container_name: prometheus + restart: unless-stopped + networks: + - monitoring-network + - homelab-network + ports: + - "9090:9090" + volumes: + - ./config/prometheus:/etc/prometheus + - prometheus-data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=30d' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--web.enable-lifecycle' + user: "${PUID:-1000}:${PGID:-1000}" + labels: + - "homelab.category=monitoring" + - "homelab.description=Metrics collection and time-series database" + + # Grafana - Metrics visualization + # Access at: http://server-ip:3000 + # Default credentials: admin / admin (change on first login) + grafana: + image: grafana/grafana:10.2.3 + container_name: grafana + restart: unless-stopped + networks: + - monitoring-network + - homelab-network + ports: + - "3000:3000" + volumes: + - grafana-data:/var/lib/grafana + - ./config/grafana/provisioning:/etc/grafana/provisioning + environment: + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin} + - GF_USERS_ALLOW_SIGN_UP=false + - GF_SERVER_ROOT_URL=http://${SERVER_IP}:3000 + - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource,grafana-piechart-panel + user: "${PUID:-1000}:${PGID:-1000}" + depends_on: + - prometheus + labels: + - "homelab.category=monitoring" + - "homelab.description=Metrics visualization and dashboards" + + # Node Exporter - Host metrics exporter + # Metrics at: http://server-ip:9100/metrics + node-exporter: + image: prom/node-exporter:v1.7.0 + container_name: node-exporter + restart: unless-stopped + networks: + - monitoring-network + ports: + - "9100:9100" + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.rootfs=/rootfs' + - '--path.sysfs=/host/sys' + - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' + labels: + - "homelab.category=monitoring" + - "homelab.description=Hardware and OS metrics exporter" + + # cAdvisor - Container metrics exporter + # Access at: http://server-ip:8082 + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.47.2 + container_name: cadvisor + restart: unless-stopped + networks: + - monitoring-network + ports: + - "8082:8080" + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker:/var/lib/docker:ro + - /dev/disk:/dev/disk:ro + privileged: true + devices: + - /dev/kmsg + labels: + - "homelab.category=monitoring" + - "homelab.description=Container metrics and performance monitoring" + + # Uptime Kuma - Uptime monitoring + # Access at: http://server-ip:3001 + uptime-kuma: + image: louislam/uptime-kuma:1.23.11 + container_name: uptime-kuma + restart: unless-stopped + networks: + - monitoring-network + - homelab-network + ports: + - "3001:3001" + volumes: + - uptime-kuma-data:/app/data + labels: + - "homelab.category=monitoring" + - "homelab.description=Service uptime monitoring and alerts" + + # Loki - Log aggregation + # Access at: http://server-ip:3100 + loki: + image: grafana/loki:2.9.3 + container_name: loki + restart: unless-stopped + networks: + - monitoring-network + ports: + - "3100:3100" + volumes: + - ./config/loki:/etc/loki + - loki-data:/loki + command: -config.file=/etc/loki/loki-config.yml + user: "${PUID:-1000}:${PGID:-1000}" + labels: + - "homelab.category=monitoring" + - "homelab.description=Log aggregation system" + + # Promtail - Log shipper for Loki + # Ships Docker container logs to Loki + promtail: + image: grafana/promtail:2.9.3 + container_name: promtail + restart: unless-stopped + networks: + - monitoring-network + volumes: + - ./config/promtail:/etc/promtail + - /var/log:/var/log:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + command: -config.file=/etc/promtail/promtail-config.yml + depends_on: + - loki + labels: + - "homelab.category=monitoring" + - "homelab.description=Log collector for Loki" + +volumes: + prometheus-data: + driver: local + grafana-data: + driver: local + uptime-kuma-data: + driver: local + loki-data: + driver: local + +networks: + monitoring-network: + driver: bridge + homelab-network: + external: true diff --git a/docs/docker-guidelines.md b/docs/docker-guidelines.md new file mode 100644 index 0000000..2d7e681 --- /dev/null +++ b/docs/docker-guidelines.md @@ -0,0 +1,858 @@ +# Docker Service Management Guidelines + +## Overview + +This document provides comprehensive guidelines for managing Docker services in your AI-powered homelab. These guidelines ensure consistency, maintainability, and reliability across your entire infrastructure. + +## Table of Contents + +1. [Philosophy](#philosophy) +2. [Docker Compose vs Docker Run](#docker-compose-vs-docker-run) +3. [Service Creation Guidelines](#service-creation-guidelines) +4. [Service Modification Guidelines](#service-modification-guidelines) +5. [Naming Conventions](#naming-conventions) +6. [Network Architecture](#network-architecture) +7. [Volume Management](#volume-management) +8. [Security Best Practices](#security-best-practices) +9. [Monitoring and Logging](#monitoring-and-logging) +10. [Troubleshooting](#troubleshooting) + +## Philosophy + +### Core Principles + +1. **Infrastructure as Code**: All services must be defined in Docker Compose files +2. **Reproducibility**: Any service should be rebuildable from compose files +3. **Documentation**: Every non-obvious configuration must be commented +4. **Consistency**: Use the same patterns across all services +5. **Safety First**: Always test changes in isolation before deploying + +### The Stack Mindset + +Think of your homelab as an interconnected stack where: +- Services depend on networks +- Networks connect services +- Volumes persist data +- Changes ripple through the system + +Always ask: "How does this change affect other services?" + +## Docker Compose vs Docker Run + +### Docker Compose: For Everything Persistent + +Use Docker Compose for: +- All production services +- Services that need to restart automatically +- Multi-container applications +- Services with complex configurations +- Anything you want to keep long-term + +**Example:** +```yaml +# docker-compose/plex.yml +services: + plex: + image: plexinc/pms-docker:1.40.0.7775-456fbaf97 + container_name: plex + restart: unless-stopped + networks: + - media-network + ports: + - "32400:32400" + volumes: + - ./config/plex:/config + - /media:/media:ro + environment: + - PUID=1000 + - PGID=1000 + - TZ=America/New_York +``` + +### Docker Run: For Temporary Operations Only + +Use `docker run` for: +- Testing new images +- One-off commands +- Debugging +- Verification tasks (like GPU testing) + +**Examples:** +```bash +# Test if NVIDIA GPU is accessible +docker run --rm --gpus all nvidia/cuda:12.0.0-base-ubuntu22.04 nvidia-smi + +# Quick test of a new image +docker run --rm -it alpine:latest /bin/sh + +# One-off database backup +docker run --rm -v mydata:/data busybox tar czf /backup/data.tar.gz /data +``` + +## Service Creation Guidelines + +### Step-by-Step Process + +#### 1. Planning Phase + +**Before writing any YAML:** + +- [ ] What problem does this service solve? +- [ ] Does a similar service already exist? +- [ ] What are the dependencies? +- [ ] What ports are needed? +- [ ] What data needs to persist? +- [ ] What environment variables are required? +- [ ] What networks should it connect to? +- [ ] Are there any security considerations? + +#### 2. Research Phase + +- Read the official image documentation +- Check example configurations +- Review resource requirements +- Understand health check requirements +- Note any special permissions needed + +#### 3. Implementation Phase + +**Start with a minimal configuration:** + +```yaml +services: + service-name: + image: vendor/image:specific-version + container_name: service-name + restart: unless-stopped +``` + +**Add networks:** +```yaml + networks: + - homelab-network +``` + +**Add ports (if externally accessible):** +```yaml + ports: + - "8080:8080" # Web UI +``` + +**Add volumes:** +```yaml + volumes: + - ./config/service-name:/config + - service-data:/data +``` + +**Add environment variables:** +```yaml + environment: + - PUID=1000 + - PGID=1000 + - TZ=${TIMEZONE} +``` + +**Add health checks (if applicable):** +```yaml + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s +``` + +#### 4. Testing Phase + +```bash +# Validate syntax +docker compose -f docker-compose/service.yml config + +# Start in foreground to see logs +docker compose -f docker-compose/service.yml up + +# If successful, restart in background +docker compose -f docker-compose/service.yml down +docker compose -f docker-compose/service.yml up -d +``` + +#### 5. Documentation Phase + +Add comments to your compose file: +```yaml +services: + sonarr: + image: lscr.io/linuxserver/sonarr:4.0.0 + container_name: sonarr + # Sonarr - TV Show management and automation + # Web UI: http://server-ip:8989 + # Connects to: Prowlarr (indexers), qBittorrent (downloads) + restart: unless-stopped +``` + +Update your main README or service-specific README with: +- Service purpose +- Access URLs +- Default credentials (if any) +- Configuration notes +- Backup instructions + +## Service Modification Guidelines + +### Before Modifying + +1. **Back up current configuration:** + ```bash + cp docker-compose/service.yml docker-compose/service.yml.backup + ``` + +2. **Document why you're making the change** + - Create a comment in the compose file + - Note in your changelog or docs + +3. **Understand the current state:** + ```bash + # Check if service is running + docker compose -f docker-compose/service.yml ps + + # Review current configuration + docker compose -f docker-compose/service.yml config + + # Check logs for any existing issues + docker compose -f docker-compose/service.yml logs --tail=50 + ``` + +### Making the Change + +1. **Edit the compose file** + - Make minimal, targeted changes + - Keep existing structure when possible + - Add comments for new configurations + +2. **Validate syntax:** + ```bash + docker compose -f docker-compose/service.yml config + ``` + +3. **Apply the change:** + ```bash + # Pull new image if version changed + docker compose -f docker-compose/service.yml pull + + # Recreate the service + docker compose -f docker-compose/service.yml up -d + ``` + +4. **Verify the change:** + ```bash + # Check service is running + docker compose -f docker-compose/service.yml ps + + # Watch logs for errors + docker compose -f docker-compose/service.yml logs -f + + # Test functionality + curl http://localhost:port/health + ``` + +### Rollback Plan + +If something goes wrong: +```bash +# Stop the service +docker compose -f docker-compose/service.yml down + +# Restore backup +mv docker-compose/service.yml.backup docker-compose/service.yml + +# Restart with old configuration +docker compose -f docker-compose/service.yml up -d +``` + +## Naming Conventions + +### Service Names + +Use lowercase with hyphens: +- ✅ `plex-media-server` +- ✅ `home-assistant` +- ❌ `PlexMediaServer` +- ❌ `home_assistant` + +### Container Names + +Match service names or be descriptive: +```yaml +services: + plex: + container_name: plex # Simple match + + database: + container_name: media-database # Descriptive +``` + +### Network Names + +Use purpose-based naming: +- `homelab-network` - Main network +- `media-network` - Media services +- `monitoring-network` - Observability stack +- `isolated-network` - Untrusted services + +### Volume Names + +Use `service-purpose` pattern: +```yaml +volumes: + plex-config: + plex-metadata: + database-data: + nginx-certs: +``` + +### File Names + +Organize by function: +- `docker-compose/media.yml` - Media services (Plex, Jellyfin, etc.) +- `docker-compose/monitoring.yml` - Monitoring stack +- `docker-compose/infrastructure.yml` - Core services (DNS, reverse proxy) +- `docker-compose/development.yml` - Dev tools + +## Network Architecture + +### Network Types + +1. **Bridge Networks** (Most Common) + ```yaml + networks: + homelab-network: + driver: bridge + ipam: + config: + - subnet: 172.20.0.0/16 + ``` + +2. **Host Network** (When Performance Critical) + ```yaml + services: + performance-critical: + network_mode: host + ``` + +3. **Overlay Networks** (For Swarm/Multi-host) + ```yaml + networks: + swarm-network: + driver: overlay + ``` + +### Network Design Patterns + +#### Pattern 1: Single Shared Network +Simplest approach for small homelabs: +```yaml +networks: + homelab-network: + external: true +``` + +Create once manually: +```bash +docker network create homelab-network +``` + +#### Pattern 2: Segmented Networks +Better security through isolation: +```yaml +networks: + frontend-network: # Web-facing services + backend-network: # Databases, internal services + monitoring-network: # Observability +``` + +#### Pattern 3: Service-Specific Networks +Each service group has its own network: +```yaml +services: + web: + networks: + - frontend + - backend + + database: + networks: + - backend # Not exposed to frontend +``` + +### Network Security + +- Place databases on internal networks only +- Use separate networks for untrusted services +- Expose minimal ports to the host +- Use reverse proxies for web services + +## Volume Management + +### Volume Types + +#### Named Volumes (Managed by Docker) +```yaml +volumes: + database-data: + driver: local +``` + +**Use for:** +- Database files +- Application data +- Anything Docker should manage + +**Advantages:** +- Docker handles permissions +- Easy to backup/restore +- Portable across systems + +#### Bind Mounts (Direct Host Paths) +```yaml +volumes: + - ./config/app:/config + - /media:/media:ro +``` + +**Use for:** +- Configuration files you edit directly +- Large media libraries +- Shared data with host + +**Advantages:** +- Direct file access +- Easy to edit +- Can share with host applications + +#### tmpfs Mounts (RAM) +```yaml +tmpfs: + - /tmp +``` + +**Use for:** +- Temporary data +- Cache that doesn't need persistence +- Sensitive data that shouldn't touch disk + +### Volume Best Practices + +1. **Consistent Paths:** + ```yaml + volumes: + - ./config/service:/config # Always use /config inside container + - service-data:/data # Always use /data for application data + ``` + +2. **Read-Only When Possible:** + ```yaml + volumes: + - /media:/media:ro # Media library is read-only + ``` + +3. **Separate Config from Data:** + ```yaml + volumes: + - ./config/plex:/config # Editable configuration + - plex-metadata:/metadata # Application-managed data + ``` + +4. **Backup Strategy:** + ```bash + # Backup named volume + docker run --rm \ + -v plex-metadata:/data \ + -v $(pwd)/backups:/backup \ + busybox tar czf /backup/plex-metadata.tar.gz /data + ``` + +## Security Best Practices + +### 1. Image Security + +**Pin Specific Versions:** +```yaml +# ✅ Good - Specific version +image: nginx:1.25.3-alpine + +# ❌ Bad - Latest tag +image: nginx:latest +``` + +**Use Official or Trusted Images:** +- Official Docker images +- LinuxServer.io (lscr.io) +- Trusted vendors + +**Scan Images:** +```bash +docker scan vendor/image:tag +``` + +### 2. Secret Management + +**Never Commit Secrets:** +```yaml +# .env file (gitignored) +DB_PASSWORD=super-secret-password +API_KEY=sk-1234567890 + +# docker-compose.yml +environment: + - DB_PASSWORD=${DB_PASSWORD} + - API_KEY=${API_KEY} +``` + +**Provide Templates:** +```bash +# .env.example (committed) +DB_PASSWORD=changeme +API_KEY=your-api-key-here +``` + +### 3. User Permissions + +**Run as Non-Root:** +```yaml +environment: + - PUID=1000 # Your user ID + - PGID=1000 # Your group ID +``` + +**Check Current User:** +```bash +id -u # Gets your UID +id -g # Gets your GID +``` + +### 4. Network Security + +**Minimal Exposure:** +```yaml +# ✅ Good - Only expose what's needed +ports: + - "127.0.0.1:8080:8080" # Only accessible from localhost + +# ❌ Bad - Exposed to all interfaces +ports: + - "8080:8080" +``` + +**Use Reverse Proxy:** +```yaml +# Don't expose services directly +# Use Nginx/Traefik to proxy with SSL +``` + +### 5. Resource Limits + +**Prevent Resource Exhaustion:** +```yaml +deploy: + resources: + limits: + cpus: '2' + memory: 4G + reservations: + cpus: '0.5' + memory: 1G +``` + +## Monitoring and Logging + +### Logging Configuration + +**Standard Logging:** +```yaml +logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" +``` + +**Centralized Logging:** +```yaml +logging: + driver: "syslog" + options: + syslog-address: "tcp://192.168.1.100:514" +``` + +### Health Checks + +**HTTP Health Check:** +```yaml +healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 30s + timeout: 10s + retries: 3 +``` + +**TCP Health Check:** +```yaml +healthcheck: + test: ["CMD-SHELL", "nc -z localhost 5432 || exit 1"] + interval: 30s + timeout: 5s + retries: 3 +``` + +**Custom Script:** +```yaml +healthcheck: + test: ["CMD", "/healthcheck.sh"] + interval: 30s + timeout: 10s + retries: 3 +``` + +### Monitoring Stack Example + +```yaml +# docker-compose/monitoring.yml +services: + prometheus: + image: prom/prometheus:v2.48.0 + container_name: prometheus + restart: unless-stopped + volumes: + - ./config/prometheus:/etc/prometheus + - prometheus-data:/prometheus + ports: + - "9090:9090" + networks: + - monitoring-network + + grafana: + image: grafana/grafana:10.2.2 + container_name: grafana + restart: unless-stopped + volumes: + - grafana-data:/var/lib/grafana + ports: + - "3000:3000" + networks: + - monitoring-network + depends_on: + - prometheus + +volumes: + prometheus-data: + grafana-data: + +networks: + monitoring-network: + driver: bridge +``` + +## Troubleshooting + +### Common Issues + +#### Service Won't Start + +**1. Check logs:** +```bash +docker compose -f docker-compose/service.yml logs +``` + +**2. Validate configuration:** +```bash +docker compose -f docker-compose/service.yml config +``` + +**3. Check for port conflicts:** +```bash +# See what's using a port +sudo netstat -tlnp | grep :8080 +``` + +**4. Verify image exists:** +```bash +docker images | grep service-name +``` + +#### Permission Errors + +**1. Check PUID/PGID:** +```bash +# Your user ID +id -u + +# Your group ID +id -g +``` + +**2. Fix directory permissions:** +```bash +sudo chown -R 1000:1000 ./config/service-name +``` + +**3. Check volume permissions:** +```bash +docker compose -f docker-compose/service.yml exec service-name ls -la /config +``` + +#### Network Connectivity Issues + +**1. Verify network exists:** +```bash +docker network ls +docker network inspect homelab-network +``` + +**2. Check if services are on same network:** +```bash +docker network inspect homelab-network | grep Name +``` + +**3. Test connectivity:** +```bash +docker compose -f docker-compose/service.yml exec service1 ping service2 +``` + +#### Container Keeps Restarting + +**1. Watch logs:** +```bash +docker compose -f docker-compose/service.yml logs -f +``` + +**2. Check health status:** +```bash +docker compose -f docker-compose/service.yml ps +``` + +**3. Inspect container:** +```bash +docker inspect container-name +``` + +### Debugging Commands + +```bash +# Enter running container +docker compose -f docker-compose/service.yml exec service-name /bin/sh + +# View full container configuration +docker inspect container-name + +# See resource usage +docker stats container-name + +# View recent events +docker events --since 10m + +# Check disk space +docker system df +``` + +### Recovery Procedures + +#### Service Corrupted + +```bash +# Stop service +docker compose -f docker-compose/service.yml down + +# Remove container and volumes (backup first!) +docker compose -f docker-compose/service.yml down -v + +# Recreate from scratch +docker compose -f docker-compose/service.yml up -d +``` + +#### Network Issues + +```bash +# Remove and recreate network +docker network rm homelab-network +docker network create homelab-network + +# Restart services +docker compose -f docker-compose/*.yml up -d +``` + +#### Full System Reset (Nuclear Option) + +```bash +# ⚠️ WARNING: This removes everything! +# Backup first! + +# Stop all containers +docker stop $(docker ps -aq) + +# Remove all containers +docker rm $(docker ps -aq) + +# Remove all volumes (careful!) +docker volume rm $(docker volume ls -q) + +# Remove all networks (except defaults) +docker network prune -f + +# Rebuild from compose files +docker compose -f docker-compose/*.yml up -d +``` + +## Maintenance + +### Regular Tasks + +**Weekly:** +- Review logs for errors +- Check disk space: `docker system df` +- Update security patches on images + +**Monthly:** +- Update images to latest versions +- Review and prune unused resources +- Backup volumes +- Review and optimize compose files + +**Quarterly:** +- Full stack review +- Documentation update +- Performance optimization +- Security audit + +### Update Procedure + +```bash +# 1. Backup current state +docker compose -f docker-compose/service.yml config > backup/service-config.yml + +# 2. Update image version in compose file +# Edit docker-compose/service.yml + +# 3. Pull new image +docker compose -f docker-compose/service.yml pull + +# 4. Recreate service +docker compose -f docker-compose/service.yml up -d + +# 5. Verify +docker compose -f docker-compose/service.yml logs -f + +# 6. Test functionality +# Access service and verify it works +``` + +## Conclusion + +Following these guidelines ensures: +- Consistent infrastructure +- Easy troubleshooting +- Reproducible deployments +- Maintainable system +- Better security + +Remember: **Infrastructure as Code** means treating your Docker Compose files as critical documentation. Keep them clean, commented, and version-controlled. diff --git a/docs/quick-reference.md b/docs/quick-reference.md new file mode 100644 index 0000000..a77b1b1 --- /dev/null +++ b/docs/quick-reference.md @@ -0,0 +1,394 @@ +# Quick Reference Guide + +## Common Commands + +### Service Management + +```bash +# Start all services in a file +docker compose -f docker-compose/file.yml up -d + +# Start specific service +docker compose -f docker-compose/file.yml up -d service-name + +# Stop all services +docker compose -f docker-compose/file.yml down + +# Stop specific service +docker compose -f docker-compose/file.yml stop service-name + +# Restart service +docker compose -f docker-compose/file.yml restart service-name + +# Remove service and volumes +docker compose -f docker-compose/file.yml down -v +``` + +### Monitoring + +```bash +# View logs +docker compose -f docker-compose/file.yml logs -f service-name + +# Check service status +docker compose -f docker-compose/file.yml ps + +# View resource usage +docker stats + +# Inspect service +docker inspect container-name +``` + +### Updates + +```bash +# Pull latest images +docker compose -f docker-compose/file.yml pull + +# Pull and update specific service +docker compose -f docker-compose/file.yml pull service-name +docker compose -f docker-compose/file.yml up -d service-name +``` + +### Network Management + +```bash +# List networks +docker network ls + +# Inspect network +docker network inspect homelab-network + +# Create network +docker network create network-name + +# Remove network +docker network rm network-name +``` + +### Volume Management + +```bash +# List volumes +docker volume ls + +# Inspect volume +docker volume inspect volume-name + +# Remove volume +docker volume rm volume-name + +# Backup volume +docker run --rm -v volume-name:/data -v $(pwd)/backups:/backup \ + busybox tar czf /backup/backup.tar.gz /data + +# Restore volume +docker run --rm -v volume-name:/data -v $(pwd)/backups:/backup \ + busybox tar xzf /backup/backup.tar.gz -C / +``` + +### System Maintenance + +```bash +# View disk usage +docker system df + +# Clean up unused resources +docker system prune + +# Clean up everything (careful!) +docker system prune -a --volumes + +# Remove unused images +docker image prune + +# Remove unused volumes +docker volume prune + +# Remove unused networks +docker network prune +``` + +## Port Reference + +### Infrastructure Services +- **80**: Nginx Proxy Manager (HTTP) +- **443**: Nginx Proxy Manager (HTTPS) +- **81**: Nginx Proxy Manager (Admin) +- **53**: Pi-hole (DNS) +- **8080**: Pi-hole (Web UI) +- **9000**: Portainer +- **9443**: Portainer (HTTPS) + +### Media Services +- **32400**: Plex +- **8096**: Jellyfin +- **8989**: Sonarr +- **7878**: Radarr +- **9696**: Prowlarr +- **8081**: qBittorrent + +### Monitoring Services +- **9090**: Prometheus +- **3000**: Grafana +- **9100**: Node Exporter +- **8082**: cAdvisor +- **3001**: Uptime Kuma +- **3100**: Loki + +### Development Services +- **8443**: Code Server +- **8929**: GitLab +- **2222**: GitLab SSH +- **5432**: PostgreSQL +- **6379**: Redis +- **5050**: pgAdmin +- **8888**: Jupyter Lab +- **1880**: Node-RED + +## Environment Variables Quick Reference + +```bash +# User/Group +PUID=1000 # Your user ID (get with: id -u) +PGID=1000 # Your group ID (get with: id -g) + +# General +TZ=America/New_York # Your timezone +SERVER_IP=192.168.1.100 # Server IP address + +# Paths +USERDIR=/home/username/homelab +MEDIADIR=/mnt/media +DOWNLOADDIR=/mnt/downloads +PROJECTDIR=/home/username/projects +``` + +## Network Setup + +```bash +# Create all networks at once +docker network create homelab-network +docker network create media-network +docker network create monitoring-network +docker network create database-network +``` + +## Service URLs + +After starting services, access them at: + +``` +Infrastructure: +http://SERVER_IP:81 - Nginx Proxy Manager +http://SERVER_IP:8080 - Pi-hole +http://SERVER_IP:9000 - Portainer + +Media: +http://SERVER_IP:32400/web - Plex +http://SERVER_IP:8096 - Jellyfin +http://SERVER_IP:8989 - Sonarr +http://SERVER_IP:7878 - Radarr +http://SERVER_IP:9696 - Prowlarr +http://SERVER_IP:8081 - qBittorrent + +Monitoring: +http://SERVER_IP:9090 - Prometheus +http://SERVER_IP:3000 - Grafana +http://SERVER_IP:3001 - Uptime Kuma + +Development: +http://SERVER_IP:8443 - Code Server +http://SERVER_IP:8929 - GitLab +http://SERVER_IP:5050 - pgAdmin +http://SERVER_IP:8888 - Jupyter Lab +http://SERVER_IP:1880 - Node-RED +``` + +## Troubleshooting Quick Fixes + +### Service won't start +```bash +# 1. Check logs +docker compose -f docker-compose/file.yml logs service-name + +# 2. Validate configuration +docker compose -f docker-compose/file.yml config + +# 3. Check what's using the port +sudo netstat -tlnp | grep PORT_NUMBER +``` + +### Permission errors +```bash +# Check your IDs +id -u # Should match PUID in .env +id -g # Should match PGID in .env + +# Fix ownership +sudo chown -R 1000:1000 ./config/service-name +``` + +### Network issues +```bash +# Check network exists +docker network inspect homelab-network + +# Recreate network +docker network rm homelab-network +docker network create homelab-network +docker compose -f docker-compose/file.yml up -d +``` + +### Container keeps restarting +```bash +# Watch logs in real-time +docker compose -f docker-compose/file.yml logs -f service-name + +# Check resource usage +docker stats container-name + +# Inspect container +docker inspect container-name | less +``` + +## Testing GPU Support (NVIDIA) + +```bash +# Test if nvidia-container-toolkit works +docker run --rm --gpus all nvidia/cuda:12.0.0-base-ubuntu22.04 nvidia-smi + +# If successful, you should see your GPU info +``` + +## Backup Commands + +```bash +# Backup all config directories +tar czf backup-config-$(date +%Y%m%d).tar.gz config/ + +# Backup a specific volume +docker run --rm \ + -v volume-name:/data \ + -v $(pwd)/backups:/backup \ + busybox tar czf /backup/volume-name-$(date +%Y%m%d).tar.gz /data + +# Backup .env file (store securely!) +cp .env .env.backup +``` + +## Health Checks + +```bash +# Check all container health status +docker ps --format "table {{.Names}}\t{{.Status}}" + +# Check specific service health +docker inspect --format='{{json .State.Health}}' container-name | jq +``` + +## Resource Limits + +Add to service definition if needed: + +```yaml +deploy: + resources: + limits: + cpus: '2' + memory: 4G + reservations: + cpus: '0.5' + memory: 1G +``` + +## Common Patterns + +### Add a new service +1. Choose the appropriate compose file +2. Add service definition following existing patterns +3. Use environment variables from .env +4. Connect to homelab-network +5. Pin specific image version +6. Add labels for organization +7. Test: `docker compose -f file.yml config` +8. Deploy: `docker compose -f file.yml up -d service-name` + +### Update a service version +1. Edit compose file with new version +2. Pull new image: `docker compose -f file.yml pull service-name` +3. Recreate: `docker compose -f file.yml up -d service-name` +4. Check logs: `docker compose -f file.yml logs -f service-name` + +### Remove a service +1. Stop service: `docker compose -f file.yml stop service-name` +2. Remove service: `docker compose -f file.yml rm service-name` +3. Remove from compose file +4. Optional: Remove volumes: `docker volume rm volume-name` +5. Optional: Remove config: `rm -rf config/service-name` + +## AI Assistant Usage in VS Code + +### Ask for help: +- "Add Jellyfin to my media stack" +- "Configure GPU for Plex" +- "Create monitoring dashboard setup" +- "Help me troubleshoot port conflicts" +- "Generate a compose file for Home Assistant" + +### The AI will: +- Check existing services +- Follow naming conventions +- Avoid port conflicts +- Use proper network configuration +- Include health checks +- Add documentation comments +- Suggest related services + +## Quick Deployment + +### Minimal setup +```bash +# 1. Clone and configure +git clone https://github.com/kelinfoxy/AI-Homelab.git +cd AI-Homelab +cp .env.example .env +nano .env # Edit values + +# 2. Create network +docker network create homelab-network + +# 3. Start Portainer (for container management) +docker compose -f docker-compose/infrastructure.yml up -d portainer + +# 4. Access at http://SERVER_IP:9000 +``` + +### Full stack deployment +```bash +# After minimal setup, deploy everything: +docker compose -f docker-compose/infrastructure.yml up -d +docker compose -f docker-compose/media.yml up -d +docker compose -f docker-compose/monitoring.yml up -d +docker compose -f docker-compose/development.yml up -d +``` + +## Maintenance Schedule + +### Daily (automated) +- Watchtower checks for updates at 4 AM + +### Weekly +- Review logs: `docker compose -f docker-compose/*.yml logs --tail=100` +- Check disk space: `docker system df` + +### Monthly +- Update pinned versions in compose files +- Backup volumes and configs +- Review security updates + +### Quarterly +- Full system audit +- Documentation review +- Performance optimization From cae7c51617ce927427fd782a142099075cdc7422 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 22:52:10 +0000 Subject: [PATCH 03/16] Fix Docker Compose syntax for devices arrays Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- docker-compose/media.yml | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/docker-compose/media.yml b/docker-compose/media.yml index 22f83d7..16389e8 100644 --- a/docker-compose/media.yml +++ b/docker-compose/media.yml @@ -23,16 +23,17 @@ services: - TZ=${TZ:-America/New_York} - PLEX_CLAIM=${PLEX_CLAIM} - ADVERTISE_IP=http://${SERVER_IP}:32400/ - devices: - # Uncomment for hardware transcoding (Intel QuickSync) - # - /dev/dri:/dev/dri - # Uncomment for NVIDIA GPU transcoding - # - /dev/nvidia0:/dev/nvidia0 - # - /dev/nvidiactl:/dev/nvidiactl - # - /dev/nvidia-modeset:/dev/nvidia-modeset - # - /dev/nvidia-uvm:/dev/nvidia-uvm - # - /dev/nvidia-uvm-tools:/dev/nvidia-uvm-tools - # Uncomment for NVIDIA GPU support + # Uncomment for hardware transcoding (Intel QuickSync) + # devices: + # - /dev/dri:/dev/dri + # Uncomment for NVIDIA GPU transcoding + # devices: + # - /dev/nvidia0:/dev/nvidia0 + # - /dev/nvidiactl:/dev/nvidiactl + # - /dev/nvidia-modeset:/dev/nvidia-modeset + # - /dev/nvidia-uvm:/dev/nvidia-uvm + # - /dev/nvidia-uvm-tools:/dev/nvidia-uvm-tools + # Uncomment for NVIDIA GPU support (requires nvidia-container-toolkit) # runtime: nvidia # environment: # - NVIDIA_VISIBLE_DEVICES=all @@ -60,9 +61,9 @@ services: - PUID=${PUID:-1000} - PGID=${PGID:-1000} - TZ=${TZ:-America/New_York} - devices: - # Uncomment for hardware transcoding - # - /dev/dri:/dev/dri + # Uncomment for hardware transcoding + # devices: + # - /dev/dri:/dev/dri labels: - "homelab.category=media" - "homelab.description=Open-source media streaming server" From 4c57c586110d5f28e2bb1afab47ee24235e85fa4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 22:54:04 +0000 Subject: [PATCH 04/16] Add configuration templates and getting started guide Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- config-templates/README.md | 196 +++++++ config-templates/loki/loki-config.yml | 46 ++ config-templates/prometheus/prometheus.yml | 49 ++ config-templates/promtail/promtail-config.yml | 53 ++ config-templates/redis/redis.conf | 42 ++ docs/getting-started.md | 506 ++++++++++++++++++ 6 files changed, 892 insertions(+) create mode 100644 config-templates/README.md create mode 100644 config-templates/loki/loki-config.yml create mode 100644 config-templates/prometheus/prometheus.yml create mode 100644 config-templates/promtail/promtail-config.yml create mode 100644 config-templates/redis/redis.conf create mode 100644 docs/getting-started.md diff --git a/config-templates/README.md b/config-templates/README.md new file mode 100644 index 0000000..c508b20 --- /dev/null +++ b/config-templates/README.md @@ -0,0 +1,196 @@ +# Configuration Templates + +This directory contains example configuration files for various services. These templates provide sensible defaults and are ready to use with minimal modifications. + +## Usage + +1. **Create your config directory** (if it doesn't exist): + ```bash + mkdir -p config/service-name + ``` + +2. **Copy the template** to your config directory: + ```bash + cp config-templates/service-name/* config/service-name/ + ``` + +3. **Edit the configuration** as needed for your environment + +4. **Start the service** using Docker Compose + +## Available Templates + +### Prometheus (`prometheus/prometheus.yml`) +Metrics collection and monitoring system configuration. + +**Features:** +- Pre-configured to scrape Node Exporter and cAdvisor +- 15-second scrape interval +- Ready for additional service monitoring + +**Setup:** +```bash +mkdir -p config/prometheus +cp config-templates/prometheus/prometheus.yml config/prometheus/ +docker compose -f docker-compose/monitoring.yml up -d prometheus +``` + +### Loki (`loki/loki-config.yml`) +Log aggregation system configuration. + +**Features:** +- Filesystem-based storage +- 30-day log retention +- Automatic log compaction +- Pre-configured for Promtail + +**Setup:** +```bash +mkdir -p config/loki +cp config-templates/loki/loki-config.yml config/loki/ +docker compose -f docker-compose/monitoring.yml up -d loki +``` + +### Promtail (`promtail/promtail-config.yml`) +Log shipper for Loki. + +**Features:** +- Automatically ships Docker container logs +- Parses Docker JSON format +- Extracts container IDs and names +- Optional system log collection + +**Setup:** +```bash +mkdir -p config/promtail +cp config-templates/promtail/promtail-config.yml config/promtail/ +docker compose -f docker-compose/monitoring.yml up -d promtail +``` + +### Redis (`redis/redis.conf`) +In-memory data store configuration. + +**Features:** +- Both AOF and RDB persistence enabled +- 256MB memory limit with LRU eviction +- Sensible defaults for homelab use +- Security options (password protection available) + +**Setup:** +```bash +mkdir -p config/redis +cp config-templates/redis/redis.conf config/redis/ +# Optional: Edit redis.conf to set a password +docker compose -f docker-compose/development.yml up -d redis +``` + +## Customization Tips + +### Prometheus +- Add more scrape targets to monitor additional services +- Adjust `scrape_interval` based on your needs (lower = more frequent, more data) +- Configure alerting by uncommenting the alertmanager section + +### Loki +- Adjust `retention_period` to keep logs longer or shorter +- Change storage from filesystem to S3 for better scalability +- Configure multiple tenants if needed + +### Promtail +- Add more scrape configs for system logs, application logs, etc. +- Customize pipeline stages to extract more labels +- Filter logs based on patterns + +### Redis +- Set `maxmemory` based on your available RAM +- Choose appropriate `maxmemory-policy` for your use case +- Enable password protection by uncommenting `requirepass` + +## Service-Specific Notes + +### Services That Don't Need Config Templates + +Many services work perfectly with just environment variables and don't require separate config files: + +- **Plex, Jellyfin**: Configure via web UI +- **Sonarr, Radarr, Prowlarr**: Configure via web UI +- **Portainer**: Configure via web UI +- **Grafana**: Can use provisioning or web UI +- **Most LinuxServer.io images**: Configured via environment variables + +### Services That Benefit from Config Files + +- **Prometheus**: Requires `prometheus.yml` for scrape configuration +- **Loki**: Requires config for storage and retention +- **Promtail**: Requires config for log sources +- **Redis**: Benefits from custom config for persistence and security +- **Nginx**: Needs config for proxy rules (use Nginx Proxy Manager UI instead) + +## Best Practices + +1. **Version Control**: Keep your config templates in git +2. **Secrets**: Never commit passwords or API keys +3. **Comments**: Add comments explaining custom settings +4. **Backups**: Backup config directories regularly +5. **Testing**: Test config changes in a separate environment first + +## Creating New Templates + +When creating templates for other services: + +1. Start with the official documentation +2. Use sensible defaults for homelab use +3. Add comments explaining important settings +4. Include examples for common customizations +5. Test the template before committing + +## Getting Help + +- Check the official documentation for each service +- Ask GitHub Copilot in VS Code for configuration help +- Review the [Docker Guidelines](../docs/docker-guidelines.md) +- Consult service-specific community forums + +## Example: Full Monitoring Stack Setup + +```bash +# Create all config directories +mkdir -p config/{prometheus,loki,promtail,grafana} + +# Copy templates +cp config-templates/prometheus/prometheus.yml config/prometheus/ +cp config-templates/loki/loki-config.yml config/loki/ +cp config-templates/promtail/promtail-config.yml config/promtail/ + +# Start the monitoring stack +docker compose -f docker-compose/monitoring.yml up -d + +# Access services +# Prometheus: http://server-ip:9090 +# Grafana: http://server-ip:3000 +# Loki: http://server-ip:3100 +``` + +## Troubleshooting + +### Config file not found +Ensure you copied the template to the correct location referenced in the docker-compose file. + +### Permission errors +Fix ownership: +```bash +sudo chown -R 1000:1000 config/service-name +``` + +### Syntax errors +Validate YAML files: +```bash +# For YAML files +python3 -c "import yaml; yaml.safe_load(open('config/service/config.yml'))" +``` + +### Service won't start +Check logs for configuration errors: +```bash +docker compose -f docker-compose/file.yml logs service-name +``` diff --git a/config-templates/loki/loki-config.yml b/config-templates/loki/loki-config.yml new file mode 100644 index 0000000..2d7c57c --- /dev/null +++ b/config-templates/loki/loki-config.yml @@ -0,0 +1,46 @@ +# Loki Configuration Template +# Copy this file to ./config/loki/loki-config.yml + +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +ruler: + alertmanager_url: http://localhost:9093 + +# Retention configuration (delete logs older than 30 days) +limits_config: + retention_period: 720h # 30 days + +# Compactor to delete old data +compactor: + working_directory: /loki/compactor + shared_store: filesystem + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + retention_delete_worker_count: 150 diff --git a/config-templates/prometheus/prometheus.yml b/config-templates/prometheus/prometheus.yml new file mode 100644 index 0000000..ab3ee50 --- /dev/null +++ b/config-templates/prometheus/prometheus.yml @@ -0,0 +1,49 @@ +# Prometheus Configuration Template +# Copy this file to ./config/prometheus/prometheus.yml + +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + monitor: 'homelab' + +# Alertmanager configuration (optional) +# alerting: +# alertmanagers: +# - static_configs: +# - targets: +# - alertmanager:9093 + +# Load rules once and periodically evaluate them +# rule_files: +# - "alerts/*.yml" + +# Scrape configurations +scrape_configs: + # Prometheus itself + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + # Node Exporter - System metrics + - job_name: 'node-exporter' + static_configs: + - targets: ['node-exporter:9100'] + labels: + instance: 'homelab-server' + + # cAdvisor - Container metrics + - job_name: 'cadvisor' + static_configs: + - targets: ['cadvisor:8080'] + labels: + instance: 'homelab-server' + + # Add your own services here + # Example: Monitor a service with /metrics endpoint + # - job_name: 'my-service' + # static_configs: + # - targets: ['my-service:8080'] + # labels: + # instance: 'homelab-server' + # service: 'my-service' diff --git a/config-templates/promtail/promtail-config.yml b/config-templates/promtail/promtail-config.yml new file mode 100644 index 0000000..19f4f24 --- /dev/null +++ b/config-templates/promtail/promtail-config.yml @@ -0,0 +1,53 @@ +# Promtail Configuration Template +# Copy this file to ./config/promtail/promtail-config.yml + +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + # Docker container logs + - job_name: docker + static_configs: + - targets: + - localhost + labels: + job: docker + __path__: /var/lib/docker/containers/*/*-json.log + + pipeline_stages: + # Parse Docker JSON logs + - json: + expressions: + output: log + stream: stream + attrs: attrs + + # Extract container name from path + - regex: + expression: '/var/lib/docker/containers/(?P[^/]+)/.*' + source: filename + + # Add labels + - labels: + stream: + container_id: + + # Output the log line + - output: + source: output + + # System logs (optional) + # - job_name: system + # static_configs: + # - targets: + # - localhost + # labels: + # job: varlogs + # __path__: /var/log/*.log diff --git a/config-templates/redis/redis.conf b/config-templates/redis/redis.conf new file mode 100644 index 0000000..d6990bb --- /dev/null +++ b/config-templates/redis/redis.conf @@ -0,0 +1,42 @@ +# Redis Configuration Template +# Copy this file to ./config/redis/redis.conf + +# Network +bind 0.0.0.0 +protected-mode yes +port 6379 + +# General +daemonize no +supervised no +pidfile /var/run/redis_6379.pid +loglevel notice +logfile "" + +# Persistence - AOF (Append Only File) +appendonly yes +appendfilename "appendonly.aof" +appendfsync everysec +no-appendfsync-on-rewrite no +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb + +# Persistence - RDB (Snapshotting) +save 900 1 +save 300 10 +save 60 10000 +stop-writes-on-bgsave-error yes +rdbcompression yes +rdbchecksum yes +dbfilename dump.rdb +dir /data + +# Memory Management +maxmemory 256mb +maxmemory-policy allkeys-lru + +# Security +# requirepass yourpassword # Uncomment and set a strong password + +# Limits +maxclients 10000 diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..36522b4 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,506 @@ +# Getting Started Guide + +This guide will walk you through setting up your AI-powered homelab from scratch. + +## Prerequisites + +Before you begin, ensure you have: + +- [ ] A Linux server (Ubuntu 22.04+ recommended) +- [ ] Docker Engine 24.0+ installed +- [ ] Docker Compose V2 installed +- [ ] Git installed +- [ ] At least 8GB RAM (16GB+ recommended) +- [ ] Sufficient disk space (100GB+ recommended) +- [ ] Static IP address for your server +- [ ] VS Code with GitHub Copilot extension (for AI assistance) + +## Step 1: Verify Docker Installation + +```bash +# Check Docker version +docker --version +# Should show: Docker version 24.0.0 or higher + +# Check Docker Compose version +docker compose version +# Should show: Docker Compose version v2.x.x + +# Test Docker works +docker run --rm hello-world +# Should download and run successfully +``` + +## Step 2: Clone the Repository + +```bash +# Navigate to your home directory +cd ~ + +# Clone the repository +git clone https://github.com/kelinfoxy/AI-Homelab.git + +# Enter the directory +cd AI-Homelab +``` + +## Step 3: Configure Environment Variables + +```bash +# Copy the example environment file +cp .env.example .env + +# Get your user and group IDs +id -u # This is your PUID +id -g # This is your PGID + +# Edit the .env file +nano .env +``` + +**Update these values in `.env`:** +```bash +# Your user/group IDs +PUID=1000 # Replace with your user ID +PGID=1000 # Replace with your group ID + +# Your timezone (find yours: timedatectl list-timezones) +TZ=America/New_York + +# Your server's IP address +SERVER_IP=192.168.1.100 # Replace with your actual IP + +# Directory paths +USERDIR=/home/yourusername/homelab # Update username +MEDIADIR=/mnt/media # Update if different +DOWNLOADDIR=/mnt/downloads # Update if different + +# Set secure passwords for services +GRAFANA_ADMIN_PASSWORD=your-secure-password-here +CODE_SERVER_PASSWORD=your-secure-password-here +POSTGRES_PASSWORD=your-secure-password-here +PGADMIN_PASSWORD=your-secure-password-here +JUPYTER_TOKEN=your-secure-token-here +PIHOLE_PASSWORD=your-secure-password-here +``` + +**Save and exit** (Ctrl+X, Y, Enter in nano) + +## Step 4: Create Docker Networks + +```bash +# Create the main homelab network +docker network create homelab-network + +# Create additional networks for better security +docker network create media-network +docker network create monitoring-network +docker network create database-network + +# Verify networks were created +docker network ls +``` + +## Step 5: Create Configuration Directories + +```bash +# Create the main config directory +mkdir -p config + +# Create config directories for services you plan to use +mkdir -p config/{nginx-proxy-manager,pihole,portainer} +mkdir -p config/{plex,sonarr,radarr,prowlarr,qbittorrent,jellyfin} +mkdir -p config/{prometheus,grafana,loki,promtail} +mkdir -p config/{code-server,postgres,redis} + +# Set proper permissions +sudo chown -R $(id -u):$(id -g) config/ +``` + +## Step 6: Copy Configuration Templates (Optional) + +For services that need config files: + +```bash +# Prometheus +mkdir -p config/prometheus +cp config-templates/prometheus/prometheus.yml config/prometheus/ + +# Loki +mkdir -p config/loki +cp config-templates/loki/loki-config.yml config/loki/ + +# Promtail +mkdir -p config/promtail +cp config-templates/promtail/promtail-config.yml config/promtail/ + +# Redis +mkdir -p config/redis +cp config-templates/redis/redis.conf config/redis/ +``` + +## Step 7: Start Your First Service (Portainer) + +Portainer provides a web UI for managing Docker containers. It's a great first service to deploy. + +```bash +# Start Portainer +docker compose -f docker-compose/infrastructure.yml up -d portainer + +# Check if it's running +docker compose -f docker-compose/infrastructure.yml ps + +# View logs +docker compose -f docker-compose/infrastructure.yml logs -f portainer +``` + +**Access Portainer:** +1. Open your browser +2. Navigate to `http://YOUR_SERVER_IP:9000` +3. Create an admin account on first login +4. Select "Get Started" and choose local Docker environment + +## Step 8: Deploy Infrastructure Services + +```bash +# Start all infrastructure services +docker compose -f docker-compose/infrastructure.yml up -d + +# Check status +docker compose -f docker-compose/infrastructure.yml ps + +# Services now running: +# - Nginx Proxy Manager: http://YOUR_SERVER_IP:81 +# - Pi-hole: http://YOUR_SERVER_IP:8080/admin +# - Portainer: http://YOUR_SERVER_IP:9000 +# - Watchtower: (runs in background, no UI) +``` + +### Configure Nginx Proxy Manager (Optional) + +1. Access: `http://YOUR_SERVER_IP:81` +2. Login with default credentials: + - Email: `admin@example.com` + - Password: `changeme` +3. Change password immediately +4. Add proxy hosts for your services + +### Configure Pi-hole (Optional) + +1. Access: `http://YOUR_SERVER_IP:8080/admin` +2. Login with password from `.env` (PIHOLE_PASSWORD) +3. Configure DNS settings +4. Update your router to use Pi-hole as DNS server + +## Step 9: Deploy Media Services (Optional) + +If you want a media server setup: + +```bash +# Start media services +docker compose -f docker-compose/media.yml up -d + +# Check status +docker compose -f docker-compose/media.yml ps + +# Services now running: +# - Plex: http://YOUR_SERVER_IP:32400/web +# - Jellyfin: http://YOUR_SERVER_IP:8096 +# - Sonarr: http://YOUR_SERVER_IP:8989 +# - Radarr: http://YOUR_SERVER_IP:7878 +# - Prowlarr: http://YOUR_SERVER_IP:9696 +# - qBittorrent: http://YOUR_SERVER_IP:8081 +``` + +### Initial Media Service Setup + +**Plex:** +1. Access: `http://YOUR_SERVER_IP:32400/web` +2. Sign in with your Plex account +3. Add your media libraries + +**Sonarr/Radarr:** +1. Access the web UI +2. Go to Settings → Profiles → Quality +3. Configure your quality preferences +4. Add Prowlarr as an indexer +5. Add qBittorrent as a download client + +## Step 10: Deploy Monitoring Services (Optional) + +For system and service monitoring: + +```bash +# Start monitoring services +docker compose -f docker-compose/monitoring.yml up -d + +# Check status +docker compose -f docker-compose/monitoring.yml ps + +# Services now running: +# - Prometheus: http://YOUR_SERVER_IP:9090 +# - Grafana: http://YOUR_SERVER_IP:3000 +# - Node Exporter: http://YOUR_SERVER_IP:9100 +# - cAdvisor: http://YOUR_SERVER_IP:8082 +# - Uptime Kuma: http://YOUR_SERVER_IP:3001 +``` + +### Configure Grafana + +1. Access: `http://YOUR_SERVER_IP:3000` +2. Login with credentials from `.env`: + - Username: `admin` + - Password: `GRAFANA_ADMIN_PASSWORD` from .env +3. Add Prometheus as a data source: + - URL: `http://prometheus:9090` +4. Import dashboards: + - Dashboard ID 1860 for Node Exporter + - Dashboard ID 893 for Docker metrics + +### Configure Uptime Kuma + +1. Access: `http://YOUR_SERVER_IP:3001` +2. Create an account on first login +3. Add monitors for your services + +## Step 11: Deploy Development Services (Optional) + +If you need development tools: + +```bash +# Start development services +docker compose -f docker-compose/development.yml up -d + +# Check status +docker compose -f docker-compose/development.yml ps + +# Services now running: +# - Code Server: http://YOUR_SERVER_IP:8443 +# - PostgreSQL: localhost:5432 +# - Redis: localhost:6379 +# - pgAdmin: http://YOUR_SERVER_IP:5050 +# - Jupyter Lab: http://YOUR_SERVER_IP:8888 +# - Node-RED: http://YOUR_SERVER_IP:1880 +``` + +## Step 12: Set Up VS Code with GitHub Copilot + +1. **Install VS Code** on your local machine (if not already installed) + +2. **Install GitHub Copilot extension:** + - Open VS Code + - Go to Extensions (Ctrl+Shift+X) + - Search for "GitHub Copilot" + - Click Install + - Sign in with your GitHub account + +3. **Clone your repository in VS Code:** + ```bash + # On your local machine + git clone https://github.com/kelinfoxy/AI-Homelab.git + cd AI-Homelab + code . + ``` + +4. **Start using AI assistance:** + - Open Copilot Chat (Ctrl+Shift+I or click the chat icon) + - The AI assistant automatically follows the guidelines in `.github/copilot-instructions.md` + - Ask questions like: + - "Help me add Home Assistant to my homelab" + - "Create a backup script for my Docker volumes" + - "How do I configure GPU support for Plex?" + +## Step 13: Verify Everything is Running + +```bash +# Check all running containers +docker ps + +# Check container health +docker ps --format "table {{.Names}}\t{{.Status}}" + +# View resource usage +docker stats --no-stream + +# Check disk usage +docker system df +``` + +## Step 14: Set Up Backups + +Create a backup script: + +```bash +# Create a backup directory +mkdir -p ~/backups + +# Create a simple backup script +cat > ~/backup-homelab.sh << 'EOF' +#!/bin/bash +BACKUP_DIR=~/backups +DATE=$(date +%Y%m%d) + +# Backup config directories +tar czf $BACKUP_DIR/config-$DATE.tar.gz ~/AI-Homelab/config/ + +# Backup .env file +cp ~/AI-Homelab/.env $BACKUP_DIR/.env-$DATE + +# Backup Docker volumes (example for Portainer) +docker run --rm \ + -v portainer-data:/data \ + -v $BACKUP_DIR:/backup \ + busybox tar czf /backup/portainer-data-$DATE.tar.gz /data + +echo "Backup completed: $DATE" +EOF + +# Make it executable +chmod +x ~/backup-homelab.sh + +# Test the backup +~/backup-homelab.sh +``` + +Set up a cron job for automated backups: + +```bash +# Open crontab +crontab -e + +# Add this line to run backup daily at 2 AM +0 2 * * * /home/yourusername/backup-homelab.sh +``` + +## Step 15: Configure Firewall (Optional but Recommended) + +If using UFW: + +```bash +# Allow SSH (if not already allowed) +sudo ufw allow 22/tcp + +# Allow web traffic (if exposing services to internet) +sudo ufw allow 80/tcp +sudo ufw allow 443/tcp + +# Allow specific services from local network only +# Replace 192.168.1.0/24 with your network +sudo ufw allow from 192.168.1.0/24 to any port 9000 proto tcp # Portainer +sudo ufw allow from 192.168.1.0/24 to any port 81 proto tcp # Nginx Proxy Manager +sudo ufw allow from 192.168.1.0/24 to any port 3000 proto tcp # Grafana + +# Enable firewall +sudo ufw enable +``` + +## Next Steps + +Now that your homelab is running: + +1. **Explore Services:** + - Access each service's web UI + - Configure settings as needed + - Set up integrations between services + +2. **Add More Services:** + - Ask GitHub Copilot for help adding new services + - Follow the patterns in existing compose files + - Check [awesome-selfhosted](https://github.com/awesome-selfhosted/awesome-selfhosted) for ideas + +3. **Optimize:** + - Review logs for errors + - Adjust resource limits + - Set up proper monitoring and alerts + +4. **Secure:** + - Change all default passwords + - Set up SSL certificates (use Nginx Proxy Manager) + - Enable 2FA where available + - Keep services updated + +5. **Learn:** + - Read the [Docker Guidelines](docker-guidelines.md) + - Experiment with new services + - Use AI assistance to understand and modify configurations + +## Troubleshooting + +### Can't access services + +1. **Check if service is running:** + ```bash + docker ps + ``` + +2. **Check service logs:** + ```bash + docker compose -f docker-compose/file.yml logs service-name + ``` + +3. **Verify network connectivity:** + ```bash + ping YOUR_SERVER_IP + ``` + +4. **Check firewall:** + ```bash + sudo ufw status + ``` + +### Permission errors + +```bash +# Fix config directory permissions +sudo chown -R $(id -u):$(id -g) config/ + +# Verify PUID/PGID in .env match your user +id -u # Should match PUID in .env +id -g # Should match PGID in .env +``` + +### Port already in use + +```bash +# Find what's using the port +sudo netstat -tlnp | grep PORT_NUMBER + +# Stop the conflicting service or change the port in docker-compose +``` + +### Out of disk space + +```bash +# Check disk usage +df -h + +# Clean up Docker resources +docker system prune -a + +# Remove old logs +docker compose logs --tail=0 service-name +``` + +## Getting Help + +- **Documentation:** Check the `docs/` directory for comprehensive guides +- **AI Assistance:** Use GitHub Copilot in VS Code for real-time help +- **Community:** Search for service-specific help in respective communities +- **Issues:** Open an issue on GitHub for problems with this repository + +## Success Checklist + +- [ ] Docker and Docker Compose installed +- [ ] Repository cloned +- [ ] `.env` file configured +- [ ] Networks created +- [ ] Config directories created +- [ ] Portainer running and accessible +- [ ] Infrastructure services deployed +- [ ] At least one service category deployed (media/monitoring/dev) +- [ ] VS Code with GitHub Copilot set up +- [ ] Backup strategy in place +- [ ] Firewall configured (if applicable) +- [ ] All services accessible and working + +Congratulations! Your AI-powered homelab is now running! 🎉 From 65a77d113849f854b0300b49571d8c094568df22 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 22:56:13 +0000 Subject: [PATCH 05/16] Address code review feedback - fix GPU config and documentation consistency Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- docker-compose/development.yml | 8 ++++++-- docker-compose/media.yml | 11 +++++++---- docs/docker-guidelines.md | 2 +- docs/quick-reference.md | 2 +- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/docker-compose/development.yml b/docker-compose/development.yml index 75a5110..c5d6b30 100644 --- a/docker-compose/development.yml +++ b/docker-compose/development.yml @@ -140,10 +140,14 @@ services: - GRANT_SUDO=yes user: root command: start-notebook.sh --NotebookApp.token='${JUPYTER_TOKEN:-changeme}' - # Uncomment for GPU support (NVIDIA) + # Uncomment for GPU support (NVIDIA, requires nvidia-container-toolkit) # runtime: nvidia - # environment: + # devices: + # - /dev/nvidia0:/dev/nvidia0 + # - /dev/nvidiactl:/dev/nvidiactl + # Add these to environment above: # - NVIDIA_VISIBLE_DEVICES=all + # - NVIDIA_DRIVER_CAPABILITIES=compute,utility labels: - "homelab.category=development" - "homelab.description=Jupyter Lab for data science and ML" diff --git a/docker-compose/media.yml b/docker-compose/media.yml index 16389e8..fe16afc 100644 --- a/docker-compose/media.yml +++ b/docker-compose/media.yml @@ -23,18 +23,21 @@ services: - TZ=${TZ:-America/New_York} - PLEX_CLAIM=${PLEX_CLAIM} - ADVERTISE_IP=http://${SERVER_IP}:32400/ - # Uncomment for hardware transcoding (Intel QuickSync) + # Hardware transcoding support + # Uncomment ONE of the following options: + + # Option 1: Intel QuickSync (most common) # devices: # - /dev/dri:/dev/dri - # Uncomment for NVIDIA GPU transcoding + + # Option 2: NVIDIA GPU (requires nvidia-container-toolkit installed) + # runtime: nvidia # devices: # - /dev/nvidia0:/dev/nvidia0 # - /dev/nvidiactl:/dev/nvidiactl # - /dev/nvidia-modeset:/dev/nvidia-modeset # - /dev/nvidia-uvm:/dev/nvidia-uvm # - /dev/nvidia-uvm-tools:/dev/nvidia-uvm-tools - # Uncomment for NVIDIA GPU support (requires nvidia-container-toolkit) - # runtime: nvidia # environment: # - NVIDIA_VISIBLE_DEVICES=all # - NVIDIA_DRIVER_CAPABILITIES=compute,video,utility diff --git a/docs/docker-guidelines.md b/docs/docker-guidelines.md index 2d7e681..8d069ca 100644 --- a/docs/docker-guidelines.md +++ b/docs/docker-guidelines.md @@ -53,7 +53,7 @@ Use Docker Compose for: # docker-compose/plex.yml services: plex: - image: plexinc/pms-docker:1.40.0.7775-456fbaf97 + image: plexinc/pms-docker:1.40.0.7998-f68041501 container_name: plex restart: unless-stopped networks: diff --git a/docs/quick-reference.md b/docs/quick-reference.md index a77b1b1..afb8400 100644 --- a/docs/quick-reference.md +++ b/docs/quick-reference.md @@ -350,7 +350,7 @@ deploy: ### Minimal setup ```bash -# 1. Clone and configure +# 1. Clone and configure (replace with your repo URL if forked) git clone https://github.com/kelinfoxy/AI-Homelab.git cd AI-Homelab cp .env.example .env From 9b466145c3c884950c78b878109440fed21ce11b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 22:57:40 +0000 Subject: [PATCH 06/16] Clarify repository URL for forked versions Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- README.md | 1 + docs/getting-started.md | 2 ++ docs/quick-reference.md | 3 ++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 46852ea..fd60268 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ This repository provides a structured approach to managing a homelab infrastruct 1. **Clone the repository:** ```bash + # Note: Replace 'kelinfoxy' with your username if you forked this repository git clone https://github.com/kelinfoxy/AI-Homelab.git cd AI-Homelab ``` diff --git a/docs/getting-started.md b/docs/getting-started.md index 36522b4..0b0f884 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -38,6 +38,7 @@ docker run --rm hello-world cd ~ # Clone the repository +# Note: Replace 'kelinfoxy' with your username if you forked this repository git clone https://github.com/kelinfoxy/AI-Homelab.git # Enter the directory @@ -297,6 +298,7 @@ docker compose -f docker-compose/development.yml ps 3. **Clone your repository in VS Code:** ```bash # On your local machine + # Note: Replace 'kelinfoxy' with your username if you forked this repository git clone https://github.com/kelinfoxy/AI-Homelab.git cd AI-Homelab code . diff --git a/docs/quick-reference.md b/docs/quick-reference.md index afb8400..930410f 100644 --- a/docs/quick-reference.md +++ b/docs/quick-reference.md @@ -350,7 +350,8 @@ deploy: ### Minimal setup ```bash -# 1. Clone and configure (replace with your repo URL if forked) +# 1. Clone and configure +# Note: Replace 'kelinfoxy' with your username if you forked this repository git clone https://github.com/kelinfoxy/AI-Homelab.git cd AI-Homelab cp .env.example .env From 0611ddbff40ee8e98ebbc190ce923c5d057ff7a9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 22:59:06 +0000 Subject: [PATCH 07/16] Fix Docker command examples and log management instructions Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- docs/getting-started.md | 10 +++++++--- docs/quick-reference.md | 7 ++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 0b0f884..e9c18ca 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -476,11 +476,15 @@ sudo netstat -tlnp | grep PORT_NUMBER # Check disk usage df -h -# Clean up Docker resources +# Clean up Docker resources (removes unused images, containers, networks) docker system prune -a -# Remove old logs -docker compose logs --tail=0 service-name +# To manage log size, configure log rotation in compose files: +# logging: +# driver: "json-file" +# options: +# max-size: "10m" +# max-file: "3" ``` ## Getting Help diff --git a/docs/quick-reference.md b/docs/quick-reference.md index 930410f..0e8fffe 100644 --- a/docs/quick-reference.md +++ b/docs/quick-reference.md @@ -381,7 +381,12 @@ docker compose -f docker-compose/development.yml up -d - Watchtower checks for updates at 4 AM ### Weekly -- Review logs: `docker compose -f docker-compose/*.yml logs --tail=100` +- Review logs for each stack: + ```bash + docker compose -f docker-compose/infrastructure.yml logs --tail=100 + docker compose -f docker-compose/media.yml logs --tail=100 + docker compose -f docker-compose/monitoring.yml logs --tail=100 + ``` - Check disk space: `docker system df` ### Monthly From 6083da70368c9eec69c517cc7dd4441c2c5d8e69 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 23:01:15 +0000 Subject: [PATCH 08/16] Improve command examples in quick reference guide Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- docs/quick-reference.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/quick-reference.md b/docs/quick-reference.md index 0e8fffe..2215f99 100644 --- a/docs/quick-reference.md +++ b/docs/quick-reference.md @@ -2,20 +2,26 @@ ## Common Commands +**Note**: Replace `infrastructure.yml` with your compose file name: +- `infrastructure.yml` - Core services +- `media.yml` - Media stack +- `monitoring.yml` - Monitoring services +- `development.yml` - Development tools + ### Service Management ```bash -# Start all services in a file -docker compose -f docker-compose/file.yml up -d +# Start all services in a compose file +docker compose -f docker-compose/infrastructure.yml up -d # Start specific service -docker compose -f docker-compose/file.yml up -d service-name +docker compose -f docker-compose/infrastructure.yml up -d service-name # Stop all services -docker compose -f docker-compose/file.yml down +docker compose -f docker-compose/infrastructure.yml down # Stop specific service -docker compose -f docker-compose/file.yml stop service-name +docker compose -f docker-compose/infrastructure.yml stop service-name # Restart service docker compose -f docker-compose/file.yml restart service-name From f9a34fe9c773df0758ed9bf5c0902bdccc8ea1a4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 12 Jan 2026 00:13:55 +0000 Subject: [PATCH 09/16] Implement Dockge structure with Traefik, Authelia, DuckDNS, and Gluetun VPN - Update AI copilot instructions for /opt/stacks structure and automated config management - Replace Nginx Proxy Manager with Traefik (file-based configuration for AI) - Add Authelia for SSO with bypass rules for Jellyfin/Plex apps - Add DuckDNS for dynamic DNS with Let's Encrypt integration - Add Gluetun VPN with Surfshark (WireGuard) for secure downloads - Update all services to use /opt/stacks paths instead of local directories - Add Traefik labels to all services for automatic routing - Configure qBittorrent to route through Gluetun VPN - Update .env.example with all new required variables - Create configuration templates for Traefik and Authelia - Add comprehensive Dockge deployment guide Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- .env.example | 38 ++- .github/copilot-instructions.md | 255 +++++++++++++-- config-templates/authelia/configuration.yml | 93 ++++++ config-templates/authelia/users_database.yml | 20 ++ config-templates/traefik/dynamic/routes.yml | 31 ++ config-templates/traefik/traefik.yml | 58 ++++ docker-compose/README-dockge.md | 324 +++++++++++++++++++ docker-compose/authelia.yml | 39 +++ docker-compose/duckdns.yml | 21 ++ docker-compose/gluetun.yml | 81 +++++ docker-compose/infrastructure.yml | 74 +++-- docker-compose/media.yml | 117 ++++--- docker-compose/traefik.yml | 43 +++ 13 files changed, 1082 insertions(+), 112 deletions(-) create mode 100644 config-templates/authelia/configuration.yml create mode 100644 config-templates/authelia/users_database.yml create mode 100644 config-templates/traefik/dynamic/routes.yml create mode 100644 config-templates/traefik/traefik.yml create mode 100644 docker-compose/README-dockge.md create mode 100644 docker-compose/authelia.yml create mode 100644 docker-compose/duckdns.yml create mode 100644 docker-compose/gluetun.yml create mode 100644 docker-compose/traefik.yml diff --git a/.env.example b/.env.example index ee0642c..9f171bb 100644 --- a/.env.example +++ b/.env.example @@ -12,12 +12,41 @@ TZ=America/New_York # Server IP address SERVER_IP=192.168.1.100 +# Domain Configuration +DOMAIN=yourdomain.duckdns.org # Your DuckDNS domain + # Directory Paths -USERDIR=/home/username/homelab -MEDIADIR=/mnt/media -DOWNLOADDIR=/mnt/downloads +USERDIR=/opt/stacks +MEDIADIR=/mnt/media # Large media files on separate drive +DOWNLOADDIR=/mnt/downloads # Downloads on separate drive PROJECTDIR=/home/username/projects +# DuckDNS Configuration +DUCKDNS_TOKEN=your-duckdns-token +DUCKDNS_SUBDOMAINS=yourdomain # Without .duckdns.org + +# Let's Encrypt / ACME +ACME_EMAIL=your-email@example.com + +# Authelia Secrets (generate with: openssl rand -hex 64) +AUTHELIA_JWT_SECRET=your-jwt-secret-here-64-chars +AUTHELIA_SESSION_SECRET=your-session-secret-here-64-chars +AUTHELIA_STORAGE_ENCRYPTION_KEY=your-encryption-key-here-64-chars + +# SMTP for Authelia Notifications (optional) +SMTP_USERNAME=your-email@example.com +SMTP_PASSWORD=your-smtp-password + +# VPN Configuration (Surfshark) +# Get WireGuard details from Surfshark dashboard +SURFSHARK_PRIVATE_KEY=your-wireguard-private-key +SURFSHARK_ADDRESSES=10.14.0.2/16 +VPN_COUNTRY=Netherlands # Preferred VPN server location + +# Alternative: OpenVPN credentials (if not using WireGuard) +# SURFSHARK_USERNAME=your-surfshark-username +# SURFSHARK_PASSWORD=your-surfshark-password + # Plex Configuration PLEX_CLAIM=claim-xxxxxxxxxx @@ -45,4 +74,7 @@ PIHOLE_PASSWORD=changeme # Watchtower Notifications (optional) # WATCHTOWER_NOTIFICATION_URL= +# Cloudflare API (optional, for DNS challenge) +# CF_DNS_API_TOKEN=your-cloudflare-api-token + # Add your own variables below diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 1186bc5..5712a42 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,26 +1,45 @@ # AI Homelab Management Assistant -You are an AI assistant specialized in managing Docker-based homelab infrastructure. Your role is to help users create, modify, and manage Docker services while maintaining consistency across the entire server stack. +You are an AI assistant specialized in managing Docker-based homelab infrastructure using Dockge. Your role is to help users create, modify, and manage Docker services while maintaining consistency across the entire server stack. ## Core Principles -### 1. Docker Compose First +### 1. Dockge and Docker Compose First - **ALWAYS** use Docker Compose stacks for persistent services +- Store all compose files in `/opt/stacks/stack-name/` directories - Only use `docker run` for temporary containers (e.g., testing nvidia-container-toolkit functionality) -- Maintain all services in organized docker-compose.yml files +- Maintain all services in organized docker-compose.yml files within their stack folders -### 2. Consistency is Key +### 2. File Structure and Storage +- **Base Path**: All stacks are stored in `/opt/stacks/` +- **Bind Mounts**: Default to `/opt/stacks/stack-name/` for configuration files +- **Large Data**: Suggest using separate mounted drives for: + - Media files (movies, TV shows, music) - typically `/mnt/media` + - Downloads - typically `/mnt/downloads` + - Database data files that grow large + - Backup storage + - Any data that may exceed 50GB or grow continuously +- **Named Volumes**: Use Docker named volumes for smaller application data + +### 3. Consistency is Key - Keep consistent naming conventions across all compose files - Use the same network naming patterns - Maintain uniform volume mount structures - Apply consistent environment variable patterns -### 3. Stack-Aware Changes +### 4. Stack-Aware Changes - Before making changes, consider the impact on the entire server stack - Check for service dependencies (networks, volumes, other services) - Ensure changes don't break existing integrations - Validate that port assignments don't conflict +### 5. Automated Configuration Management +- Configure all services via configuration files, not web UIs +- Traefik routes configured via Docker labels +- Authelia rules configured via YAML files +- Enable AI to manage and update configurations automatically +- Maintain homelab functionality through code, not manual UI clicks + ## Creating a New Docker Service When creating a new service, follow these steps: @@ -46,10 +65,12 @@ When creating a new service, follow these steps: networks: - homelab-network # Use shared networks ports: - - "host_port:container_port" # Document port purpose + - "host_port:container_port" # Document port purpose (if not using Traefik) volumes: - - ./config/service-name:/config # Config in local directory + - /opt/stacks/stack-name/config:/config # Config in stack directory - service-data:/data # Named volumes for persistent data + # For large data, use separate mount: + # - /mnt/media:/media # Large media files on separate drive environment: - PUID=1000 # Standard user/group IDs - PGID=1000 @@ -57,6 +78,13 @@ When creating a new service, follow these steps: labels: - "homelab.category=category-name" # For organization - "homelab.description=Service description" + # Traefik labels (if using Traefik): + # - "traefik.enable=true" + # - "traefik.http.routers.service-name.rule=Host(`service.domain.com`)" + # - "traefik.http.routers.service-name.entrypoints=websecure" + # - "traefik.http.routers.service-name.tls.certresolver=letsencrypt" + # Authelia middleware (if SSO required): + # - "traefik.http.routers.service-name.middlewares=authelia@docker" volumes: service-data: @@ -213,41 +241,220 @@ environment: ## File Organization ``` -/home/user/homelab/ -├── docker-compose/ -│ ├── media.yml # Media server services -│ ├── monitoring.yml # Monitoring stack -│ ├── development.yml # Dev tools -│ └── infrastructure.yml # Core services -├── config/ -│ ├── service1/ -│ ├── service2/ -│ └── ... -├── data/ # Bind mount data -│ └── ... -├── .env # Global secrets (gitignored) -└── README.md # Stack documentation +/opt/stacks/ +├── stack-name/ +│ ├── docker-compose.yml # Stack definition +│ ├── config/ # Service configurations +│ ├── .env # Stack-specific secrets +│ └── README.md # Stack documentation +├── traefik/ +│ ├── docker-compose.yml +│ ├── traefik.yml # Traefik static config +│ ├── dynamic/ # Dynamic configuration +│ │ └── routes.yml # Route definitions +│ ├── acme.json # Let's Encrypt certificates +│ └── .env +├── authelia/ +│ ├── docker-compose.yml +│ ├── configuration.yml # Authelia config +│ ├── users_database.yml # User definitions +│ └── .env +├── gluetun/ +│ ├── docker-compose.yml +│ └── .env # VPN credentials +└── duckdns/ + ├── docker-compose.yml + └── .env # DuckDNS token ``` +## VPN Integration with Gluetun + +### When to Use VPN +- Download clients (qBittorrent, SABnzbd, etc.) +- Services that need to hide their origin IP +- Services accessing geo-restricted content + +### Gluetun Configuration +- **Default VPN**: Surfshark +- Services connect through Gluetun's network namespace +- Use `network_mode: "service:gluetun"` for VPN routing +- Access via Gluetun's ports: map ports in Gluetun service + +**Example:** +```yaml +services: + gluetun: + image: qmcgaw/gluetun:latest + container_name: gluetun + cap_add: + - NET_ADMIN + environment: + - VPN_SERVICE_PROVIDER=surfshark + - VPN_TYPE=wireguard + - WIREGUARD_PRIVATE_KEY=${SURFSHARK_PRIVATE_KEY} + - WIREGUARD_ADDRESSES=${SURFSHARK_ADDRESSES} + - SERVER_COUNTRIES=Netherlands + ports: + - 8080:8080 # qBittorrent web UI + - 6881:6881 # qBittorrent ports + + qbittorrent: + image: lscr.io/linuxserver/qbittorrent:latest + container_name: qbittorrent + network_mode: "service:gluetun" # Route through VPN + depends_on: + - gluetun + volumes: + - /opt/stacks/qbittorrent/config:/config + - /mnt/downloads:/downloads +``` + +## SSO with Authelia + +### Authentication Strategy +- **Protected Services**: Most web UIs (require SSO login) +- **Bypass Services**: Apps that need direct access (Jellyfin, Plex, mobile apps) +- **API Endpoints**: Configure bypass rules for API access + +### Authelia Configuration +- Users defined in `users_database.yml` +- Access rules in `configuration.yml` +- Integrate with Traefik via middleware + +### Services Requiring Authelia +- Monitoring dashboards (Grafana, Prometheus, etc.) +- Admin panels (Portainer, etc.) +- Download clients web UIs +- Development tools +- Any service with sensitive data + +### Services Bypassing Authelia +- Jellyfin (for app access - Roku, Fire TV, mobile apps) +- Plex (for app access) +- Home Assistant (has its own auth) +- Services with API-only access +- Public-facing services (if any) + +**Example Traefik Labels with Authelia:** +```yaml +labels: + - "traefik.enable=true" + - "traefik.http.routers.sonarr.rule=Host(`sonarr.${DOMAIN}`)" + - "traefik.http.routers.sonarr.entrypoints=websecure" + - "traefik.http.routers.sonarr.tls.certresolver=letsencrypt" + - "traefik.http.routers.sonarr.middlewares=authelia@docker" # SSO enabled +``` + +**Example Bypassing Authelia (Jellyfin):** +```yaml +labels: + - "traefik.enable=true" + - "traefik.http.routers.jellyfin.rule=Host(`jellyfin.${DOMAIN}`)" + - "traefik.http.routers.jellyfin.entrypoints=websecure" + - "traefik.http.routers.jellyfin.tls.certresolver=letsencrypt" + # No authelia middleware - direct access for apps +``` + +## Traefik Reverse Proxy + +### Why Traefik Instead of Nginx Proxy Manager +- **File-based configuration**: AI can modify YAML files +- **Docker label integration**: Automatic service discovery +- **No web UI dependency**: Fully automated management +- **Let's Encrypt automation**: Automatic SSL certificate management +- **Dynamic configuration**: Changes without restarts + +### Traefik Configuration Pattern +1. **Static config** (`traefik.yml`): Core settings, entry points, certificate resolvers +2. **Dynamic config** (Docker labels): Per-service routing rules +3. **File provider**: Additional route definitions in `dynamic/` directory + +### Managing Routes via AI +- Traefik routes defined in Docker labels +- AI can read compose files and add/modify labels +- Automatic service discovery when containers start +- Update routes by modifying compose files and redeploying + +**Example Service with Traefik:** +```yaml +services: + service-name: + image: service:latest + container_name: service-name + networks: + - traefik-network + labels: + - "traefik.enable=true" + - "traefik.http.routers.service-name.rule=Host(`service.${DOMAIN}`)" + - "traefik.http.routers.service-name.entrypoints=websecure" + - "traefik.http.routers.service-name.tls.certresolver=letsencrypt" + - "traefik.http.routers.service-name.middlewares=authelia@docker" + - "traefik.http.services.service-name.loadbalancer.server.port=8080" +``` + +## DuckDNS for Dynamic DNS + +### Purpose +- Provides dynamic DNS for home IP addresses +- Integrates with Let's Encrypt for SSL certificates +- Updates automatically when IP changes + +### Configuration +- Single container updates your domain periodically +- Works with Traefik's Let's Encrypt resolver +- Set up once and forget + +## Automated Homelab Management + +### AI's Role in Maintenance +1. **Service Addition**: Create compose files with proper Traefik labels +2. **Route Management**: Update labels to modify proxy routes +3. **SSL Certificates**: Traefik handles automatically via Let's Encrypt +4. **SSO Configuration**: Add/remove authelia middleware as needed +5. **VPN Routing**: Configure services to use Gluetun when required +6. **Monitoring**: Ensure all services are properly configured + +### Configuration Files AI Can Manage +- `docker-compose.yml` files for all stacks +- `traefik/dynamic/routes.yml` for custom routes +- `authelia/configuration.yml` for access rules +- Environment variables in `.env` files +- Service-specific config files in `/opt/stacks/stack-name/config/` + +### What AI Should Monitor +- Port conflicts +- Network connectivity +- Certificate expiration (Traefik handles renewal) +- Service health +- VPN connection status +- Authentication bypass requirements + ## Safety Checks Before deploying any changes: - [ ] YAML syntax is valid - [ ] Ports don't conflict with existing services - [ ] Networks exist or are defined -- [ ] Volume paths are correct +- [ ] Volume paths are correct (use /opt/stacks/ or /mnt/ for large data) - [ ] Environment variables are set - [ ] No secrets in compose files - [ ] Service dependencies are met - [ ] Backup of current configuration exists +- [ ] Traefik labels are correct for routing +- [ ] Authelia middleware applied appropriately +- [ ] VPN routing configured if needed ## Remember - **Think before you act**: Consider the entire stack - **Be consistent**: Follow established patterns +- **Use /opt/stacks/**: All compose files go in stack directories +- **Large data on /mnt/**: Media and downloads go on separate drives +- **Configure via files**: Traefik labels, Authelia YAML, not web UIs - **Document everything**: Future you will thank you - **Test safely**: Use temporary containers first - **Back up first**: Always have a rollback plan -- **Security matters**: Keep secrets secret, update regularly +- **Security matters**: Use Authelia SSO, keep secrets in .env files +- **VPN when needed**: Route download clients through Gluetun -When a user asks you to create or modify a Docker service, follow these guidelines carefully, ask clarifying questions if needed, and always prioritize the stability and consistency of the entire homelab infrastructure. +When a user asks you to create or modify a Docker service, follow these guidelines carefully, ask clarifying questions if needed, and always prioritize the stability, security, and consistency of the entire homelab infrastructure. diff --git a/config-templates/authelia/configuration.yml b/config-templates/authelia/configuration.yml new file mode 100644 index 0000000..6dc44b3 --- /dev/null +++ b/config-templates/authelia/configuration.yml @@ -0,0 +1,93 @@ +# Authelia Configuration +# Copy to /opt/stacks/authelia/configuration.yml + +server: + host: 0.0.0.0 + port: 9091 + +log: + level: info + +theme: dark + +jwt_secret: ${AUTHELIA_JWT_SECRET} + +default_redirection_url: https://auth.${DOMAIN} + +totp: + issuer: ${DOMAIN} + period: 30 + skew: 1 + +authentication_backend: + file: + path: /config/users_database.yml + password: + algorithm: argon2id + iterations: 1 + key_length: 32 + salt_length: 16 + memory: 1024 + parallelism: 8 + +access_control: + default_policy: deny + + rules: + # Bypass Authelia for Jellyfin (allow app access) + - domain: jellyfin.${DOMAIN} + policy: bypass + + # Bypass for Plex (allow app access) + - domain: plex.${DOMAIN} + policy: bypass + + # Bypass for Home Assistant (has its own auth) + - domain: ha.${DOMAIN} + policy: bypass + + # Protected: All other services require authentication + - domain: "*.${DOMAIN}" + policy: one_factor + + # Two-factor for admin services (optional) + # - domain: + # - "admin.${DOMAIN}" + # - "portainer.${DOMAIN}" + # policy: two_factor + +session: + name: authelia_session + secret: ${AUTHELIA_SESSION_SECRET} + expiration: 1h + inactivity: 5m + remember_me_duration: 1M + domain: ${DOMAIN} + +regulation: + max_retries: 3 + find_time: 2m + ban_time: 5m + +storage: + encryption_key: ${AUTHELIA_STORAGE_ENCRYPTION_KEY} + local: + path: /config/db.sqlite3 + +notifier: + # File-based notifications (for testing) + filesystem: + filename: /config/notification.txt + + # SMTP notifications (recommended for production) + # smtp: + # host: smtp.gmail.com + # port: 587 + # username: ${SMTP_USERNAME} + # password: ${AUTHELIA_NOTIFIER_SMTP_PASSWORD} + # sender: authelia@${DOMAIN} + # identifier: localhost + # subject: "[Authelia] {title}" + # startup_check_address: test@authelia.com + # disable_require_tls: false + # disable_html_emails: false diff --git a/config-templates/authelia/users_database.yml b/config-templates/authelia/users_database.yml new file mode 100644 index 0000000..dec2f75 --- /dev/null +++ b/config-templates/authelia/users_database.yml @@ -0,0 +1,20 @@ +# Authelia Users Database +# Copy to /opt/stacks/authelia/users_database.yml +# Generate password hashes with: docker run authelia/authelia:latest authelia crypto hash generate argon2 --password 'yourpassword' + +users: + admin: + displayname: "Admin User" + password: "$argon2id$v=19$m=65536,t=3,p=4$CHANGEME" # Replace with your hashed password + email: admin@example.com + groups: + - admins + - users + + # Example: Additional user + # user1: + # displayname: "User One" + # password: "$argon2id$v=19$m=65536,t=3,p=4$CHANGEME" + # email: user1@example.com + # groups: + # - users diff --git a/config-templates/traefik/dynamic/routes.yml b/config-templates/traefik/dynamic/routes.yml new file mode 100644 index 0000000..cdaf10e --- /dev/null +++ b/config-templates/traefik/dynamic/routes.yml @@ -0,0 +1,31 @@ +# Traefik Dynamic Configuration +# Copy to /opt/stacks/traefik/dynamic/routes.yml +# Add custom routes here that aren't defined via Docker labels + +http: + routers: + # Example custom route + # custom-service: + # rule: "Host(`custom.example.com`)" + # entryPoints: + # - websecure + # middlewares: + # - authelia@docker + # tls: + # certResolver: letsencrypt + # service: custom-service + + services: + # Example custom service + # custom-service: + # loadBalancer: + # servers: + # - url: "http://192.168.1.100:8080" + + middlewares: + # Additional middlewares can be defined here + # Example: Rate limiting + # rate-limit: + # rateLimit: + # average: 100 + # burst: 50 diff --git a/config-templates/traefik/traefik.yml b/config-templates/traefik/traefik.yml new file mode 100644 index 0000000..60529fb --- /dev/null +++ b/config-templates/traefik/traefik.yml @@ -0,0 +1,58 @@ +# Traefik Static Configuration +# Copy to /opt/stacks/traefik/traefik.yml + +global: + checkNewVersion: true + sendAnonymousUsage: false + +api: + dashboard: true + insecure: false # Dashboard accessible via Traefik route with Authelia + +entryPoints: + web: + address: ":80" + http: + redirections: + entryPoint: + to: websecure + scheme: https + + websecure: + address: ":443" + http: + tls: + certResolver: letsencrypt + +certificatesResolvers: + letsencrypt: + acme: + email: ${ACME_EMAIL} + storage: /acme.json + # Use HTTP challenge (port 80 must be accessible) + httpChallenge: + entryPoint: web + # Or use DNS challenge (requires API token): + # dnsChallenge: + # provider: duckdns + # resolvers: + # - "1.1.1.1:53" + # - "8.8.8.8:53" + +providers: + docker: + endpoint: "unix:///var/run/docker.sock" + exposedByDefault: false # Only expose services with traefik.enable=true + network: traefik-network + + file: + directory: /dynamic + watch: true + +log: + level: INFO # DEBUG, INFO, WARN, ERROR + filePath: /var/log/traefik/traefik.log + +accessLog: + filePath: /var/log/traefik/access.log + bufferingSize: 100 diff --git a/docker-compose/README-dockge.md b/docker-compose/README-dockge.md new file mode 100644 index 0000000..ae00ec6 --- /dev/null +++ b/docker-compose/README-dockge.md @@ -0,0 +1,324 @@ +# Docker Compose Stacks - Dockge Structure + +This directory contains Docker Compose files designed for use with Dockge. Each stack should be placed in `/opt/stacks/stack-name/` on your server. + +## Structure + +``` +/opt/stacks/ +├── traefik/ +│ ├── docker-compose.yml # Copy from traefik.yml +│ ├── traefik.yml # Static configuration +│ ├── dynamic/ # Dynamic routes +│ ├── acme.json # SSL certificates (chmod 600) +│ └── .env +├── authelia/ +│ ├── docker-compose.yml # Copy from authelia.yml +│ ├── configuration.yml # Authelia config +│ ├── users_database.yml # User definitions +│ └── .env +├── duckdns/ +│ ├── docker-compose.yml # Copy from duckdns.yml +│ └── .env +├── gluetun/ +│ ├── docker-compose.yml # Copy from gluetun.yml (includes qBittorrent) +│ └── .env +├── infrastructure/ +│ ├── docker-compose.yml # Copy from infrastructure.yml +│ └── .env +├── media/ +│ ├── docker-compose.yml # Copy from media.yml +│ └── .env +├── monitoring/ +│ ├── docker-compose.yml # Copy from monitoring.yml +│ └── .env +└── development/ + ├── docker-compose.yml # Copy from development.yml + └── .env +``` + +## Core Infrastructure Stacks + +### 1. Traefik (REQUIRED - Deploy First) +**File**: `traefik.yml` +**Location**: `/opt/stacks/traefik/` + +Reverse proxy with automatic SSL certificates via Let's Encrypt. + +**Features**: +- Automatic HTTPS with Let's Encrypt +- Docker service discovery via labels +- File-based configuration for AI management +- HTTP to HTTPS redirect + +**Setup**: +```bash +mkdir -p /opt/stacks/traefik/dynamic +cd /opt/stacks/traefik +# Copy traefik.yml to docker-compose.yml +# Copy config templates from config-templates/traefik/ +# Create acme.json and set permissions +touch acme.json && chmod 600 acme.json +# Edit .env with your domain and email +docker compose up -d +``` + +### 2. Authelia (REQUIRED - Deploy Second) +**File**: `authelia.yml` +**Location**: `/opt/stacks/authelia/` + +Single Sign-On (SSO) authentication for all services. + +**Features**: +- Protects services with authentication +- Bypass rules for apps (Jellyfin, Plex) +- Integrates with Traefik via middleware +- TOTP 2FA support + +**Setup**: +```bash +mkdir -p /opt/stacks/authelia +cd /opt/stacks/authelia +# Copy authelia.yml to docker-compose.yml +# Copy config templates from config-templates/authelia/ +# Generate secrets: openssl rand -hex 64 +# Edit configuration.yml and users_database.yml +# Hash password: docker run authelia/authelia:latest authelia crypto hash generate argon2 --password 'yourpassword' +docker compose up -d +``` + +### 3. DuckDNS (RECOMMENDED) +**File**: `duckdns.yml` +**Location**: `/opt/stacks/duckdns/` + +Dynamic DNS updater for your domain. + +**Setup**: +```bash +mkdir -p /opt/stacks/duckdns +cd /opt/stacks/duckdns +# Copy duckdns.yml to docker-compose.yml +# Add DUCKDNS_TOKEN and DUCKDNS_SUBDOMAINS to .env +docker compose up -d +``` + +### 4. Gluetun VPN (REQUIRED for torrenting) +**File**: `gluetun.yml` +**Location**: `/opt/stacks/gluetun/` + +VPN client (Surfshark) for routing download clients securely. + +**Includes**: qBittorrent configured to route through VPN + +**Setup**: +```bash +mkdir -p /opt/stacks/gluetun +mkdir -p /opt/stacks/qbittorrent +cd /opt/stacks/gluetun +# Copy gluetun.yml to docker-compose.yml +# Add Surfshark WireGuard credentials to .env +# Get WireGuard config from Surfshark dashboard +docker compose up -d +``` + +## Application Stacks + +### Infrastructure +**File**: `infrastructure.yml` +**Location**: `/opt/stacks/infrastructure/` + +- Pi-hole: Network-wide ad blocking +- Portainer: Docker management UI +- Watchtower: Automatic container updates + +### Media +**File**: `media.yml` +**Location**: `/opt/stacks/media/` + +- Plex: Media streaming (NO Authelia - app access) +- Jellyfin: Open-source streaming (NO Authelia - app access) +- Sonarr: TV show automation (WITH Authelia) +- Radarr: Movie automation (WITH Authelia) +- Prowlarr: Indexer manager (WITH Authelia) + +**Note**: qBittorrent is in gluetun.yml (VPN routing) + +### Monitoring +**File**: `monitoring.yml` +**Location**: `/opt/stacks/monitoring/` + +- Prometheus: Metrics collection +- Grafana: Visualization +- Node Exporter: System metrics +- cAdvisor: Container metrics +- Uptime Kuma: Service monitoring +- Loki: Log aggregation +- Promtail: Log shipping + +### Development +**File**: `development.yml` +**Location**: `/opt/stacks/development/` + +- Code Server: VS Code in browser +- GitLab: Git repository manager +- PostgreSQL: Database +- Redis: In-memory store +- pgAdmin: Database UI +- Jupyter Lab: Data science notebooks +- Node-RED: Automation + +## Networks + +Create these networks before deploying stacks: + +```bash +docker network create traefik-network +docker network create homelab-network +``` + +## Environment Variables + +Each stack needs a `.env` file. Use `/home/runner/work/AI-Homelab/AI-Homelab/.env.example` as a template. + +**Required variables**: +- `DOMAIN`: Your DuckDNS domain (e.g., `yourdomain.duckdns.org`) +- `DUCKDNS_TOKEN`: Your DuckDNS token +- `ACME_EMAIL`: Email for Let's Encrypt +- `AUTHELIA_JWT_SECRET`: Generate with `openssl rand -hex 64` +- `AUTHELIA_SESSION_SECRET`: Generate with `openssl rand -hex 64` +- `AUTHELIA_STORAGE_ENCRYPTION_KEY`: Generate with `openssl rand -hex 64` +- `SURFSHARK_PRIVATE_KEY`: From Surfshark WireGuard config +- `SURFSHARK_ADDRESSES`: From Surfshark WireGuard config + +## Deployment Order + +1. **Create networks**: + ```bash + docker network create traefik-network + docker network create homelab-network + ``` + +2. **Deploy Traefik** (reverse proxy): + ```bash + cd /opt/stacks/traefik + docker compose up -d + ``` + +3. **Deploy Authelia** (SSO): + ```bash + cd /opt/stacks/authelia + docker compose up -d + ``` + +4. **Deploy DuckDNS** (optional but recommended): + ```bash + cd /opt/stacks/duckdns + docker compose up -d + ``` + +5. **Deploy Gluetun** (VPN for downloads): + ```bash + cd /opt/stacks/gluetun + docker compose up -d + ``` + +6. **Deploy other stacks** as needed: + ```bash + cd /opt/stacks/infrastructure + docker compose up -d + + cd /opt/stacks/media + docker compose up -d + ``` + +## Accessing Services + +All services are accessible via your domain: + +- **With Authelia (SSO required)**: + - `https://traefik.yourdomain.duckdns.org` - Traefik dashboard + - `https://portainer.yourdomain.duckdns.org` - Portainer + - `https://sonarr.yourdomain.duckdns.org` - Sonarr + - `https://radarr.yourdomain.duckdns.org` - Radarr + - `https://prowlarr.yourdomain.duckdns.org` - Prowlarr + - `https://qbit.yourdomain.duckdns.org` - qBittorrent + - `https://grafana.yourdomain.duckdns.org` - Grafana + - And more... + +- **Without Authelia (direct app access)**: + - `https://plex.yourdomain.duckdns.org` - Plex + - `https://jellyfin.yourdomain.duckdns.org` - Jellyfin + +- **Authentication page**: + - `https://auth.yourdomain.duckdns.org` - Authelia login + +## AI Management + +The AI assistant (GitHub Copilot) can: + +1. **Add new services**: Creates compose files with proper Traefik labels and Authelia middleware +2. **Modify routes**: Updates Docker labels to change proxy routing +3. **Manage SSO**: Adds or removes Authelia middleware as needed +4. **Configure VPN**: Sets up services to route through Gluetun +5. **Update configurations**: Modifies config files in `/opt/stacks/*/config/` + +All configuration is file-based, allowing the AI to manage everything without web UI dependencies. + +## Storage Strategy + +- **Config files**: `/opt/stacks/stack-name/config/` (on system drive) +- **Small data**: Docker named volumes +- **Large data**: + - Media: `/mnt/media` (separate drive) + - Downloads: `/mnt/downloads` (separate drive) + - Backups: `/mnt/backups` (separate drive) + +## Troubleshooting + +### Service won't start +```bash +cd /opt/stacks/stack-name +docker compose logs -f +``` + +### Check Traefik routing +```bash +docker logs traefik +# Or visit: https://traefik.yourdomain.duckdns.org +``` + +### Test VPN connection +```bash +docker exec gluetun sh -c "curl ifconfig.me" +# Should show VPN IP, not your home IP +``` + +### Authelia issues +```bash +cd /opt/stacks/authelia +docker compose logs -f authelia +# Check configuration.yml for syntax errors +``` + +## Backup Important Files + +Regular backups of: +- `/opt/stacks/` (all compose files and configs) +- `/opt/stacks/traefik/acme.json` (SSL certificates) +- `/opt/stacks/authelia/users_database.yml` (user accounts) +- Environment files (`.env` - store securely, not in git!) + +## Security Notes + +1. **Secrets**: Never commit `.env` files or `acme.json` to git +2. **Authelia**: Use strong passwords, hash them properly +3. **VPN**: Always route download clients through Gluetun +4. **Updates**: Watchtower keeps containers updated automatically +5. **Firewall**: Only expose ports 80 and 443 to the internet + +## Getting Help + +- Check the main [README.md](../README.md) +- Review [Docker Guidelines](../docs/docker-guidelines.md) +- Use GitHub Copilot in VS Code for AI assistance +- Check service-specific logs diff --git a/docker-compose/authelia.yml b/docker-compose/authelia.yml new file mode 100644 index 0000000..0dd329c --- /dev/null +++ b/docker-compose/authelia.yml @@ -0,0 +1,39 @@ +# Authelia SSO Stack +# Single Sign-On authentication for all services +# Place in /opt/stacks/authelia/docker-compose.yml + +services: + authelia: + image: authelia/authelia:4.37 + container_name: authelia + restart: unless-stopped + networks: + - traefik-network + volumes: + - /opt/stacks/authelia/configuration.yml:/config/configuration.yml:ro + - /opt/stacks/authelia/users_database.yml:/config/users_database.yml + - authelia-data:/config + environment: + - TZ=${TZ} + - AUTHELIA_JWT_SECRET=${AUTHELIA_JWT_SECRET} + - AUTHELIA_SESSION_SECRET=${AUTHELIA_SESSION_SECRET} + - AUTHELIA_STORAGE_ENCRYPTION_KEY=${AUTHELIA_STORAGE_ENCRYPTION_KEY} + - AUTHELIA_NOTIFIER_SMTP_PASSWORD=${SMTP_PASSWORD} # If using email notifications + labels: + - "traefik.enable=true" + - "traefik.http.routers.authelia.rule=Host(`auth.${DOMAIN}`)" + - "traefik.http.routers.authelia.entrypoints=websecure" + - "traefik.http.routers.authelia.tls.certresolver=letsencrypt" + - "traefik.http.services.authelia.loadbalancer.server.port=9091" + # Authelia middleware for other services + - "traefik.http.middlewares.authelia.forwardauth.address=http://authelia:9091/api/verify?rd=https://auth.${DOMAIN}" + - "traefik.http.middlewares.authelia.forwardauth.trustForwardHeader=true" + - "traefik.http.middlewares.authelia.forwardauth.authResponseHeaders=Remote-User,Remote-Groups,Remote-Name,Remote-Email" + +volumes: + authelia-data: + driver: local + +networks: + traefik-network: + external: true diff --git a/docker-compose/duckdns.yml b/docker-compose/duckdns.yml new file mode 100644 index 0000000..16dec0a --- /dev/null +++ b/docker-compose/duckdns.yml @@ -0,0 +1,21 @@ +# DuckDNS Dynamic DNS Stack +# Updates your DuckDNS domain with current public IP +# Place in /opt/stacks/duckdns/docker-compose.yml + +services: + duckdns: + image: lscr.io/linuxserver/duckdns:latest + container_name: duckdns + restart: unless-stopped + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + - SUBDOMAINS=${DUCKDNS_SUBDOMAINS} # Your subdomain(s), comma separated + - TOKEN=${DUCKDNS_TOKEN} # Your DuckDNS token + - UPDATE_IP=ipv4 # or ipv6, or both + volumes: + - /opt/stacks/duckdns/config:/config + labels: + - "homelab.category=infrastructure" + - "homelab.description=Dynamic DNS updater" diff --git a/docker-compose/gluetun.yml b/docker-compose/gluetun.yml new file mode 100644 index 0000000..534d376 --- /dev/null +++ b/docker-compose/gluetun.yml @@ -0,0 +1,81 @@ +# Gluetun VPN Stack +# VPN client for routing services through Surfshark (or other VPN providers) +# Place in /opt/stacks/gluetun/docker-compose.yml +# Services that need VPN use: network_mode: "service:gluetun" + +services: + gluetun: + image: qmcgaw/gluetun:latest + container_name: gluetun + restart: unless-stopped + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun:/dev/net/tun + networks: + - gluetun-network + - traefik-network + ports: + # qBittorrent ports (service runs through Gluetun) + - "8080:8080" # qBittorrent WebUI + - "6881:6881" # qBittorrent TCP + - "6881:6881/udp" # qBittorrent UDP + environment: + - VPN_SERVICE_PROVIDER=surfshark + - VPN_TYPE=wireguard # or openvpn + - WIREGUARD_PRIVATE_KEY=${SURFSHARK_PRIVATE_KEY} + - WIREGUARD_ADDRESSES=${SURFSHARK_ADDRESSES} + - SERVER_COUNTRIES=${VPN_COUNTRY:-Netherlands} # Preferred VPN server country + - TZ=${TZ} + # For OpenVPN instead of WireGuard: + # - OPENVPN_USER=${SURFSHARK_USERNAME} + # - OPENVPN_PASSWORD=${SURFSHARK_PASSWORD} + volumes: + - /opt/stacks/gluetun/config:/gluetun + labels: + - "homelab.category=infrastructure" + - "homelab.description=VPN client for secure routing (Surfshark)" + + # qBittorrent - Torrent client routing through VPN + # Access at: https://qbit.yourdomain.duckdns.org + qbittorrent: + image: lscr.io/linuxserver/qbittorrent:4.6.2 + container_name: qbittorrent + network_mode: "service:gluetun" # Routes all traffic through VPN + depends_on: + - gluetun + volumes: + - /opt/stacks/qbittorrent/config:/config + - /mnt/downloads:/downloads # Large downloads on separate drive + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + - WEBUI_PORT=8080 + labels: + - "homelab.category=media" + - "homelab.description=Torrent download client (via VPN)" + # Traefik labels (applied to Gluetun since qBittorrent uses its network) + # Configure these on the Gluetun container instead: + + # Traefik routing for qBittorrent (via Gluetun) + # Since qBittorrent uses Gluetun's network, we add a sidecar label container + qbit-labels: + image: alpine:latest + container_name: qbit-labels + command: tail -f /dev/null + networks: + - traefik-network + labels: + - "traefik.enable=true" + - "traefik.http.routers.qbittorrent.rule=Host(`qbit.${DOMAIN}`)" + - "traefik.http.routers.qbittorrent.entrypoints=websecure" + - "traefik.http.routers.qbittorrent.tls.certresolver=letsencrypt" + - "traefik.http.routers.qbittorrent.middlewares=authelia@docker" + - "traefik.http.services.qbittorrent.loadbalancer.server.url=http://gluetun:8080" + +networks: + gluetun-network: + driver: bridge + traefik-network: + external: true diff --git a/docker-compose/infrastructure.yml b/docker-compose/infrastructure.yml index 6c7c617..2e628d5 100644 --- a/docker-compose/infrastructure.yml +++ b/docker-compose/infrastructure.yml @@ -1,43 +1,44 @@ # Infrastructure Services # Core services that other services depend on +# NOTE: Traefik, Authelia, DuckDNS, and Gluetun have their own compose files +# See traefik.yml, authelia.yml, duckdns.yml, and gluetun.yml services: - # Nginx Proxy Manager - Web-based reverse proxy management - # Access at: http://server-ip:81 - # Default credentials: admin@example.com / changeme - nginx-proxy-manager: - image: jc21/nginx-proxy-manager:2.10.4 - container_name: nginx-proxy-manager - restart: unless-stopped - networks: - - homelab-network - ports: - - "80:80" # HTTP - - "443:443" # HTTPS - - "81:81" # Admin UI - volumes: - - ./config/nginx-proxy-manager/data:/data - - ./config/nginx-proxy-manager/letsencrypt:/etc/letsencrypt - environment: - - PUID=${PUID:-1000} - - PGID=${PGID:-1000} - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:81"] - interval: 30s - timeout: 10s - retries: 3 - labels: - - "homelab.category=infrastructure" - - "homelab.description=Reverse proxy with Let's Encrypt support" - # Pi-hole - Network-wide ad blocker and DNS server - # Access at: http://server-ip:8080/admin + # Access at: http://server-ip:8080/admin or https://pihole.yourdomain.duckdns.org pihole: image: pihole/pihole:2024.01.0 container_name: pihole restart: unless-stopped networks: - homelab-network + - traefik-network + ports: + - "53:53/tcp" # DNS TCP + - "53:53/udp" # DNS UDP + - "8082:80/tcp" # Web interface (changed from 8080 to avoid conflicts) + volumes: + - /opt/stacks/pihole/etc-pihole:/etc/pihole + - /opt/stacks/pihole/etc-dnsmasq.d:/etc/dnsmasq.d + environment: + - TZ=${TZ:-America/New_York} + - WEBPASSWORD=${PIHOLE_PASSWORD:-changeme} + - FTLCONF_LOCAL_IPV4=${SERVER_IP} + dns: + - 127.0.0.1 + - 1.1.1.1 + cap_add: + - NET_ADMIN + labels: + - "homelab.category=infrastructure" + - "homelab.description=Network-wide ad blocking and DNS" + # Traefik labels + - "traefik.enable=true" + - "traefik.http.routers.pihole.rule=Host(`pihole.${DOMAIN}`)" + - "traefik.http.routers.pihole.entrypoints=websecure" + - "traefik.http.routers.pihole.tls.certresolver=letsencrypt" + - "traefik.http.routers.pihole.middlewares=authelia@docker" + - "traefik.http.services.pihole.loadbalancer.server.port=80" ports: - "53:53/tcp" # DNS TCP - "53:53/udp" # DNS UDP @@ -59,16 +60,14 @@ services: - "homelab.description=Network-wide ad blocking and DNS" # Portainer - Docker management UI - # Access at: http://server-ip:9000 + # Access at: https://portainer.yourdomain.duckdns.org portainer: image: portainer/portainer-ce:2.19.4 container_name: portainer restart: unless-stopped networks: - homelab-network - ports: - - "9000:9000" - - "9443:9443" + - traefik-network volumes: - /var/run/docker.sock:/var/run/docker.sock - portainer-data:/data @@ -77,6 +76,13 @@ services: labels: - "homelab.category=infrastructure" - "homelab.description=Docker container management UI" + # Traefik labels + - "traefik.enable=true" + - "traefik.http.routers.portainer.rule=Host(`portainer.${DOMAIN}`)" + - "traefik.http.routers.portainer.entrypoints=websecure" + - "traefik.http.routers.portainer.tls.certresolver=letsencrypt" + - "traefik.http.routers.portainer.middlewares=authelia@docker" + - "traefik.http.services.portainer.loadbalancer.server.port=9000" # Watchtower - Automatic container updates # Runs silently in background, no UI @@ -105,3 +111,5 @@ volumes: networks: homelab-network: external: true + traefik-network: + external: true diff --git a/docker-compose/media.yml b/docker-compose/media.yml index fe16afc..ba3f1c2 100644 --- a/docker-compose/media.yml +++ b/docker-compose/media.yml @@ -1,9 +1,12 @@ # Media Services # Services for media management and streaming +# Place in /opt/stacks/media/docker-compose.yml +# NOTE: qBittorrent is configured to use Gluetun VPN (see gluetun.yml) services: # Plex Media Server - Media streaming platform - # Access at: http://server-ip:32400/web + # Access at: https://plex.yourdomain.duckdns.org + # NOTE: No Authelia - allows app access from Roku, Fire TV, mobile, etc. plex: image: plexinc/pms-docker:1.40.0.7998-f68041501 container_name: plex @@ -11,18 +14,16 @@ services: networks: - media-network - homelab-network - ports: - - "32400:32400" # Plex web UI + - traefik-network volumes: - - ./config/plex:/config - - ${MEDIADIR:-/media}:/media:ro + - /opt/stacks/plex/config:/config + - /mnt/media:/media:ro # Large media files on separate drive - plex-transcode:/transcode environment: - PUID=${PUID:-1000} - PGID=${PGID:-1000} - TZ=${TZ:-America/New_York} - PLEX_CLAIM=${PLEX_CLAIM} - - ADVERTISE_IP=http://${SERVER_IP}:32400/ # Hardware transcoding support # Uncomment ONE of the following options: @@ -44,9 +45,16 @@ services: labels: - "homelab.category=media" - "homelab.description=Plex media streaming server" + # Traefik labels - NO Authelia for app access + - "traefik.enable=true" + - "traefik.http.routers.plex.rule=Host(`plex.${DOMAIN}`)" + - "traefik.http.routers.plex.entrypoints=websecure" + - "traefik.http.routers.plex.tls.certresolver=letsencrypt" + - "traefik.http.services.plex.loadbalancer.server.port=32400" # Jellyfin - Free alternative to Plex - # Access at: http://server-ip:8096 + # Access at: https://jellyfin.yourdomain.duckdns.org + # NOTE: No Authelia - allows app access from Roku, Fire TV, mobile, etc. jellyfin: image: jellyfin/jellyfin:10.8.13 container_name: jellyfin @@ -54,12 +62,11 @@ services: networks: - media-network - homelab-network - ports: - - "8096:8096" + - traefik-network volumes: - - ./config/jellyfin:/config - - ./config/jellyfin/cache:/cache - - ${MEDIADIR:-/media}:/media:ro + - /opt/stacks/jellyfin/config:/config + - /opt/stacks/jellyfin/cache:/cache + - /mnt/media:/media:ro # Large media files on separate drive environment: - PUID=${PUID:-1000} - PGID=${PGID:-1000} @@ -70,9 +77,15 @@ services: labels: - "homelab.category=media" - "homelab.description=Open-source media streaming server" + # Traefik labels - NO Authelia for app access + - "traefik.enable=true" + - "traefik.http.routers.jellyfin.rule=Host(`jellyfin.${DOMAIN}`)" + - "traefik.http.routers.jellyfin.entrypoints=websecure" + - "traefik.http.routers.jellyfin.tls.certresolver=letsencrypt" + - "traefik.http.services.jellyfin.loadbalancer.server.port=8096" # Sonarr - TV show automation - # Access at: http://server-ip:8989 + # Access at: https://sonarr.yourdomain.duckdns.org sonarr: image: lscr.io/linuxserver/sonarr:4.0.0 container_name: sonarr @@ -80,12 +93,11 @@ services: networks: - media-network - homelab-network - ports: - - "8989:8989" + - traefik-network volumes: - - ./config/sonarr:/config - - ${MEDIADIR:-/media}:/media - - ${DOWNLOADDIR:-/downloads}:/downloads + - /opt/stacks/sonarr/config:/config + - /mnt/media:/media + - /mnt/downloads:/downloads # Large downloads on separate drive environment: - PUID=${PUID:-1000} - PGID=${PGID:-1000} @@ -93,9 +105,16 @@ services: labels: - "homelab.category=media" - "homelab.description=TV show management and automation" + # Traefik labels with Authelia + - "traefik.enable=true" + - "traefik.http.routers.sonarr.rule=Host(`sonarr.${DOMAIN}`)" + - "traefik.http.routers.sonarr.entrypoints=websecure" + - "traefik.http.routers.sonarr.tls.certresolver=letsencrypt" + - "traefik.http.routers.sonarr.middlewares=authelia@docker" + - "traefik.http.services.sonarr.loadbalancer.server.port=8989" # Radarr - Movie automation - # Access at: http://server-ip:7878 + # Access at: https://radarr.yourdomain.duckdns.org radarr: image: lscr.io/linuxserver/radarr:5.2.6 container_name: radarr @@ -103,12 +122,11 @@ services: networks: - media-network - homelab-network - ports: - - "7878:7878" + - traefik-network volumes: - - ./config/radarr:/config - - ${MEDIADIR:-/media}:/media - - ${DOWNLOADDIR:-/downloads}:/downloads + - /opt/stacks/radarr/config:/config + - /mnt/media:/media + - /mnt/downloads:/downloads # Large downloads on separate drive environment: - PUID=${PUID:-1000} - PGID=${PGID:-1000} @@ -116,9 +134,16 @@ services: labels: - "homelab.category=media" - "homelab.description=Movie management and automation" + # Traefik labels with Authelia + - "traefik.enable=true" + - "traefik.http.routers.radarr.rule=Host(`radarr.${DOMAIN}`)" + - "traefik.http.routers.radarr.entrypoints=websecure" + - "traefik.http.routers.radarr.tls.certresolver=letsencrypt" + - "traefik.http.routers.radarr.middlewares=authelia@docker" + - "traefik.http.services.radarr.loadbalancer.server.port=7878" # Prowlarr - Indexer manager - # Access at: http://server-ip:9696 + # Access at: https://prowlarr.yourdomain.duckdns.org prowlarr: image: lscr.io/linuxserver/prowlarr:1.11.4 container_name: prowlarr @@ -126,10 +151,9 @@ services: networks: - media-network - homelab-network - ports: - - "9696:9696" + - traefik-network volumes: - - ./config/prowlarr:/config + - /opt/stacks/prowlarr/config:/config environment: - PUID=${PUID:-1000} - PGID=${PGID:-1000} @@ -137,32 +161,19 @@ services: labels: - "homelab.category=media" - "homelab.description=Indexer manager for Sonarr/Radarr" + # Traefik labels with Authelia + - "traefik.enable=true" + - "traefik.http.routers.prowlarr.rule=Host(`prowlarr.${DOMAIN}`)" + - "traefik.http.routers.prowlarr.entrypoints=websecure" + - "traefik.http.routers.prowlarr.tls.certresolver=letsencrypt" + - "traefik.http.routers.prowlarr.middlewares=authelia@docker" + - "traefik.http.services.prowlarr.loadbalancer.server.port=9696" # qBittorrent - Torrent client - # Access at: http://server-ip:8081 - # Default credentials: admin / adminadmin - qbittorrent: - image: lscr.io/linuxserver/qbittorrent:4.6.2 - container_name: qbittorrent - restart: unless-stopped - networks: - - media-network - - homelab-network - ports: - - "8081:8081" # Web UI - - "6881:6881" # Torrent port TCP - - "6881:6881/udp" # Torrent port UDP - volumes: - - ./config/qbittorrent:/config - - ${DOWNLOADDIR:-/downloads}:/downloads - environment: - - PUID=${PUID:-1000} - - PGID=${PGID:-1000} - - TZ=${TZ:-America/New_York} - - WEBUI_PORT=8081 - labels: - - "homelab.category=media" - - "homelab.description=Torrent download client" + # Access at: https://qbit.yourdomain.duckdns.org + # Routes through Gluetun VPN - configure in gluetun.yml + # NOTE: This is a placeholder. Configure qBittorrent in gluetun.yml with network_mode: "service:gluetun" + # See gluetun.yml for the actual qBittorrent configuration volumes: plex-transcode: @@ -173,3 +184,5 @@ networks: driver: bridge homelab-network: external: true + traefik-network: + external: true diff --git a/docker-compose/traefik.yml b/docker-compose/traefik.yml new file mode 100644 index 0000000..adde6b1 --- /dev/null +++ b/docker-compose/traefik.yml @@ -0,0 +1,43 @@ +# Traefik Reverse Proxy Stack +# Main reverse proxy with Let's Encrypt SSL automation +# Place in /opt/stacks/traefik/docker-compose.yml + +services: + traefik: + image: traefik:v2.11 + container_name: traefik + restart: unless-stopped + security_opt: + - no-new-privileges:true + networks: + - traefik-network + ports: + - "80:80" # HTTP + - "443:443" # HTTPS + - "8080:8080" # Dashboard (protect with Authelia) + volumes: + - /etc/localtime:/etc/localtime:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /opt/stacks/traefik/traefik.yml:/traefik.yml:ro + - /opt/stacks/traefik/dynamic:/dynamic:ro + - /opt/stacks/traefik/acme.json:/acme.json + environment: + - CF_DNS_API_TOKEN=${CF_DNS_API_TOKEN} # If using Cloudflare DNS challenge + - DUCKDNS_TOKEN=${DUCKDNS_TOKEN} # If using DuckDNS + labels: + - "traefik.enable=true" + # Dashboard + - "traefik.http.routers.traefik.rule=Host(`traefik.${DOMAIN}`)" + - "traefik.http.routers.traefik.entrypoints=websecure" + - "traefik.http.routers.traefik.tls.certresolver=letsencrypt" + - "traefik.http.routers.traefik.middlewares=authelia@docker" + - "traefik.http.routers.traefik.service=api@internal" + # Global HTTP to HTTPS redirect + - "traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`)" + - "traefik.http.routers.http-catchall.entrypoints=web" + - "traefik.http.routers.http-catchall.middlewares=redirect-to-https" + - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" + +networks: + traefik-network: + external: true From 63f39e51a4331214a6a6fb4391c9227164a8010d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 12 Jan 2026 01:00:30 +0000 Subject: [PATCH 10/16] Add comprehensive service stacks: Dockge, Homepage, Home Assistant, and all user services - Add Dockge to infrastructure (primary over Portainer) - Create dashboards.yml with Homepage and Homarr (AI-configurable) - Create homeassistant.yml with HA, ESPHome, TasmoAdmin, Node-RED, Mosquitto, Zigbee2MQTT, MotionEye - Create media-extended.yml with Readarr, Lidarr, Lazy Librarian, Mylar3, Calibre-Web, Jellyseerr, FlareSolverr, Tdarr, Unmanic - Create productivity.yml with Nextcloud, Mealie, WordPress, Gitea, DokuWiki, BookStack, MediaWiki (all with databases) - Create utilities.yml with Backrest, Duplicati, Uptime Kuma, Code Server, Form.io, Authelia Redis - Add Homepage configuration templates (services.yaml, docker.yaml, settings.yaml, widgets.yaml) - All services include container names for Docker integration - Widgets configured for services that support them (Sonarr, Radarr, Plex, Jellyfin, etc.) - Organized by category with proper layouts - Create docs/proxying-external-hosts.md - comprehensive guide for proxying Raspberry Pi and other external hosts via Traefik - Update .env.example with all new service credentials and Homepage API keys - Update infrastructure.yml to prioritize Dockge, add Dozzle, Glances, Docker Proxy - All services configured with /opt/stacks paths, Traefik labels, and appropriate Authelia middleware Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- .env.example | 96 ++++++ config-templates/homepage/docker.yaml | 13 + config-templates/homepage/services.yaml | 373 +++++++++++++++++++++++ config-templates/homepage/settings.yaml | 63 ++++ config-templates/homepage/widgets.yaml | 49 +++ docker-compose/dashboards.yml | 66 +++++ docker-compose/homeassistant.yml | 173 +++++++++++ docker-compose/infrastructure.yml | 178 ++++++++--- docker-compose/media-extended.yml | 282 ++++++++++++++++++ docker-compose/productivity.yml | 324 ++++++++++++++++++++ docker-compose/utilities.yml | 177 +++++++++++ docs/proxying-external-hosts.md | 379 ++++++++++++++++++++++++ 12 files changed, 2124 insertions(+), 49 deletions(-) create mode 100644 config-templates/homepage/docker.yaml create mode 100644 config-templates/homepage/services.yaml create mode 100644 config-templates/homepage/settings.yaml create mode 100644 config-templates/homepage/widgets.yaml create mode 100644 docker-compose/dashboards.yml create mode 100644 docker-compose/homeassistant.yml create mode 100644 docker-compose/media-extended.yml create mode 100644 docker-compose/productivity.yml create mode 100644 docker-compose/utilities.yml create mode 100644 docs/proxying-external-hosts.md diff --git a/.env.example b/.env.example index 9f171bb..7519bb9 100644 --- a/.env.example +++ b/.env.example @@ -47,6 +47,102 @@ VPN_COUNTRY=Netherlands # Preferred VPN server location # SURFSHARK_USERNAME=your-surfshark-username # SURFSHARK_PASSWORD=your-surfshark-password +# Media Services +PLEX_CLAIM=claim-xxxxxxxxxx + +# Monitoring & Dashboards +GRAFANA_ADMIN_PASSWORD=changeme + +# Development Tools +CODE_SERVER_PASSWORD=changeme +CODE_SERVER_SUDO_PASSWORD=changeme + +# Databases - General +POSTGRES_USER=postgres +POSTGRES_PASSWORD=changeme +POSTGRES_DB=homelab + +PGADMIN_EMAIL=admin@example.com +PGADMIN_PASSWORD=changeme + +# Infrastructure +PIHOLE_PASSWORD=changeme +WATCHTOWER_NOTIFICATION_URL= + +# Productivity Services - Nextcloud +NEXTCLOUD_ADMIN_USER=admin +NEXTCLOUD_ADMIN_PASSWORD=changeme +NEXTCLOUD_DB_PASSWORD=changeme +NEXTCLOUD_DB_ROOT_PASSWORD=changeme + +# Productivity Services - Gitea +GITEA_DB_PASSWORD=changeme + +# Productivity Services - WordPress +WORDPRESS_DB_PASSWORD=changeme +WORDPRESS_DB_ROOT_PASSWORD=changeme + +# Productivity Services - BookStack +BOOKSTACK_DB_PASSWORD=changeme +BOOKSTACK_DB_ROOT_PASSWORD=changeme + +# Productivity Services - MediaWiki +MEDIAWIKI_DB_PASSWORD=changeme +MEDIAWIKI_DB_ROOT_PASSWORD=changeme + +# Utilities - Form.io +FORMIO_JWT_SECRET=changeme +FORMIO_DB_SECRET=changeme + +# Development - Jupyter +JUPYTER_TOKEN=changeme + +# Cloudflare API (optional, for DNS challenge) +# CF_DNS_API_TOKEN=your-cloudflare-api-token + +# qBittorrent +QBITTORRENT_USER=admin +QBITTORRENT_PASS=changeme + +# Homepage Dashboard - API Keys and Tokens +# Generate these from each service's settings page +HOMEPAGE_VAR_DOMAIN=${DOMAIN} +HOMEPAGE_VAR_SERVER_IP=${SERVER_IP} +HOMEPAGE_VAR_PORTAINER_KEY=your-portainer-api-key +HOMEPAGE_VAR_PIHOLE_KEY=your-pihole-api-key +HOMEPAGE_VAR_PLEX_KEY=your-plex-token +HOMEPAGE_VAR_JELLYFIN_KEY=your-jellyfin-api-key +HOMEPAGE_VAR_SONARR_KEY=your-sonarr-api-key +HOMEPAGE_VAR_RADARR_KEY=your-radarr-api-key +HOMEPAGE_VAR_LIDARR_KEY=your-lidarr-api-key +HOMEPAGE_VAR_READARR_KEY=your-readarr-api-key +HOMEPAGE_VAR_PROWLARR_KEY=your-prowlarr-api-key +HOMEPAGE_VAR_JELLYSEERR_KEY=your-jellyseerr-api-key +HOMEPAGE_VAR_QBITTORRENT_USER=${QBITTORRENT_USER} +HOMEPAGE_VAR_QBITTORRENT_PASS=${QBITTORRENT_PASS} +HOMEPAGE_VAR_HA_KEY=your-home-assistant-long-lived-token +HOMEPAGE_VAR_NEXTCLOUD_USER=${NEXTCLOUD_ADMIN_USER} +HOMEPAGE_VAR_NEXTCLOUD_PASS=${NEXTCLOUD_ADMIN_PASSWORD} +HOMEPAGE_VAR_GRAFANA_USER=admin +HOMEPAGE_VAR_GRAFANA_PASS=${GRAFANA_ADMIN_PASSWORD} +HOMEPAGE_VAR_BOOKSTACK_KEY=your-bookstack-api-token +HOMEPAGE_VAR_UPTIMEKUMA_SLUG=your-uptime-kuma-slug +HOMEPAGE_VAR_OPENWEATHER_KEY=your-openweather-api-key +HOMEPAGE_VAR_WEATHERAPI_KEY=your-weatherapi-key +HOMEPAGE_VAR_UNIFI_USER=your-unifi-username +HOMEPAGE_VAR_UNIFI_PASS=your-unifi-password + +# Add your own variables below + +# Get WireGuard details from Surfshark dashboard +SURFSHARK_PRIVATE_KEY=your-wireguard-private-key +SURFSHARK_ADDRESSES=10.14.0.2/16 +VPN_COUNTRY=Netherlands # Preferred VPN server location + +# Alternative: OpenVPN credentials (if not using WireGuard) +# SURFSHARK_USERNAME=your-surfshark-username +# SURFSHARK_PASSWORD=your-surfshark-password + # Plex Configuration PLEX_CLAIM=claim-xxxxxxxxxx diff --git a/config-templates/homepage/docker.yaml b/config-templates/homepage/docker.yaml new file mode 100644 index 0000000..57ca23a --- /dev/null +++ b/config-templates/homepage/docker.yaml @@ -0,0 +1,13 @@ +# Homepage Configuration - Docker Integration +# Copy to /opt/stacks/homepage/config/docker.yaml +# Enables auto-discovery of containers and status monitoring + +--- +# Docker socket (via proxy for security) +my-docker: + socket: /var/run/docker.sock + +# Or use Docker socket proxy (recommended for production) +# my-docker: +# host: dockerproxy +# port: 2375 diff --git a/config-templates/homepage/services.yaml b/config-templates/homepage/services.yaml new file mode 100644 index 0000000..47a5e4c --- /dev/null +++ b/config-templates/homepage/services.yaml @@ -0,0 +1,373 @@ +# Homepage Configuration - Services +# Copy to /opt/stacks/homepage/config/services.yaml +# This file is AI-configurable - Homepage will auto-discover services via Docker labels + +--- +# Infrastructure Services +- Infrastructure: + - Dockge: + icon: dockge.png + href: https://dockge.{{HOMEPAGE_VAR_DOMAIN}} + description: Docker Compose Stack Manager (PRIMARY) + container: dockge + widget: + type: dockge + url: http://dockge:5001 + + - Traefik: + icon: traefik.png + href: https://traefik.{{HOMEPAGE_VAR_DOMAIN}} + description: Reverse Proxy & SSL + container: traefik + widget: + type: traefik + url: http://traefik:8080 + + - Authelia: + icon: authelia.png + href: https://auth.{{HOMEPAGE_VAR_DOMAIN}} + description: Single Sign-On + container: authelia + widget: + type: authelia + url: http://authelia:9091 + + - Portainer: + icon: portainer.png + href: https://portainer.{{HOMEPAGE_VAR_DOMAIN}} + description: Docker Management (Secondary) + container: portainer + widget: + type: portainer + url: http://portainer:9000 + env: 1 + key: {{HOMEPAGE_VAR_PORTAINER_KEY}} + + - Pi-hole: + icon: pi-hole.png + href: https://pihole.{{HOMEPAGE_VAR_DOMAIN}} + description: Network-wide Ad Blocking + container: pihole + widget: + type: pihole + url: http://pihole + key: {{HOMEPAGE_VAR_PIHOLE_KEY}} + + - Dozzle: + icon: dozzle.png + href: https://dozzle.{{HOMEPAGE_VAR_DOMAIN}} + description: Real-time Docker Logs + container: dozzle + + - Glances: + icon: glances.png + href: https://glances.{{HOMEPAGE_VAR_DOMAIN}} + description: System Monitoring + container: glances + widget: + type: glances + url: http://glances:61208 + metric: cpu + +# Dashboards +- Dashboards: + - Homarr: + icon: homarr.png + href: https://homarr.{{HOMEPAGE_VAR_DOMAIN}} + description: Alternative Dashboard + container: homarr + + - Uptime Kuma: + icon: uptime-kuma.png + href: https://status.{{HOMEPAGE_VAR_DOMAIN}} + description: Uptime Monitoring + container: uptime-kuma + widget: + type: uptimekuma + url: http://uptime-kuma:3001 + slug: {{HOMEPAGE_VAR_UPTIMEKUMA_SLUG}} + +# Media - Streaming +- Media Streaming: + - Plex: + icon: plex.png + href: https://plex.{{HOMEPAGE_VAR_DOMAIN}} + description: Media Server + container: plex + widget: + type: plex + url: http://plex:32400 + key: {{HOMEPAGE_VAR_PLEX_KEY}} + + - Jellyfin: + icon: jellyfin.png + href: https://jellyfin.{{HOMEPAGE_VAR_DOMAIN}} + description: Open Source Media Server + container: jellyfin + widget: + type: jellyfin + url: http://jellyfin:8096 + key: {{HOMEPAGE_VAR_JELLYFIN_KEY}} + + - Jellyseerr: + icon: jellyseerr.png + href: https://jellyseerr.{{HOMEPAGE_VAR_DOMAIN}} + description: Media Requests + container: jellyseerr + widget: + type: jellyseerr + url: http://jellyseerr:5055 + key: {{HOMEPAGE_VAR_JELLYSEERR_KEY}} + +# Media - Management +- Media Management: + - Sonarr: + icon: sonarr.png + href: https://sonarr.{{HOMEPAGE_VAR_DOMAIN}} + description: TV Show Management + container: sonarr + widget: + type: sonarr + url: http://sonarr:8989 + key: {{HOMEPAGE_VAR_SONARR_KEY}} + + - Radarr: + icon: radarr.png + href: https://radarr.{{HOMEPAGE_VAR_DOMAIN}} + description: Movie Management + container: radarr + widget: + type: radarr + url: http://radarr:7878 + key: {{HOMEPAGE_VAR_RADARR_KEY}} + + - Lidarr: + icon: lidarr.png + href: https://lidarr.{{HOMEPAGE_VAR_DOMAIN}} + description: Music Management + container: lidarr + widget: + type: lidarr + url: http://lidarr:8686 + key: {{HOMEPAGE_VAR_LIDARR_KEY}} + + - Readarr: + icon: readarr.png + href: https://readarr.{{HOMEPAGE_VAR_DOMAIN}} + description: Book Management + container: readarr + widget: + type: readarr + url: http://readarr:8787 + key: {{HOMEPAGE_VAR_READARR_KEY}} + + - Prowlarr: + icon: prowlarr.png + href: https://prowlarr.{{HOMEPAGE_VAR_DOMAIN}} + description: Indexer Manager + container: prowlarr + widget: + type: prowlarr + url: http://prowlarr:9696 + key: {{HOMEPAGE_VAR_PROWLARR_KEY}} + +# Downloads +- Downloads: + - qBittorrent: + icon: qbittorrent.png + href: https://qbit.{{HOMEPAGE_VAR_DOMAIN}} + description: Torrent Client (via VPN) + container: qbittorrent + widget: + type: qbittorrent + url: http://gluetun:8080 + username: {{HOMEPAGE_VAR_QBITTORRENT_USER}} + password: {{HOMEPAGE_VAR_QBITTORRENT_PASS}} + + - Gluetun: + icon: gluetun.png + href: http://gluetun:8000 + description: VPN Client (Surfshark) + container: gluetun + +# Books & Comics +- Books & Comics: + - Calibre-Web: + icon: calibre-web.png + href: https://calibre.{{HOMEPAGE_VAR_DOMAIN}} + description: Ebook Library + container: calibre-web + + - Lazy Librarian: + icon: lazylibrarian.png + href: https://lazylibrarian.{{HOMEPAGE_VAR_DOMAIN}} + description: Book Manager + container: lazylibrarian + + - Mylar3: + icon: mylar3.png + href: https://mylar.{{HOMEPAGE_VAR_DOMAIN}} + description: Comic Book Manager + container: mylar3 + +# Transcoding +- Transcoding: + - Tdarr: + icon: tdarr.png + href: https://tdarr.{{HOMEPAGE_VAR_DOMAIN}} + description: Distributed Transcoding + container: tdarr-server + widget: + type: tdarr + url: http://tdarr-server:8265 + + - Unmanic: + icon: unmanic.png + href: https://unmanic.{{HOMEPAGE_VAR_DOMAIN}} + description: Library Optimizer + container: unmanic + +# Home Automation +- Home Automation: + - Home Assistant: + icon: home-assistant.png + href: https://ha.{{HOMEPAGE_VAR_DOMAIN}} + description: Home Automation Hub + # Note: Uses host network, configure manually + widget: + type: homeassistant + url: http://{{HOMEPAGE_VAR_SERVER_IP}}:8123 + key: {{HOMEPAGE_VAR_HA_KEY}} + + - ESPHome: + icon: esphome.png + href: https://esphome.{{HOMEPAGE_VAR_DOMAIN}} + description: ESP Device Manager + container: esphome + + - Node-RED: + icon: node-red.png + href: https://nodered.{{HOMEPAGE_VAR_DOMAIN}} + description: Flow Automation + container: nodered + + - TasmoAdmin: + icon: tasmota.png + href: https://tasmoadmin.{{HOMEPAGE_VAR_DOMAIN}} + description: Tasmota Device Manager + container: tasmoadmin + + - Zigbee2MQTT: + icon: zigbee2mqtt.png + href: https://zigbee2mqtt.{{HOMEPAGE_VAR_DOMAIN}} + description: Zigbee Bridge + container: zigbee2mqtt + + - MotionEye: + icon: motioneye.png + href: https://motioneye.{{HOMEPAGE_VAR_DOMAIN}} + description: Video Surveillance + container: motioneye + +# Productivity +- Productivity: + - Nextcloud: + icon: nextcloud.png + href: https://nextcloud.{{HOMEPAGE_VAR_DOMAIN}} + description: File Sync & Share + container: nextcloud + widget: + type: nextcloud + url: http://nextcloud + username: {{HOMEPAGE_VAR_NEXTCLOUD_USER}} + password: {{HOMEPAGE_VAR_NEXTCLOUD_PASS}} + + - Mealie: + icon: mealie.png + href: https://mealie.{{HOMEPAGE_VAR_DOMAIN}} + description: Recipe Manager + container: mealie + + - Gitea: + icon: gitea.png + href: https://git.{{HOMEPAGE_VAR_DOMAIN}} + description: Git Service + container: gitea + + - Code Server: + icon: vscode.png + href: https://code.{{HOMEPAGE_VAR_DOMAIN}} + description: VS Code in Browser + container: code-server + +# Documentation +- Documentation: + - BookStack: + icon: bookstack.png + href: https://docs.{{HOMEPAGE_VAR_DOMAIN}} + description: Documentation Platform + container: bookstack + widget: + type: bookstack + url: http://bookstack + key: {{HOMEPAGE_VAR_BOOKSTACK_KEY}} + + - DokuWiki: + icon: dokuwiki.png + href: https://wiki.{{HOMEPAGE_VAR_DOMAIN}} + description: File-based Wiki + container: dokuwiki + + - MediaWiki: + icon: mediawiki.png + href: https://mediawiki.{{HOMEPAGE_VAR_DOMAIN}} + description: Wiki Platform + container: mediawiki + + - WordPress: + icon: wordpress.png + href: https://blog.{{HOMEPAGE_VAR_DOMAIN}} + description: Blog Platform + container: wordpress + +# Backups & Monitoring +- Backups & Tools: + - Backrest: + icon: backrest.png + href: https://backrest.{{HOMEPAGE_VAR_DOMAIN}} + description: Backup Manager (Restic) + container: backrest + + - Duplicati: + icon: duplicati.png + href: https://duplicati.{{HOMEPAGE_VAR_DOMAIN}} + description: Backup Software + container: duplicati + +# Monitoring Stack +- Monitoring: + - Grafana: + icon: grafana.png + href: https://grafana.{{HOMEPAGE_VAR_DOMAIN}} + description: Metrics Visualization + container: grafana + widget: + type: grafana + url: http://grafana:3000 + username: {{HOMEPAGE_VAR_GRAFANA_USER}} + password: {{HOMEPAGE_VAR_GRAFANA_PASS}} + + - Prometheus: + icon: prometheus.png + href: https://prometheus.{{HOMEPAGE_VAR_DOMAIN}} + description: Metrics Collection + container: prometheus + widget: + type: prometheus + url: http://prometheus:9090 + + - Uptime Kuma: + icon: uptime-kuma.png + href: https://status.{{HOMEPAGE_VAR_DOMAIN}} + description: Status Page + container: uptime-kuma diff --git a/config-templates/homepage/settings.yaml b/config-templates/homepage/settings.yaml new file mode 100644 index 0000000..6fc2f89 --- /dev/null +++ b/config-templates/homepage/settings.yaml @@ -0,0 +1,63 @@ +# Homepage Configuration - Settings +# Copy to /opt/stacks/homepage/config/settings.yaml + +--- +title: Homelab Dashboard +background: https://images.unsplash.com/photo-1558591710-4b4a1ae0f04d +backgroundOpacity: 0.2 +theme: dark +color: slate +headerStyle: boxed +hideVersion: true +hideErrors: false +showStats: true +target: _self # Open links in same tab + +# Layout configuration +layout: + Infrastructure: + style: row + columns: 4 + Dashboards: + style: row + columns: 2 + Media Streaming: + style: row + columns: 3 + Media Management: + style: row + columns: 4 + Downloads: + style: row + columns: 2 + Books & Comics: + style: row + columns: 3 + Home Automation: + style: row + columns: 4 + Productivity: + style: row + columns: 4 + Documentation: + style: row + columns: 4 + Backups & Tools: + style: row + columns: 2 + Monitoring: + style: row + columns: 3 + +# Quick search +quicklaunch: + searchDescriptions: true + hideInternetSearch: false + showSearchSuggestions: true + +# Providers for additional functionality +providers: + longhorn: + url: http://longhorn:9500 + openweathermap: {{HOMEPAGE_VAR_OPENWEATHER_KEY}} + weatherapi: {{HOMEPAGE_VAR_WEATHERAPI_KEY}} diff --git a/config-templates/homepage/widgets.yaml b/config-templates/homepage/widgets.yaml new file mode 100644 index 0000000..48ad40d --- /dev/null +++ b/config-templates/homepage/widgets.yaml @@ -0,0 +1,49 @@ +# Homepage Configuration - Widgets +# Copy to /opt/stacks/homepage/config/widgets.yaml +# Displays system resources and other information + +--- +- logo: + icon: https://avatars.githubusercontent.com/u/... # Your logo here + +- search: + provider: google + target: _blank + +- datetime: + text_size: xl + format: + dateStyle: long + timeStyle: short + hourCycle: h23 + +- resources: + label: System + cpu: true + memory: true + disk: / + cputemp: true + uptime: true + units: metric + +- resources: + label: Storage + disk: /mnt/media + expanded: true + +- resources: + label: Downloads + disk: /mnt/downloads + expanded: true + +- openmeteo: + label: Weather + latitude: 40.7128 + longitude: -74.0060 + units: metric + cache: 5 + +- unifi_console: + url: http://unifi:8443 + username: {{HOMEPAGE_VAR_UNIFI_USER}} + password: {{HOMEPAGE_VAR_UNIFI_PASS}} diff --git a/docker-compose/dashboards.yml b/docker-compose/dashboards.yml new file mode 100644 index 0000000..2e83333 --- /dev/null +++ b/docker-compose/dashboards.yml @@ -0,0 +1,66 @@ +# Dashboard Services +# Homepage and Homarr for homelab dashboards +# Place in /opt/stacks/dashboards/docker-compose.yml + +services: + # Homepage - Application dashboard (AI-configurable via YAML) + # Access at: https://home.${DOMAIN} + homepage: + image: ghcr.io/gethomepage/homepage:latest + container_name: homepage + restart: unless-stopped + networks: + - homelab-network + - traefik-network + - dockerproxy-network + volumes: + - /opt/stacks/homepage/config:/app/config + - /var/run/docker.sock:/var/run/docker.sock:ro # For Docker integration + - /opt/stacks:/opt/stacks:ro # To discover other stacks + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + labels: + - "homelab.category=dashboard" + - "homelab.description=Application dashboard (AI-configurable)" + - "traefik.enable=true" + - "traefik.http.routers.homepage.rule=Host(`home.${DOMAIN}`)" + - "traefik.http.routers.homepage.entrypoints=websecure" + - "traefik.http.routers.homepage.tls.certresolver=letsencrypt" + - "traefik.http.services.homepage.loadbalancer.server.port=3000" + # No Authelia - make it the default landing page + + # Homarr - Modern dashboard + # Access at: https://homarr.${DOMAIN} + homarr: + image: ghcr.io/ajnart/homarr:latest + container_name: homarr + restart: unless-stopped + networks: + - homelab-network + - traefik-network + volumes: + - /opt/stacks/homarr/configs:/app/data/configs + - /opt/stacks/homarr/data:/data + - /opt/stacks/homarr/icons:/app/public/icons + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + - TZ=${TZ} + labels: + - "homelab.category=dashboard" + - "homelab.description=Modern homelab dashboard" + - "traefik.enable=true" + - "traefik.http.routers.homarr.rule=Host(`homarr.${DOMAIN}`)" + - "traefik.http.routers.homarr.entrypoints=websecure" + - "traefik.http.routers.homarr.tls.certresolver=letsencrypt" + - "traefik.http.services.homarr.loadbalancer.server.port=7575" + # No Authelia - dashboard should be accessible + +networks: + homelab-network: + external: true + traefik-network: + external: true + dockerproxy-network: + external: true diff --git a/docker-compose/homeassistant.yml b/docker-compose/homeassistant.yml new file mode 100644 index 0000000..199330e --- /dev/null +++ b/docker-compose/homeassistant.yml @@ -0,0 +1,173 @@ +# Home Assistant and IoT Services +# Home automation platform and related tools +# Place in /opt/stacks/homeassistant/docker-compose.yml + +services: + # Home Assistant - Home automation platform + # Access at: https://ha.${DOMAIN} + # NOTE: No Authelia - HA has its own authentication + homeassistant: + image: ghcr.io/home-assistant/home-assistant:2024.1 + container_name: homeassistant + restart: unless-stopped + network_mode: host # Required for device discovery + volumes: + - /opt/stacks/homeassistant/config:/config + - /etc/localtime:/etc/localtime:ro + environment: + - TZ=${TZ} + privileged: true + labels: + - "homelab.category=iot" + - "homelab.description=Home automation platform" + # Note: network_mode: host means Traefik can't proxy this directly + # Use Traefik's file provider or external host routing + + # ESPHome - ESP8266/ESP32 firmware manager + # Access at: https://esphome.${DOMAIN} + esphome: + image: ghcr.io/esphome/esphome:latest + container_name: esphome + restart: unless-stopped + networks: + - homelab-network + - traefik-network + volumes: + - /opt/stacks/esphome/config:/config + - /etc/localtime:/etc/localtime:ro + environment: + - TZ=${TZ} + - ESPHOME_DASHBOARD_USE_PING=true + privileged: true # For USB device access + labels: + - "homelab.category=iot" + - "homelab.description=ESP8266/ESP32 firmware manager" + - "traefik.enable=true" + - "traefik.http.routers.esphome.rule=Host(`esphome.${DOMAIN}`)" + - "traefik.http.routers.esphome.entrypoints=websecure" + - "traefik.http.routers.esphome.tls.certresolver=letsencrypt" + - "traefik.http.routers.esphome.middlewares=authelia@docker" + - "traefik.http.services.esphome.loadbalancer.server.port=6052" + + # TasmoAdmin - Tasmota device manager + # Access at: https://tasmoadmin.${DOMAIN} + tasmoadmin: + image: ghcr.io/tasmoadmin/tasmoadmin:latest + container_name: tasmoadmin + restart: unless-stopped + networks: + - homelab-network + - traefik-network + volumes: + - /opt/stacks/tasmoadmin/data:/data + environment: + - TZ=${TZ} + labels: + - "homelab.category=iot" + - "homelab.description=Tasmota device management" + - "traefik.enable=true" + - "traefik.http.routers.tasmoadmin.rule=Host(`tasmoadmin.${DOMAIN}`)" + - "traefik.http.routers.tasmoadmin.entrypoints=websecure" + - "traefik.http.routers.tasmoadmin.tls.certresolver=letsencrypt" + - "traefik.http.routers.tasmoadmin.middlewares=authelia@docker" + - "traefik.http.services.tasmoadmin.loadbalancer.server.port=80" + + # MotionEye - Video surveillance + # Access at: https://motioneye.${DOMAIN} + motioneye: + image: ccrisan/motioneye:master-amd64 + container_name: motioneye + restart: unless-stopped + networks: + - homelab-network + - traefik-network + ports: + - "8765:8765" # Optional: direct access + volumes: + - /opt/stacks/motioneye/config:/etc/motioneye + - /mnt/surveillance:/var/lib/motioneye # Large video files on separate drive + environment: + - TZ=${TZ} + labels: + - "homelab.category=iot" + - "homelab.description=Video surveillance system" + - "traefik.enable=true" + - "traefik.http.routers.motioneye.rule=Host(`motioneye.${DOMAIN}`)" + - "traefik.http.routers.motioneye.entrypoints=websecure" + - "traefik.http.routers.motioneye.tls.certresolver=letsencrypt" + - "traefik.http.routers.motioneye.middlewares=authelia@docker" + - "traefik.http.services.motioneye.loadbalancer.server.port=8765" + + # Node-RED - Flow-based automation (Home Assistant addon alternative) + # Access at: https://nodered.${DOMAIN} + nodered: + image: nodered/node-red:latest + container_name: nodered + restart: unless-stopped + networks: + - homelab-network + - traefik-network + volumes: + - /opt/stacks/nodered/data:/data + environment: + - TZ=${TZ} + labels: + - "homelab.category=iot" + - "homelab.description=Flow-based automation programming" + - "traefik.enable=true" + - "traefik.http.routers.nodered.rule=Host(`nodered.${DOMAIN}`)" + - "traefik.http.routers.nodered.entrypoints=websecure" + - "traefik.http.routers.nodered.tls.certresolver=letsencrypt" + - "traefik.http.routers.nodered.middlewares=authelia@docker" + - "traefik.http.services.nodered.loadbalancer.server.port=1880" + + # Mosquitto - MQTT broker (Home Assistant addon alternative) + # Used by: Home Assistant, ESPHome, Tasmota devices + mosquitto: + image: eclipse-mosquitto:latest + container_name: mosquitto + restart: unless-stopped + networks: + - homelab-network + ports: + - "1883:1883" # MQTT + - "9001:9001" # Websockets + volumes: + - /opt/stacks/mosquitto/config:/mosquitto/config + - /opt/stacks/mosquitto/data:/mosquitto/data + - /opt/stacks/mosquitto/log:/mosquitto/log + labels: + - "homelab.category=iot" + - "homelab.description=MQTT message broker" + + # Zigbee2MQTT - Zigbee to MQTT bridge (Home Assistant addon alternative) + # Access at: https://zigbee2mqtt.${DOMAIN} + zigbee2mqtt: + image: koenkk/zigbee2mqtt:latest + container_name: zigbee2mqtt + restart: unless-stopped + networks: + - homelab-network + - traefik-network + volumes: + - /opt/stacks/zigbee2mqtt/data:/app/data + - /run/udev:/run/udev:ro + devices: + - /dev/ttyACM0:/dev/ttyACM0 # Zigbee adapter - adjust as needed + environment: + - TZ=${TZ} + labels: + - "homelab.category=iot" + - "homelab.description=Zigbee to MQTT bridge" + - "traefik.enable=true" + - "traefik.http.routers.zigbee2mqtt.rule=Host(`zigbee2mqtt.${DOMAIN}`)" + - "traefik.http.routers.zigbee2mqtt.entrypoints=websecure" + - "traefik.http.routers.zigbee2mqtt.tls.certresolver=letsencrypt" + - "traefik.http.routers.zigbee2mqtt.middlewares=authelia@docker" + - "traefik.http.services.zigbee2mqtt.loadbalancer.server.port=8080" + +networks: + homelab-network: + external: true + traefik-network: + external: true diff --git a/docker-compose/infrastructure.yml b/docker-compose/infrastructure.yml index 2e628d5..8a7d752 100644 --- a/docker-compose/infrastructure.yml +++ b/docker-compose/infrastructure.yml @@ -1,11 +1,63 @@ # Infrastructure Services # Core services that other services depend on -# NOTE: Traefik, Authelia, DuckDNS, and Gluetun have their own compose files -# See traefik.yml, authelia.yml, duckdns.yml, and gluetun.yml +# Place in /opt/stacks/infrastructure/docker-compose.yml +# NOTE: Traefik, Authelia, DuckDNS, and Gluetun have their own separate stacks +# See /opt/stacks/traefik/, /opt/stacks/authelia/, etc. services: + # Dockge - Docker Compose Stack Manager (PRIMARY - preferred over Portainer) + # Access at: https://dockge.${DOMAIN} + dockge: + image: louislam/dockge:1 + container_name: dockge + restart: unless-stopped + networks: + - homelab-network + - traefik-network + ports: + - "5001:5001" # Optional: direct access + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /opt/stacks:/opt/stacks # Dockge manages stacks in this directory + - /opt/dockge/data:/app/data + environment: + - DOCKGE_STACKS_DIR=/opt/stacks + labels: + - "homelab.category=infrastructure" + - "homelab.description=Docker Compose stack manager (PRIMARY)" + - "traefik.enable=true" + - "traefik.http.routers.dockge.rule=Host(`dockge.${DOMAIN}`)" + - "traefik.http.routers.dockge.entrypoints=websecure" + - "traefik.http.routers.dockge.tls.certresolver=letsencrypt" + - "traefik.http.routers.dockge.middlewares=authelia@docker" + - "traefik.http.services.dockge.loadbalancer.server.port=5001" + + # Portainer - Docker management UI (SECONDARY - use Dockge instead) + # Access at: https://portainer.${DOMAIN} + portainer: + image: portainer/portainer-ce:2.19.4 + container_name: portainer + restart: unless-stopped + networks: + - homelab-network + - traefik-network + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - portainer-data:/data + security_opt: + - no-new-privileges:true + labels: + - "homelab.category=infrastructure" + - "homelab.description=Docker container management UI (SECONDARY)" + - "traefik.enable=true" + - "traefik.http.routers.portainer.rule=Host(`portainer.${DOMAIN}`)" + - "traefik.http.routers.portainer.entrypoints=websecure" + - "traefik.http.routers.portainer.tls.certresolver=letsencrypt" + - "traefik.http.routers.portainer.middlewares=authelia@docker" + - "traefik.http.services.portainer.loadbalancer.server.port=9000" + # Pi-hole - Network-wide ad blocker and DNS server - # Access at: http://server-ip:8080/admin or https://pihole.yourdomain.duckdns.org + # Access at: https://pihole.${DOMAIN} pihole: image: pihole/pihole:2024.01.0 container_name: pihole @@ -16,7 +68,6 @@ services: ports: - "53:53/tcp" # DNS TCP - "53:53/udp" # DNS UDP - - "8082:80/tcp" # Web interface (changed from 8080 to avoid conflicts) volumes: - /opt/stacks/pihole/etc-pihole:/etc/pihole - /opt/stacks/pihole/etc-dnsmasq.d:/etc/dnsmasq.d @@ -32,57 +83,12 @@ services: labels: - "homelab.category=infrastructure" - "homelab.description=Network-wide ad blocking and DNS" - # Traefik labels - "traefik.enable=true" - "traefik.http.routers.pihole.rule=Host(`pihole.${DOMAIN}`)" - "traefik.http.routers.pihole.entrypoints=websecure" - "traefik.http.routers.pihole.tls.certresolver=letsencrypt" - "traefik.http.routers.pihole.middlewares=authelia@docker" - "traefik.http.services.pihole.loadbalancer.server.port=80" - ports: - - "53:53/tcp" # DNS TCP - - "53:53/udp" # DNS UDP - - "8080:80/tcp" # Web interface - volumes: - - ./config/pihole/etc-pihole:/etc/pihole - - ./config/pihole/etc-dnsmasq.d:/etc/dnsmasq.d - environment: - - TZ=${TZ:-America/New_York} - - WEBPASSWORD=${PIHOLE_PASSWORD:-changeme} - - FTLCONF_LOCAL_IPV4=${SERVER_IP} - dns: - - 127.0.0.1 - - 1.1.1.1 - cap_add: - - NET_ADMIN - labels: - - "homelab.category=infrastructure" - - "homelab.description=Network-wide ad blocking and DNS" - - # Portainer - Docker management UI - # Access at: https://portainer.yourdomain.duckdns.org - portainer: - image: portainer/portainer-ce:2.19.4 - container_name: portainer - restart: unless-stopped - networks: - - homelab-network - - traefik-network - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - portainer-data:/data - security_opt: - - no-new-privileges:true - labels: - - "homelab.category=infrastructure" - - "homelab.description=Docker container management UI" - # Traefik labels - - "traefik.enable=true" - - "traefik.http.routers.portainer.rule=Host(`portainer.${DOMAIN}`)" - - "traefik.http.routers.portainer.entrypoints=websecure" - - "traefik.http.routers.portainer.tls.certresolver=letsencrypt" - - "traefik.http.routers.portainer.middlewares=authelia@docker" - - "traefik.http.services.portainer.loadbalancer.server.port=9000" # Watchtower - Automatic container updates # Runs silently in background, no UI @@ -104,6 +110,78 @@ services: - "homelab.category=infrastructure" - "homelab.description=Automatic Docker container updates" + # Dozzle - Real-time Docker log viewer + # Access at: https://dozzle.${DOMAIN} + dozzle: + image: amir20/dozzle:latest + container_name: dozzle + restart: unless-stopped + networks: + - homelab-network + - traefik-network + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + - DOZZLE_LEVEL=info + - DOZZLE_TAILSIZE=300 + - DOZZLE_FILTER=status=running + labels: + - "homelab.category=infrastructure" + - "homelab.description=Real-time Docker log viewer" + - "traefik.enable=true" + - "traefik.http.routers.dozzle.rule=Host(`dozzle.${DOMAIN}`)" + - "traefik.http.routers.dozzle.entrypoints=websecure" + - "traefik.http.routers.dozzle.tls.certresolver=letsencrypt" + - "traefik.http.routers.dozzle.middlewares=authelia@docker" + - "traefik.http.services.dozzle.loadbalancer.server.port=8080" + + # Docker Proxy - Socket proxy for security + # Used by services that need Docker socket access + dockerproxy: + image: tecnativa/docker-socket-proxy:latest + container_name: dockerproxy + restart: unless-stopped + networks: + - dockerproxy-network + ports: + - "127.0.0.1:2375:2375" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + - CONTAINERS=1 + - SERVICES=1 + - TASKS=1 + - NETWORKS=1 + - NODES=1 + labels: + - "homelab.category=infrastructure" + - "homelab.description=Docker socket proxy for security" + + # Glances - System monitoring + # Access at: https://glances.${DOMAIN} + glances: + image: nicolargo/glances:latest-full + container_name: glances + restart: unless-stopped + networks: + - homelab-network + - traefik-network + pid: host + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - /opt/stacks/glances/config:/glances/conf + environment: + - GLANCES_OPT=-w + labels: + - "homelab.category=infrastructure" + - "homelab.description=System and Docker monitoring" + - "traefik.enable=true" + - "traefik.http.routers.glances.rule=Host(`glances.${DOMAIN}`)" + - "traefik.http.routers.glances.entrypoints=websecure" + - "traefik.http.routers.glances.tls.certresolver=letsencrypt" + - "traefik.http.routers.glances.middlewares=authelia@docker" + - "traefik.http.services.glances.loadbalancer.server.port=61208" + volumes: portainer-data: driver: local @@ -113,3 +191,5 @@ networks: external: true traefik-network: external: true + dockerproxy-network: + driver: bridge diff --git a/docker-compose/media-extended.yml b/docker-compose/media-extended.yml new file mode 100644 index 0000000..78997b6 --- /dev/null +++ b/docker-compose/media-extended.yml @@ -0,0 +1,282 @@ +# Extended Media Services +# Additional media management tools +# Place in /opt/stacks/media-extended/docker-compose.yml + +services: + # Readarr - Ebook and audiobook management + # Access at: https://readarr.${DOMAIN} + readarr: + image: lscr.io/linuxserver/readarr:develop + container_name: readarr + restart: unless-stopped + networks: + - media-network + - homelab-network + - traefik-network + volumes: + - /opt/stacks/readarr/config:/config + - /mnt/media/books:/books + - /mnt/downloads:/downloads + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + labels: + - "homelab.category=media" + - "homelab.description=Ebook and audiobook management" + - "traefik.enable=true" + - "traefik.http.routers.readarr.rule=Host(`readarr.${DOMAIN}`)" + - "traefik.http.routers.readarr.entrypoints=websecure" + - "traefik.http.routers.readarr.tls.certresolver=letsencrypt" + - "traefik.http.routers.readarr.middlewares=authelia@docker" + - "traefik.http.services.readarr.loadbalancer.server.port=8787" + + # Lidarr - Music collection manager + # Access at: https://lidarr.${DOMAIN} + lidarr: + image: lscr.io/linuxserver/lidarr:latest + container_name: lidarr + restart: unless-stopped + networks: + - media-network + - homelab-network + - traefik-network + volumes: + - /opt/stacks/lidarr/config:/config + - /mnt/media/music:/music + - /mnt/downloads:/downloads + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + labels: + - "homelab.category=media" + - "homelab.description=Music collection manager" + - "traefik.enable=true" + - "traefik.http.routers.lidarr.rule=Host(`lidarr.${DOMAIN}`)" + - "traefik.http.routers.lidarr.entrypoints=websecure" + - "traefik.http.routers.lidarr.tls.certresolver=letsencrypt" + - "traefik.http.routers.lidarr.middlewares=authelia@docker" + - "traefik.http.services.lidarr.loadbalancer.server.port=8686" + + # Lazy Librarian - Book manager + # Access at: https://lazylibrarian.${DOMAIN} + lazylibrarian: + image: lscr.io/linuxserver/lazylibrarian:latest + container_name: lazylibrarian + restart: unless-stopped + networks: + - media-network + - homelab-network + - traefik-network + volumes: + - /opt/stacks/lazylibrarian/config:/config + - /mnt/media/books:/books + - /mnt/downloads:/downloads + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + - DOCKER_MODS=linuxserver/mods:lazylibrarian-ffmpeg + labels: + - "homelab.category=media" + - "homelab.description=Book download automation" + - "traefik.enable=true" + - "traefik.http.routers.lazylibrarian.rule=Host(`lazylibrarian.${DOMAIN}`)" + - "traefik.http.routers.lazylibrarian.entrypoints=websecure" + - "traefik.http.routers.lazylibrarian.tls.certresolver=letsencrypt" + - "traefik.http.routers.lazylibrarian.middlewares=authelia@docker" + - "traefik.http.services.lazylibrarian.loadbalancer.server.port=5299" + + # Mylar3 - Comic book manager + # Access at: https://mylar.${DOMAIN} + mylar3: + image: lscr.io/linuxserver/mylar3:latest + container_name: mylar3 + restart: unless-stopped + networks: + - media-network + - homelab-network + - traefik-network + volumes: + - /opt/stacks/mylar3/config:/config + - /mnt/media/comics:/comics + - /mnt/downloads:/downloads + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + labels: + - "homelab.category=media" + - "homelab.description=Comic book collection manager" + - "traefik.enable=true" + - "traefik.http.routers.mylar.rule=Host(`mylar.${DOMAIN}`)" + - "traefik.http.routers.mylar.entrypoints=websecure" + - "traefik.http.routers.mylar.tls.certresolver=letsencrypt" + - "traefik.http.routers.mylar.middlewares=authelia@docker" + - "traefik.http.services.mylar.loadbalancer.server.port=8090" + + # Calibre-Web - Ebook reader and server + # Access at: https://calibre.${DOMAIN} + calibre-web: + image: lscr.io/linuxserver/calibre-web:latest + container_name: calibre-web + restart: unless-stopped + networks: + - media-network + - homelab-network + - traefik-network + volumes: + - /opt/stacks/calibre-web/config:/config + - /mnt/media/books:/books + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + - DOCKER_MODS=linuxserver/mods:universal-calibre + labels: + - "homelab.category=media" + - "homelab.description=Ebook reader and library management" + - "traefik.enable=true" + - "traefik.http.routers.calibre.rule=Host(`calibre.${DOMAIN}`)" + - "traefik.http.routers.calibre.entrypoints=websecure" + - "traefik.http.routers.calibre.tls.certresolver=letsencrypt" + - "traefik.http.routers.calibre.middlewares=authelia@docker" + - "traefik.http.services.calibre.loadbalancer.server.port=8083" + + # Jellyseerr - Request management for Jellyfin/Plex + # Access at: https://jellyseerr.${DOMAIN} + jellyseerr: + image: fallenbagel/jellyseerr:latest + container_name: jellyseerr + restart: unless-stopped + networks: + - media-network + - homelab-network + - traefik-network + volumes: + - /opt/stacks/jellyseerr/config:/app/config + environment: + - LOG_LEVEL=info + - TZ=${TZ} + labels: + - "homelab.category=media" + - "homelab.description=Media request management" + - "traefik.enable=true" + - "traefik.http.routers.jellyseerr.rule=Host(`jellyseerr.${DOMAIN}`)" + - "traefik.http.routers.jellyseerr.entrypoints=websecure" + - "traefik.http.routers.jellyseerr.tls.certresolver=letsencrypt" + - "traefik.http.services.jellyseerr.loadbalancer.server.port=5055" + # No Authelia - users should be able to request easily + + # FlareSolverr - Cloudflare bypass for Prowlarr + # No web UI - used by Prowlarr + flaresolverr: + image: ghcr.io/flaresolverr/flaresolverr:latest + container_name: flaresolverr + restart: unless-stopped + networks: + - media-network + environment: + - LOG_LEVEL=info + - TZ=${TZ} + labels: + - "homelab.category=media" + - "homelab.description=Cloudflare bypass for indexers" + + # Tdarr Server - Distributed transcoding server + # Access at: https://tdarr.${DOMAIN} + tdarr-server: + image: ghcr.io/haveagitgat/tdarr:latest + container_name: tdarr-server + restart: unless-stopped + networks: + - media-network + - homelab-network + - traefik-network + ports: + - "8266:8266" # Server port + volumes: + - /opt/stacks/tdarr/server:/app/server + - /opt/stacks/tdarr/configs:/app/configs + - /opt/stacks/tdarr/logs:/app/logs + - /mnt/media:/media + - /mnt/tdarr-transcode:/temp # Transcode cache on separate drive + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + - serverIP=0.0.0.0 + - serverPort=8266 + - webUIPort=8265 + labels: + - "homelab.category=media" + - "homelab.description=Distributed transcoding server" + - "traefik.enable=true" + - "traefik.http.routers.tdarr.rule=Host(`tdarr.${DOMAIN}`)" + - "traefik.http.routers.tdarr.entrypoints=websecure" + - "traefik.http.routers.tdarr.tls.certresolver=letsencrypt" + - "traefik.http.routers.tdarr.middlewares=authelia@docker" + - "traefik.http.services.tdarr.loadbalancer.server.port=8265" + + # Tdarr Node - Transcoding worker + # No web UI - controlled by server + tdarr-node: + image: ghcr.io/haveagitgat/tdarr_node:latest + container_name: tdarr-node + restart: unless-stopped + networks: + - media-network + volumes: + - /opt/stacks/tdarr/configs:/app/configs + - /opt/stacks/tdarr/logs:/app/logs + - /mnt/media:/media + - /mnt/tdarr-transcode:/temp + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + - nodeID=MainNode + - nodeIP=0.0.0.0 + - nodePort=8267 + - serverIP=tdarr-server + - serverPort=8266 + labels: + - "homelab.category=media" + - "homelab.description=Tdarr transcoding worker node" + + # Unmanic - Another transcoding option + # Access at: https://unmanic.${DOMAIN} + unmanic: + image: josh5/unmanic:latest + container_name: unmanic + restart: unless-stopped + networks: + - media-network + - homelab-network + - traefik-network + volumes: + - /opt/stacks/unmanic/config:/config + - /mnt/media:/library + - /mnt/unmanic-cache:/tmp/unmanic # Transcode cache on separate drive + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + labels: + - "homelab.category=media" + - "homelab.description=Library optimization and transcoding" + - "traefik.enable=true" + - "traefik.http.routers.unmanic.rule=Host(`unmanic.${DOMAIN}`)" + - "traefik.http.routers.unmanic.entrypoints=websecure" + - "traefik.http.routers.unmanic.tls.certresolver=letsencrypt" + - "traefik.http.routers.unmanic.middlewares=authelia@docker" + - "traefik.http.services.unmanic.loadbalancer.server.port=8888" + +networks: + media-network: + external: true + homelab-network: + external: true + traefik-network: + external: true diff --git a/docker-compose/productivity.yml b/docker-compose/productivity.yml new file mode 100644 index 0000000..5d85fe3 --- /dev/null +++ b/docker-compose/productivity.yml @@ -0,0 +1,324 @@ +# Productivity and Content Management Services +# Place in /opt/stacks/productivity/docker-compose.yml + +services: + # Nextcloud - File sync and collaboration + # Access at: https://nextcloud.${DOMAIN} + nextcloud: + image: nextcloud:latest + container_name: nextcloud + restart: unless-stopped + networks: + - homelab-network + - traefik-network + - nextcloud-network + volumes: + - /opt/stacks/nextcloud/html:/var/www/html + - /mnt/nextcloud-data:/var/www/html/data # Large data on separate drive + environment: + - MYSQL_HOST=nextcloud-db + - MYSQL_DATABASE=nextcloud + - MYSQL_USER=nextcloud + - MYSQL_PASSWORD=${NEXTCLOUD_DB_PASSWORD} + - NEXTCLOUD_ADMIN_USER=${NEXTCLOUD_ADMIN_USER:-admin} + - NEXTCLOUD_ADMIN_PASSWORD=${NEXTCLOUD_ADMIN_PASSWORD} + - NEXTCLOUD_TRUSTED_DOMAINS=${DOMAIN} + - TRUSTED_PROXIES=172.18.0.0/16 + - OVERWRITEPROTOCOL=https + depends_on: + - nextcloud-db + labels: + - "homelab.category=productivity" + - "homelab.description=File sync and collaboration" + - "traefik.enable=true" + - "traefik.http.routers.nextcloud.rule=Host(`nextcloud.${DOMAIN}`)" + - "traefik.http.routers.nextcloud.entrypoints=websecure" + - "traefik.http.routers.nextcloud.tls.certresolver=letsencrypt" + - "traefik.http.routers.nextcloud.middlewares=authelia@docker" + - "traefik.http.services.nextcloud.loadbalancer.server.port=80" + + nextcloud-db: + image: mariadb:10.11 + container_name: nextcloud-db + restart: unless-stopped + networks: + - nextcloud-network + volumes: + - nextcloud-db-data:/var/lib/mysql + environment: + - MYSQL_ROOT_PASSWORD=${NEXTCLOUD_DB_ROOT_PASSWORD} + - MYSQL_DATABASE=nextcloud + - MYSQL_USER=nextcloud + - MYSQL_PASSWORD=${NEXTCLOUD_DB_PASSWORD} + command: --transaction-isolation=READ-COMMITTED --log-bin=binlog --binlog-format=ROW + labels: + - "homelab.category=productivity" + - "homelab.description=Nextcloud database" + + # Mealie - Recipe manager + # Access at: https://mealie.${DOMAIN} + mealie: + image: ghcr.io/mealie-recipes/mealie:latest + container_name: mealie + restart: unless-stopped + networks: + - homelab-network + - traefik-network + volumes: + - /opt/stacks/mealie/data:/app/data + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + - BASE_URL=https://mealie.${DOMAIN} + - DB_ENGINE=sqlite + labels: + - "homelab.category=productivity" + - "homelab.description=Recipe manager and meal planner" + - "traefik.enable=true" + - "traefik.http.routers.mealie.rule=Host(`mealie.${DOMAIN}`)" + - "traefik.http.routers.mealie.entrypoints=websecure" + - "traefik.http.routers.mealie.tls.certresolver=letsencrypt" + - "traefik.http.services.mealie.loadbalancer.server.port=9000" + # No Authelia - family members should access easily + + # WordPress - Blog/website platform + # Access at: https://blog.${DOMAIN} + wordpress: + image: wordpress:latest + container_name: wordpress + restart: unless-stopped + networks: + - homelab-network + - traefik-network + - wordpress-network + volumes: + - /opt/stacks/wordpress/html:/var/www/html + environment: + - WORDPRESS_DB_HOST=wordpress-db + - WORDPRESS_DB_USER=wordpress + - WORDPRESS_DB_PASSWORD=${WORDPRESS_DB_PASSWORD} + - WORDPRESS_DB_NAME=wordpress + depends_on: + - wordpress-db + labels: + - "homelab.category=productivity" + - "homelab.description=Blog and website platform" + - "traefik.enable=true" + - "traefik.http.routers.wordpress.rule=Host(`blog.${DOMAIN}`)" + - "traefik.http.routers.wordpress.entrypoints=websecure" + - "traefik.http.routers.wordpress.tls.certresolver=letsencrypt" + - "traefik.http.services.wordpress.loadbalancer.server.port=80" + # No Authelia - public blog + + wordpress-db: + image: mariadb:10.11 + container_name: wordpress-db + restart: unless-stopped + networks: + - wordpress-network + volumes: + - wordpress-db-data:/var/lib/mysql + environment: + - MYSQL_ROOT_PASSWORD=${WORDPRESS_DB_ROOT_PASSWORD} + - MYSQL_DATABASE=wordpress + - MYSQL_USER=wordpress + - MYSQL_PASSWORD=${WORDPRESS_DB_PASSWORD} + labels: + - "homelab.category=productivity" + - "homelab.description=WordPress database" + + # Gitea - Self-hosted Git service + # Access at: https://git.${DOMAIN} + gitea: + image: gitea/gitea:latest + container_name: gitea + restart: unless-stopped + networks: + - homelab-network + - traefik-network + - gitea-network + volumes: + - /opt/stacks/gitea/data:/data + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + environment: + - USER_UID=${PUID:-1000} + - USER_GID=${PGID:-1000} + - GITEA__database__DB_TYPE=postgres + - GITEA__database__HOST=gitea-db:5432 + - GITEA__database__NAME=gitea + - GITEA__database__USER=gitea + - GITEA__database__PASSWD=${GITEA_DB_PASSWORD} + depends_on: + - gitea-db + labels: + - "homelab.category=productivity" + - "homelab.description=Self-hosted Git service" + - "traefik.enable=true" + - "traefik.http.routers.gitea.rule=Host(`git.${DOMAIN}`)" + - "traefik.http.routers.gitea.entrypoints=websecure" + - "traefik.http.routers.gitea.tls.certresolver=letsencrypt" + - "traefik.http.routers.gitea.middlewares=authelia@docker" + - "traefik.http.services.gitea.loadbalancer.server.port=3000" + + gitea-db: + image: postgres:14-alpine + container_name: gitea-db + restart: unless-stopped + networks: + - gitea-network + volumes: + - gitea-db-data:/var/lib/postgresql/data + environment: + - POSTGRES_USER=gitea + - POSTGRES_PASSWORD=${GITEA_DB_PASSWORD} + - POSTGRES_DB=gitea + labels: + - "homelab.category=productivity" + - "homelab.description=Gitea database" + + # DokuWiki - Wiki without database + # Access at: https://wiki.${DOMAIN} + dokuwiki: + image: lscr.io/linuxserver/dokuwiki:latest + container_name: dokuwiki + restart: unless-stopped + networks: + - homelab-network + - traefik-network + volumes: + - /opt/stacks/dokuwiki/config:/config + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + labels: + - "homelab.category=productivity" + - "homelab.description=File-based wiki" + - "traefik.enable=true" + - "traefik.http.routers.dokuwiki.rule=Host(`wiki.${DOMAIN}`)" + - "traefik.http.routers.dokuwiki.entrypoints=websecure" + - "traefik.http.routers.dokuwiki.tls.certresolver=letsencrypt" + - "traefik.http.routers.dokuwiki.middlewares=authelia@docker" + - "traefik.http.services.dokuwiki.loadbalancer.server.port=80" + + # BookStack - Documentation platform + # Access at: https://docs.${DOMAIN} + bookstack: + image: lscr.io/linuxserver/bookstack:latest + container_name: bookstack + restart: unless-stopped + networks: + - homelab-network + - traefik-network + - bookstack-network + volumes: + - /opt/stacks/bookstack/config:/config + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - APP_URL=https://docs.${DOMAIN} + - DB_HOST=bookstack-db + - DB_PORT=3306 + - DB_DATABASE=bookstack + - DB_USERNAME=bookstack + - DB_PASSWORD=${BOOKSTACK_DB_PASSWORD} + depends_on: + - bookstack-db + labels: + - "homelab.category=productivity" + - "homelab.description=Documentation and wiki platform" + - "traefik.enable=true" + - "traefik.http.routers.bookstack.rule=Host(`docs.${DOMAIN}`)" + - "traefik.http.routers.bookstack.entrypoints=websecure" + - "traefik.http.routers.bookstack.tls.certresolver=letsencrypt" + - "traefik.http.routers.bookstack.middlewares=authelia@docker" + - "traefik.http.services.bookstack.loadbalancer.server.port=80" + + bookstack-db: + image: mariadb:10.11 + container_name: bookstack-db + restart: unless-stopped + networks: + - bookstack-network + volumes: + - bookstack-db-data:/var/lib/mysql + environment: + - MYSQL_ROOT_PASSWORD=${BOOKSTACK_DB_ROOT_PASSWORD} + - MYSQL_DATABASE=bookstack + - MYSQL_USER=bookstack + - MYSQL_PASSWORD=${BOOKSTACK_DB_PASSWORD} + labels: + - "homelab.category=productivity" + - "homelab.description=BookStack database" + + # MediaWiki - Wiki platform + # Access at: https://mediawiki.${DOMAIN} + mediawiki: + image: mediawiki:latest + container_name: mediawiki + restart: unless-stopped + networks: + - homelab-network + - traefik-network + - mediawiki-network + volumes: + - /opt/stacks/mediawiki/images:/var/www/html/images + - /opt/stacks/mediawiki/LocalSettings.php:/var/www/html/LocalSettings.php + environment: + - MEDIAWIKI_DB_HOST=mediawiki-db + - MEDIAWIKI_DB_NAME=mediawiki + - MEDIAWIKI_DB_USER=mediawiki + - MEDIAWIKI_DB_PASSWORD=${MEDIAWIKI_DB_PASSWORD} + depends_on: + - mediawiki-db + labels: + - "homelab.category=productivity" + - "homelab.description=MediaWiki platform" + - "traefik.enable=true" + - "traefik.http.routers.mediawiki.rule=Host(`mediawiki.${DOMAIN}`)" + - "traefik.http.routers.mediawiki.entrypoints=websecure" + - "traefik.http.routers.mediawiki.tls.certresolver=letsencrypt" + - "traefik.http.routers.mediawiki.middlewares=authelia@docker" + - "traefik.http.services.mediawiki.loadbalancer.server.port=80" + + mediawiki-db: + image: mariadb:10.11 + container_name: mediawiki-db + restart: unless-stopped + networks: + - mediawiki-network + volumes: + - mediawiki-db-data:/var/lib/mysql + environment: + - MYSQL_ROOT_PASSWORD=${MEDIAWIKI_DB_ROOT_PASSWORD} + - MYSQL_DATABASE=mediawiki + - MYSQL_USER=mediawiki + - MYSQL_PASSWORD=${MEDIAWIKI_DB_PASSWORD} + labels: + - "homelab.category=productivity" + - "homelab.description=MediaWiki database" + +volumes: + nextcloud-db-data: + wordpress-db-data: + gitea-db-data: + bookstack-db-data: + mediawiki-db-data: + +networks: + homelab-network: + external: true + traefik-network: + external: true + nextcloud-network: + driver: bridge + wordpress-network: + driver: bridge + gitea-network: + driver: bridge + bookstack-network: + driver: bridge + mediawiki-network: + driver: bridge diff --git a/docker-compose/utilities.yml b/docker-compose/utilities.yml new file mode 100644 index 0000000..95003ee --- /dev/null +++ b/docker-compose/utilities.yml @@ -0,0 +1,177 @@ +# Backup and Utility Services +# Place in /opt/stacks/utilities/docker-compose.yml + +services: + # Backrest - Backup solution for restic + # Access at: https://backrest.${DOMAIN} + backrest: + image: garethgeorge/backrest:latest + container_name: backrest + restart: unless-stopped + networks: + - homelab-network + - traefik-network + volumes: + - /opt/stacks/backrest/data:/data + - /opt/stacks/backrest/config:/config + - /opt/stacks:/opt/stacks:ro # Backup source + - /mnt:/mnt:ro # Backup additional drives + - backrest-cache:/cache + environment: + - BACKREST_DATA=/data + - BACKREST_CONFIG=/config/config.json + - TZ=${TZ} + labels: + - "homelab.category=utilities" + - "homelab.description=Backup management with restic" + - "traefik.enable=true" + - "traefik.http.routers.backrest.rule=Host(`backrest.${DOMAIN}`)" + - "traefik.http.routers.backrest.entrypoints=websecure" + - "traefik.http.routers.backrest.tls.certresolver=letsencrypt" + - "traefik.http.routers.backrest.middlewares=authelia@docker" + - "traefik.http.services.backrest.loadbalancer.server.port=9898" + + # Duplicati - Backup solution + # Access at: https://duplicati.${DOMAIN} + duplicati: + image: lscr.io/linuxserver/duplicati:latest + container_name: duplicati + restart: unless-stopped + networks: + - homelab-network + - traefik-network + volumes: + - /opt/stacks/duplicati/config:/config + - /opt/stacks:/source/stacks:ro + - /mnt:/source/mnt:ro + - /mnt/backups:/backups + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + labels: + - "homelab.category=utilities" + - "homelab.description=Backup software with encryption" + - "traefik.enable=true" + - "traefik.http.routers.duplicati.rule=Host(`duplicati.${DOMAIN}`)" + - "traefik.http.routers.duplicati.entrypoints=websecure" + - "traefik.http.routers.duplicati.tls.certresolver=letsencrypt" + - "traefik.http.routers.duplicati.middlewares=authelia@docker" + - "traefik.http.services.duplicati.loadbalancer.server.port=8200" + + # Uptime Kuma - Status monitoring + # Access at: https://status.${DOMAIN} + uptime-kuma: + image: louislam/uptime-kuma:1 + container_name: uptime-kuma + restart: unless-stopped + networks: + - homelab-network + - traefik-network + volumes: + - /opt/stacks/uptime-kuma/data:/app/data + - /var/run/docker.sock:/var/run/docker.sock:ro + labels: + - "homelab.category=utilities" + - "homelab.description=Uptime monitoring and status page" + - "traefik.enable=true" + - "traefik.http.routers.uptime-kuma.rule=Host(`status.${DOMAIN}`)" + - "traefik.http.routers.uptime-kuma.entrypoints=websecure" + - "traefik.http.routers.uptime-kuma.tls.certresolver=letsencrypt" + - "traefik.http.services.uptime-kuma.loadbalancer.server.port=3001" + # No Authelia - public status page + + # Code Server - VS Code in browser + # Access at: https://code.${DOMAIN} + code-server: + image: lscr.io/linuxserver/code-server:latest + container_name: code-server + restart: unless-stopped + networks: + - homelab-network + - traefik-network + volumes: + - /opt/stacks/code-server/config:/config + - /opt/stacks:/opt/stacks # Access to all stacks + - /mnt:/mnt:ro # Read-only access to data + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + - PASSWORD=${CODE_SERVER_PASSWORD} + - SUDO_PASSWORD=${CODE_SERVER_SUDO_PASSWORD} + labels: + - "homelab.category=utilities" + - "homelab.description=VS Code in browser" + - "traefik.enable=true" + - "traefik.http.routers.code-server.rule=Host(`code.${DOMAIN}`)" + - "traefik.http.routers.code-server.entrypoints=websecure" + - "traefik.http.routers.code-server.tls.certresolver=letsencrypt" + - "traefik.http.routers.code-server.middlewares=authelia@docker" + - "traefik.http.services.code-server.loadbalancer.server.port=8443" + + # Form.io - Form builder (if needed) + # Access at: https://forms.${DOMAIN} + formio: + image: formio/formio:latest + container_name: formio + restart: unless-stopped + networks: + - homelab-network + - traefik-network + - formio-network + environment: + - MONGO_URL=mongodb://formio-mongo:27017/formio + - JWT_SECRET=${FORMIO_JWT_SECRET} + - DB_SECRET=${FORMIO_DB_SECRET} + depends_on: + - formio-mongo + labels: + - "homelab.category=utilities" + - "homelab.description=Form builder platform" + - "traefik.enable=true" + - "traefik.http.routers.formio.rule=Host(`forms.${DOMAIN}`)" + - "traefik.http.routers.formio.entrypoints=websecure" + - "traefik.http.routers.formio.tls.certresolver=letsencrypt" + - "traefik.http.routers.formio.middlewares=authelia@docker" + - "traefik.http.services.formio.loadbalancer.server.port=3000" + + formio-mongo: + image: mongo:6 + container_name: formio-mongo + restart: unless-stopped + networks: + - formio-network + volumes: + - formio-mongo-data:/data/db + labels: + - "homelab.category=utilities" + - "homelab.description=Form.io database" + + # Authelia Redis - Session storage for Authelia + # No web UI - backend service + authelia-redis: + image: redis:alpine + container_name: authelia-redis + restart: unless-stopped + networks: + - homelab-network + volumes: + - authelia-redis-data:/data + command: redis-server --save 60 1 --loglevel warning + labels: + - "homelab.category=utilities" + - "homelab.description=Session storage for Authelia" + +volumes: + backrest-cache: + formio-mongo-data: + authelia-redis-data: + +networks: + homelab-network: + external: true + traefik-network: + external: true + formio-network: + driver: bridge diff --git a/docs/proxying-external-hosts.md b/docs/proxying-external-hosts.md new file mode 100644 index 0000000..5e5d3d0 --- /dev/null +++ b/docs/proxying-external-hosts.md @@ -0,0 +1,379 @@ +# Proxying External Hosts with Traefik and Authelia + +This guide explains how to use Traefik and Authelia to proxy external services (like a Raspberry Pi running Home Assistant) through your domain with HTTPS and optional SSO protection. + +## Overview + +Traefik can proxy services that aren't running in Docker, such as: +- Home Assistant on a Raspberry Pi +- Other physical servers on your network +- Services running on different machines +- Any HTTP/HTTPS service accessible via IP:PORT + +## Method 1: Using Traefik File Provider (Recommended) + +### Step 1: Create External Service Configuration + +Create a file in `/opt/stacks/traefik/dynamic/external-hosts.yml`: + +```yaml +http: + routers: + # Home Assistant on Raspberry Pi + homeassistant-external: + rule: "Host(`ha.yourdomain.duckdns.org`)" + entryPoints: + - websecure + service: homeassistant-external + tls: + certResolver: letsencrypt + # Uncomment to add Authelia protection: + # middlewares: + # - authelia@docker + + services: + homeassistant-external: + loadBalancer: + servers: + - url: "http://192.168.1.50:8123" # Replace with your Pi's IP and port + passHostHeader: true + + middlewares: + # Optional: Add headers for WebSocket support + homeassistant-headers: + headers: + customRequestHeaders: + X-Forwarded-Proto: "https" + customResponseHeaders: + X-Frame-Options: "SAMEORIGIN" +``` + +### Step 2: Reload Traefik + +Traefik watches the `/opt/stacks/traefik/dynamic/` directory automatically and reloads configurations: + +```bash +# Verify configuration is loaded +docker logs traefik | grep external-hosts + +# If needed, restart Traefik +cd /opt/stacks/traefik +docker compose restart +``` + +### Step 3: Test Access + +Visit `https://ha.yourdomain.duckdns.org` - Traefik will: +1. Accept the HTTPS connection +2. Proxy the request to `http://192.168.1.50:8123` +3. Return the response with proper SSL +4. (Optionally) Require Authelia login if middleware is configured + +## Method 2: Using Docker Labels (Dummy Container) + +If you prefer managing routes via Docker labels (so the AI can modify them), create a dummy container: + +### Create a Label Container + +In `/opt/stacks/external-proxies/docker-compose.yml`: + +```yaml +services: + # Dummy container for Raspberry Pi Home Assistant + homeassistant-proxy-labels: + image: alpine:latest + container_name: homeassistant-proxy-labels + command: tail -f /dev/null # Keep container running + restart: unless-stopped + networks: + - traefik-network + labels: + - "traefik.enable=true" + - "traefik.http.routers.ha-external.rule=Host(`ha.${DOMAIN}`)" + - "traefik.http.routers.ha-external.entrypoints=websecure" + - "traefik.http.routers.ha-external.tls.certresolver=letsencrypt" + # Point to external service + - "traefik.http.services.ha-external.loadbalancer.server.url=http://192.168.1.50:8123" + # Optional: Add Authelia (usually not for HA) + # - "traefik.http.routers.ha-external.middlewares=authelia@docker" + +networks: + traefik-network: + external: true +``` + +Deploy: +```bash +cd /opt/stacks/external-proxies +docker compose up -d +``` + +## Method 3: Hybrid Approach (File + Docker Discovery) + +Combine both methods for maximum flexibility: +- Use file provider for static external hosts +- Use Docker labels for frequently changing services +- AI can manage both! + +## Common External Services to Proxy + +### Home Assistant (Raspberry Pi) +```yaml +homeassistant-pi: + rule: "Host(`ha.yourdomain.duckdns.org`)" + service: http://192.168.1.50:8123 + # No Authelia - HA has its own auth +``` + +### Router/Firewall Admin Panel +```yaml +router-admin: + rule: "Host(`router.yourdomain.duckdns.org`)" + service: http://192.168.1.1:80 + middlewares: + - authelia@docker # Add SSO protection +``` + +### Proxmox Server +```yaml +proxmox: + rule: "Host(`proxmox.yourdomain.duckdns.org`)" + service: https://192.168.1.100:8006 + middlewares: + - authelia@docker + # Note: Use https:// if backend uses HTTPS +``` + +### TrueNAS/FreeNAS +```yaml +truenas: + rule: "Host(`nas.yourdomain.duckdns.org`)" + service: http://192.168.1.200:80 + middlewares: + - authelia@docker +``` + +### Security Camera NVR +```yaml +nvr: + rule: "Host(`cameras.yourdomain.duckdns.org`)" + service: http://192.168.1.10:80 + middlewares: + - authelia@docker # Definitely protect cameras! +``` + +## Advanced Configuration + +### WebSocket Support + +Some services (like Home Assistant) need WebSocket support: + +```yaml +http: + middlewares: + websocket-headers: + headers: + customRequestHeaders: + X-Forwarded-Proto: "https" + Connection: "upgrade" + Upgrade: "websocket" + + routers: + homeassistant-external: + middlewares: + - websocket-headers +``` + +### HTTPS Backend + +If your external service already uses HTTPS: + +```yaml +http: + services: + https-backend: + loadBalancer: + servers: + - url: "https://192.168.1.50:8123" + serversTransport: insecureTransport + + serversTransports: + insecureTransport: + insecureSkipVerify: true # Only if using self-signed cert +``` + +### IP Whitelist + +Restrict access to specific IPs: + +```yaml +http: + middlewares: + local-only: + ipWhiteList: + sourceRange: + - "192.168.1.0/24" + - "10.0.0.0/8" + + routers: + sensitive-service: + middlewares: + - local-only + - authelia@docker +``` + +## Authelia Bypass Rules + +Configure Authelia to bypass authentication for specific external hosts. + +Edit `/opt/stacks/authelia/configuration.yml`: + +```yaml +access_control: + rules: + # Bypass for Home Assistant (app access) + - domain: ha.yourdomain.duckdns.org + policy: bypass + + # Require auth for router admin + - domain: router.yourdomain.duckdns.org + policy: one_factor + + # Two-factor for critical services + - domain: proxmox.yourdomain.duckdns.org + policy: two_factor +``` + +## DNS Configuration + +Ensure your DuckDNS domain points to your public IP: + +1. DuckDNS container automatically updates your IP +2. Port forward 80 and 443 to your Traefik server +3. All subdomains (`*.yourdomain.duckdns.org`) point to same IP +4. Traefik routes based on Host header + +## Troubleshooting + +### Check Traefik Routing +```bash +# View active routes +docker logs traefik | grep "Creating router" + +# Check if external host route is loaded +docker logs traefik | grep homeassistant + +# View Traefik dashboard +# Visit: https://traefik.yourdomain.duckdns.org +``` + +### Test Without SSL +```bash +# Temporarily test direct connection +curl -H "Host: ha.yourdomain.duckdns.org" http://localhost/ +``` + +### Check Authelia Logs +```bash +cd /opt/stacks/authelia +docker compose logs -f authelia +``` + +### Verify External Service +```bash +# Test that external service is reachable +curl http://192.168.1.50:8123 +``` + +## AI Management + +The AI can manage external host proxying by: + +1. **Reading existing configurations**: Parse `/opt/stacks/traefik/dynamic/*.yml` +2. **Adding new routes**: Create/update YAML files in dynamic directory +3. **Modifying Docker labels**: Update dummy container labels +4. **Configuring Authelia rules**: Edit `configuration.yml` for bypass/require auth +5. **Testing connectivity**: Suggest verification steps + +Example AI prompt: +> "Add proxying for my Unifi Controller at 192.168.1.5:8443 with Authelia protection" + +AI will: +1. Create route configuration file +2. Add HTTPS backend support (Unifi uses HTTPS) +3. Configure Authelia middleware +4. Add to Homepage dashboard +5. Provide testing instructions + +## Security Best Practices + +1. **Always use Authelia** for admin interfaces (routers, NAS, etc.) +2. **Bypass Authelia** only for services with their own auth (HA, Plex) +3. **Use IP whitelist** for highly sensitive services +4. **Enable two-factor** for critical infrastructure +5. **Monitor access logs** in Traefik and Authelia +6. **Keep services updated** - Traefik, Authelia, and external services + +## Example: Complete External Host Setup + +Let's proxy a Raspberry Pi Home Assistant: + +1. **Traefik configuration** (`/opt/stacks/traefik/dynamic/raspberry-pi.yml`): +```yaml +http: + routers: + ha-pi: + rule: "Host(`ha.yourdomain.duckdns.org`)" + entryPoints: + - websecure + service: ha-pi + tls: + certResolver: letsencrypt + middlewares: + - ha-headers + + services: + ha-pi: + loadBalancer: + servers: + - url: "http://192.168.1.50:8123" + + middlewares: + ha-headers: + headers: + customRequestHeaders: + X-Forwarded-Proto: "https" +``` + +2. **Authelia bypass** (in `/opt/stacks/authelia/configuration.yml`): +```yaml +access_control: + rules: + - domain: ha.yourdomain.duckdns.org + policy: bypass +``` + +3. **Homepage entry** (in `/opt/stacks/homepage/config/services.yaml`): +```yaml +- Home Automation: + - Home Assistant (Pi): + icon: home-assistant.png + href: https://ha.yourdomain.duckdns.org + description: HA on Raspberry Pi + ping: 192.168.1.50 + widget: + type: homeassistant + url: http://192.168.1.50:8123 + key: your-long-lived-token +``` + +4. **Test**: +```bash +# Reload Traefik (automatic, but verify) +docker logs traefik | grep ha-pi + +# Visit +https://ha.yourdomain.duckdns.org +``` + +Done! Your Raspberry Pi Home Assistant is now accessible via your domain with HTTPS. 🎉 From 6fec6b501e23f79987c28622811ed01608037065 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 12 Jan 2026 01:16:12 +0000 Subject: [PATCH 11/16] Update all documentation to reflect 60+ services, Dockge, Traefik, Authelia, and new features - Update README.md with comprehensive overview of all 40+ services - Add Quick Setup guide with Dockge deployment steps - List all service stacks by category (Infrastructure, Media, Home Automation, Productivity, etc.) - Update AI assistant examples for Traefik, Authelia, VPN routing - Add network architecture with Traefik routing patterns - Update common operations for Dockge structure - Add storage strategy (opt/stacks vs /mnt/) - Update configuration management section - Add proxying external hosts overview - Update getting started checklist with all new steps - Update docker-guidelines.md with Dockge and Traefik sections - Add comprehensive Dockge structure explanation - Add Traefik and Authelia integration patterns - Document when to use SSO vs bypass - Add VPN routing patterns with Gluetun - Update storage strategy recommendations - Completely rewrite getting-started.md with step-by-step Dockge deployment - Add DuckDNS signup instructions - Add detailed core infrastructure deployment (DuckDNS, Traefik, Authelia, Dockge) - Add Homepage configuration steps - Add API key configuration for widgets - Add troubleshooting section - Add security checklist - Create docs/services-reference.md - comprehensive catalog of all 60+ services - Organized by category with descriptions - Access URLs and SSO requirements - Storage recommendations - Quick deployment guide - Summary table by stack All documentation now accurately reflects the complete homelab setup with Dockge, Traefik, Authelia, Gluetun, Homepage, and all 40+ services across 10 stack files. Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- README.md | 559 +++++++++++++++++++++++++--------- docs/docker-guidelines.md | 182 +++++++++-- docs/getting-started.md | 603 ++++++++++++++++--------------------- docs/services-reference.md | 462 ++++++++++++++++++++++++++++ 4 files changed, 1297 insertions(+), 509 deletions(-) create mode 100644 docs/services-reference.md diff --git a/README.md b/README.md index fd60268..b0f9f4a 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,23 @@ AI-Powered Homelab Administration with GitHub Copilot ## Overview -This repository provides a structured approach to managing a homelab infrastructure using Docker Compose, with integrated AI assistance through GitHub Copilot. The AI assistant is specifically trained to help you create, modify, and manage Docker services while maintaining consistency across your entire server stack. +This repository provides a comprehensive, production-ready homelab infrastructure using Docker Compose with Dockge, featuring 40+ pre-configured services. Integrated AI assistance through GitHub Copilot helps you create, modify, and manage Docker services while maintaining consistency across your entire server stack. + +The infrastructure uses Traefik for reverse proxy with automatic SSL, Authelia for Single Sign-On, Gluetun for VPN routing, and DuckDNS for dynamic DNS - all managed through file-based configurations that the AI can modify. ## Features - **AI-Powered Management**: GitHub Copilot integration with specialized instructions for Docker service management -- **Docker Compose First**: All persistent services defined in organized compose files -- **Consistent Structure**: Standardized naming conventions and patterns across all services +- **Dockge Structure**: All stacks organized in `/opt/stacks/` for easy management via Dockge +- **40+ Pre-configured Services**: Production-ready compose files across infrastructure, media, home automation, productivity, and monitoring +- **Traefik Reverse Proxy**: Automatic HTTPS with Let's Encrypt via file-based configuration (no web UI needed) +- **Authelia SSO**: Single Sign-On protection for all admin interfaces with smart bypass rules for media apps +- **Gluetun VPN**: Surfshark WireGuard integration for secure downloads +- **Homepage Dashboard**: AI-configurable dashboard with Docker integration and service widgets +- **External Host Proxying**: Proxy external services (Raspberry Pi, routers, NAS) through Traefik - **Stack-Aware Changes**: AI considers the entire infrastructure when making changes -- **Comprehensive Documentation**: Detailed guidelines and examples for all operations -- **Example Services**: Ready-to-use compose files for common homelab services +- **Comprehensive Documentation**: Detailed guidelines including proxying external hosts +- **File-Based Configuration**: Everything managed via YAML files - no web UI dependencies ## Quick Start @@ -23,8 +30,10 @@ This repository provides a structured approach to managing a homelab infrastruct - Docker Compose V2 - Git - VS Code with GitHub Copilot extension (for AI assistance) +- A domain from DuckDNS (free) +- Surfshark VPN account (optional, for VPN features) -### Initial Setup +### Quick Setup (Dockge Structure) 1. **Clone the repository:** ```bash @@ -36,45 +45,90 @@ This repository provides a structured approach to managing a homelab infrastruct 2. **Create environment file:** ```bash cp .env.example .env - # Edit .env with your values + # Edit .env with your values (domain, API keys, passwords) nano .env ``` -3. **Create the main network:** +3. **Create required directories:** + ```bash + sudo mkdir -p /opt/stacks + sudo chown -R $USER:$USER /opt/stacks + ``` + +4. **Create Docker networks:** ```bash docker network create homelab-network + docker network create traefik-network + docker network create media-network ``` -4. **Create config directories:** +5. **Deploy core infrastructure (in order):** ```bash - mkdir -p config + # 1. DuckDNS (Dynamic DNS) + mkdir -p /opt/stacks/duckdns && cp docker-compose/duckdns.yml /opt/stacks/duckdns/docker-compose.yml + cd /opt/stacks/duckdns && docker compose up -d + + # 2. Traefik (Reverse Proxy with SSL) + mkdir -p /opt/stacks/traefik/dynamic + cp docker-compose/traefik.yml /opt/stacks/traefik/docker-compose.yml + cp config-templates/traefik/* /opt/stacks/traefik/ + cd /opt/stacks/traefik && docker compose up -d + + # 3. Authelia (SSO) + mkdir -p /opt/stacks/authelia + cp docker-compose/authelia.yml /opt/stacks/authelia/docker-compose.yml + cp config-templates/authelia/* /opt/stacks/authelia/ + cd /opt/stacks/authelia && docker compose up -d + + # 4. Dockge (Stack Manager) + mkdir -p /opt/stacks/infrastructure + cp docker-compose/infrastructure.yml /opt/stacks/infrastructure/docker-compose.yml + cd /opt/stacks/infrastructure && docker compose up -d dockge ``` -5. **Start your first service:** - ```bash - # Example: Start Portainer for container management - docker compose -f docker-compose/infrastructure.yml up -d portainer - ``` - -6. **Access the service:** - Open `http://your-server-ip:9000` in your browser +6. **Access Dockge:** + Open `https://dockge.yourdomain.duckdns.org` (use Authelia login) + + Now deploy remaining stacks through Dockge's UI! ## Repository Structure ``` AI-Homelab/ ├── .github/ -│ └── copilot-instructions.md # AI assistant guidelines +│ └── copilot-instructions.md # AI assistant guidelines (Dockge, Traefik, Authelia aware) ├── docker-compose/ -│ ├── infrastructure.yml # Core services (proxy, DNS, Portainer) -│ ├── media.yml # Media services (Plex, Sonarr, Radarr) -│ ├── monitoring.yml # Observability (Prometheus, Grafana) -│ ├── development.yml # Dev tools (code-server, databases) -│ └── README.md # Docker Compose documentation +│ ├── traefik.yml # Reverse proxy (deploy first) +│ ├── authelia.yml # SSO authentication +│ ├── duckdns.yml # Dynamic DNS +│ ├── gluetun.yml # VPN client (Surfshark) + qBittorrent +│ ├── infrastructure.yml # Dockge, Portainer, Pi-hole, Watchtower, Dozzle, Glances +│ ├── dashboards.yml # Homepage, Homarr +│ ├── media.yml # Plex, Jellyfin, Sonarr, Radarr, Prowlarr +│ ├── media-extended.yml # Readarr, Lidarr, Lazy Librarian, Mylar3, Calibre-Web, +│ │ # Jellyseerr, FlareSolverr, Tdarr, Unmanic +│ ├── homeassistant.yml # Home Assistant, ESPHome, TasmoAdmin, Node-RED, +│ │ # Mosquitto, Zigbee2MQTT, MotionEye +│ ├── productivity.yml # Nextcloud, Mealie, WordPress, Gitea, DokuWiki, +│ │ # BookStack, MediaWiki (all with databases) +│ ├── utilities.yml # Backrest, Duplicati, Uptime Kuma, Code Server, +│ │ # Form.io, Authelia-Redis +│ ├── monitoring.yml # Prometheus, Grafana, Loki, Promtail, cAdvisor +│ ├── development.yml # GitLab, PostgreSQL, Redis, pgAdmin, Jupyter +│ └── README-dockge.md # Dockge deployment guide +├── config-templates/ +│ ├── traefik/ # Traefik static and dynamic configs +│ ├── authelia/ # Authelia config and user database +│ ├── homepage/ # Homepage dashboard configs (with widgets) +│ ├── prometheus/ # Prometheus scrape configs +│ ├── loki/ # Loki log aggregation config +│ └── ... # Other service templates ├── docs/ -│ └── docker-guidelines.md # Comprehensive Docker guidelines -├── config/ # Service configurations (gitignored) -├── .env.example # Environment variable template +│ ├── docker-guidelines.md # Comprehensive Docker guidelines +│ ├── getting-started.md # Step-by-step setup guide +│ ├── quick-reference.md # Command reference +│ └── proxying-external-hosts.md # Guide for proxying Raspberry Pi, routers, etc. +├── .env.example # Environment variable template (40+ vars) ├── .gitignore # Git ignore patterns └── README.md # This file ``` @@ -87,193 +141,384 @@ AI-Homelab/ 2. **Open this repository** in VS Code 3. **Start Copilot Chat** and ask questions like: - "Help me add a new media service to my homelab" - - "Create a docker-compose file for Home Assistant" - - "How do I configure GPU support for Plex?" - - "Check my compose file for port conflicts" + - "Configure Traefik routing for my new service" + - "Add Authelia SSO protection to this service" + - "How do I proxy my Raspberry Pi through Traefik?" + - "Create a Homepage widget for this service" + - "Route this download client through Gluetun VPN" The AI assistant automatically follows the guidelines in `.github/copilot-instructions.md` to: +- Use `/opt/stacks/` directory structure (Dockge compatible) +- Configure Traefik labels for automatic routing +- Apply Authelia middleware where appropriate +- Suggest `/mnt/` for large data storage +- Add services to Homepage dashboard with widgets - Maintain consistency with existing services -- Use Docker Compose for all persistent services - Consider the entire stack when making changes -- Follow naming conventions and best practices ### Example Interactions -**Creating a new service:** +**Adding a new service:** ``` -You: "I want to add Home Assistant to my homelab" +You: "Add Tautulli to monitor my Plex server" -Copilot: [Analyzes existing stack, checks for conflicts, creates compose configuration] -- Checks available ports -- Uses consistent naming -- Connects to appropriate networks -- Follows established patterns +Copilot: [Creates compose configuration with]: +- /opt/stacks/tautulli/ directory structure +- Traefik labels for HTTPS access +- Authelia middleware for SSO protection +- Homepage dashboard entry with widget +- Connection to existing Plex service ``` -**Modifying a service:** +**Proxying external service:** ``` -You: "Add GPU support to my Plex container" +You: "Proxy my Raspberry Pi Home Assistant through Traefik" -Copilot: [Reviews current Plex configuration and updates it] -- Checks if NVIDIA runtime is available -- Updates device mappings -- Adds required environment variables -- Maintains existing configuration +Copilot: [Creates Traefik route configuration]: +- File in /opt/stacks/traefik/dynamic/ +- HTTPS with Let's Encrypt +- Authelia bypass (HA has its own auth) +- WebSocket support +- Homepage dashboard entry +``` + +**Configuring VPN routing:** +``` +You: "Route SABnzbd through the VPN" + +Copilot: [Updates compose to use Gluetun]: +- network_mode: "service:gluetun" +- Exposes ports through Gluetun +- Maintains Traefik routing +- Updates documentation ``` ## Available Service Stacks -### Infrastructure (`infrastructure.yml`) -- **Nginx Proxy Manager**: Web-based reverse proxy with SSL -- **Pi-hole**: Network-wide ad blocking and DNS -- **Portainer**: Docker container management UI -- **Watchtower**: Automatic container updates +### Core Infrastructure (Required) -### Media (`media.yml`) -- **Plex**: Media streaming server -- **Jellyfin**: Open-source media server alternative -- **Sonarr**: TV show automation -- **Radarr**: Movie automation -- **Prowlarr**: Indexer manager -- **qBittorrent**: Torrent client +#### 1. Traefik (`traefik.yml`) +**Reverse proxy with automatic SSL** - Deploy first! +- Automatic HTTPS via Let's Encrypt +- File-based and Docker label routing +- HTTP to HTTPS redirect +- Dashboard at `https://traefik.${DOMAIN}` + +#### 2. Authelia (`authelia.yml`) +**Single Sign-On authentication** +- TOTP 2FA support +- LDAP/file-based user database +- Smart bypass rules for media apps +- Login at `https://auth.${DOMAIN}` + +#### 3. DuckDNS (`duckdns.yml`) +**Dynamic DNS updater** +- Automatic IP updates +- Integrates with Let's Encrypt +- No web UI - runs silently + +#### 4. Gluetun (`gluetun.yml`) +**VPN client (Surfshark WireGuard)** +- Includes qBittorrent +- Download via `https://qbit.${DOMAIN}` +- Easy to route other services through VPN + +### Infrastructure Tools (`infrastructure.yml`) + +- **Dockge**: Docker Compose stack manager (PRIMARY) - `https://dockge.${DOMAIN}` +- **Portainer**: Docker management UI (secondary) - `https://portainer.${DOMAIN}` +- **Pi-hole**: Network-wide ad blocking - `https://pihole.${DOMAIN}` +- **Watchtower**: Automatic container updates +- **Dozzle**: Real-time Docker logs - `https://dozzle.${DOMAIN}` +- **Glances**: System monitoring - `https://glances.${DOMAIN}` +- **Docker Proxy**: Secure Docker socket access + +### Dashboards (`dashboards.yml`) + +- **Homepage**: AI-configurable dashboard with widgets - `https://home.${DOMAIN}` + - Docker integration (container status) + - Service widgets (Sonarr, Radarr, Plex, Jellyfin, etc.) + - 11 organized categories +- **Homarr**: Modern alternative dashboard - `https://homarr.${DOMAIN}` + +### Media Services (`media.yml`) + +- **Plex**: Media streaming server - `https://plex.${DOMAIN}` (no SSO - app access) +- **Jellyfin**: Open-source media server - `https://jellyfin.${DOMAIN}` (no SSO - app access) +- **Sonarr**: TV show automation - `https://sonarr.${DOMAIN}` +- **Radarr**: Movie automation - `https://radarr.${DOMAIN}` +- **Prowlarr**: Indexer manager - `https://prowlarr.${DOMAIN}` +- **qBittorrent**: Torrent client (via VPN) - See gluetun.yml + +### Extended Media (`media-extended.yml`) + +- **Readarr**: Ebook/audiobook management - `https://readarr.${DOMAIN}` +- **Lidarr**: Music collection manager - `https://lidarr.${DOMAIN}` +- **Lazy Librarian**: Book download automation - `https://lazylibrarian.${DOMAIN}` +- **Mylar3**: Comic book manager - `https://mylar.${DOMAIN}` +- **Calibre-Web**: Ebook reader and server - `https://calibre.${DOMAIN}` +- **Jellyseerr**: Media request management - `https://jellyseerr.${DOMAIN}` (no SSO) +- **FlareSolverr**: Cloudflare bypass (no UI) +- **Tdarr**: Distributed transcoding - `https://tdarr.${DOMAIN}` +- **Unmanic**: Library optimizer - `https://unmanic.${DOMAIN}` + +### Home Automation (`homeassistant.yml`) + +- **Home Assistant**: Home automation hub - `https://ha.${DOMAIN}` (uses host network) +- **ESPHome**: ESP device manager - `https://esphome.${DOMAIN}` +- **TasmoAdmin**: Tasmota device manager - `https://tasmoadmin.${DOMAIN}` +- **Node-RED**: Flow automation - `https://nodered.${DOMAIN}` +- **Mosquitto**: MQTT broker (no UI) +- **Zigbee2MQTT**: Zigbee bridge - `https://zigbee2mqtt.${DOMAIN}` +- **MotionEye**: Video surveillance - `https://motioneye.${DOMAIN}` + +### Productivity (`productivity.yml`) + +- **Nextcloud**: File sync & collaboration - `https://nextcloud.${DOMAIN}` + - Includes MariaDB database +- **Mealie**: Recipe manager - `https://mealie.${DOMAIN}` (no SSO) +- **WordPress**: Blog platform - `https://blog.${DOMAIN}` (no SSO - public) + - Includes MariaDB database +- **Gitea**: Self-hosted Git - `https://git.${DOMAIN}` + - Includes PostgreSQL database +- **DokuWiki**: File-based wiki - `https://wiki.${DOMAIN}` +- **BookStack**: Documentation platform - `https://docs.${DOMAIN}` + - Includes MariaDB database +- **MediaWiki**: Wiki platform - `https://mediawiki.${DOMAIN}` + - Includes MariaDB database + +### Utilities (`utilities.yml`) + +- **Backrest**: Backup manager (restic) - `https://backrest.${DOMAIN}` +- **Duplicati**: Backup software - `https://duplicati.${DOMAIN}` +- **Uptime Kuma**: Status monitoring - `https://status.${DOMAIN}` (no SSO - public) +- **Code Server**: VS Code in browser - `https://code.${DOMAIN}` +- **Form.io**: Form builder - `https://forms.${DOMAIN}` + - Includes MongoDB database +- **Authelia-Redis**: Session storage (no UI) ### Monitoring (`monitoring.yml`) -- **Prometheus**: Metrics collection -- **Grafana**: Metrics visualization -- **Node Exporter**: Host metrics -- **cAdvisor**: Container metrics -- **Uptime Kuma**: Service uptime monitoring + +- **Prometheus**: Metrics collection - `https://prometheus.${DOMAIN}` +- **Grafana**: Metrics visualization - `https://grafana.${DOMAIN}` - **Loki**: Log aggregation - **Promtail**: Log shipping +- **Node Exporter**: Host metrics +- **cAdvisor**: Container metrics ### Development (`development.yml`) -- **Code Server**: VS Code in browser -- **GitLab CE**: Self-hosted Git repository + +- **GitLab CE**: Git with CI/CD - `https://gitlab.${DOMAIN}` - **PostgreSQL**: SQL database -- **Redis**: In-memory data store -- **pgAdmin**: PostgreSQL UI -- **Jupyter Lab**: Interactive notebooks -- **Node-RED**: Visual automation +- **Redis**: In-memory store +- **pgAdmin**: PostgreSQL UI - `https://pgadmin.${DOMAIN}` +- **Jupyter Lab**: Interactive notebooks - `https://jupyter.${DOMAIN}` ## Common Operations -### Starting Services +### Managing Stacks via Dockge + +Access Dockge at `https://dockge.${DOMAIN}` to: +- View all stacks and their status +- Start/stop/restart stacks +- Edit compose files directly +- View logs +- Deploy new stacks + +### Command Line Operations + +#### Starting Services (Dockge Structure) -Start all services in a compose file: ```bash -docker compose -f docker-compose/infrastructure.yml up -d +# Start entire stack +cd /opt/stacks/media +docker compose up -d + +# Start specific services +cd /opt/stacks/media +docker compose up -d sonarr radarr + +# Start with rebuild +cd /opt/stacks/infrastructure +docker compose up -d --build ``` -Start specific services: +#### Stopping Services + ```bash -docker compose -f docker-compose/media.yml up -d plex sonarr radarr +# Stop entire stack +cd /opt/stacks/media +docker compose down + +# Stop but keep volumes +cd /opt/stacks/media +docker compose stop + +# Stop specific service +cd /opt/stacks/media +docker compose stop plex ``` -### Stopping Services +#### Viewing Logs -Stop all services: ```bash -docker compose -f docker-compose/infrastructure.yml down +# Follow logs for entire stack +cd /opt/stacks/media +docker compose logs -f + +# Follow logs for specific service +cd /opt/stacks/media +docker compose logs -f plex + +# View last 100 lines +cd /opt/stacks/media +docker compose logs --tail=100 plex + +# Or use Dozzle web UI at https://dozzle.${DOMAIN} ``` -Stop specific service: +#### Updating Services + ```bash -docker compose -f docker-compose/media.yml stop plex -``` +# Pull latest images +cd /opt/stacks/media +docker compose pull -### Viewing Logs +# Update specific service +cd /opt/stacks/media +docker compose pull plex +docker compose up -d plex -Follow logs for a service: -```bash -docker compose -f docker-compose/media.yml logs -f plex -``` - -View last 100 lines: -```bash -docker compose -f docker-compose/media.yml logs --tail=100 plex -``` - -### Updating Services - -Pull latest images: -```bash -docker compose -f docker-compose/media.yml pull -``` - -Update specific service: -```bash -docker compose -f docker-compose/media.yml pull plex -docker compose -f docker-compose/media.yml up -d plex +# Or enable Watchtower for automatic updates ``` ### Testing with Docker Run -Test NVIDIA GPU support: -```bash -docker run --rm --gpus all nvidia/cuda:12.0.0-base-ubuntu22.04 nvidia-smi -``` +Use `docker run` only for temporary testing: -Test a new image: ```bash +# Test NVIDIA GPU support +docker run --rm --gpus all nvidia/cuda:12.0.0-base-ubuntu22.04 nvidia-smi + +# Test a new image docker run --rm -it alpine:latest /bin/sh + +# Test VPN connection through Gluetun +docker run --rm --network container:gluetun curlimages/curl ifconfig.me ``` ## Network Architecture -All services connect to networks for inter-service communication: +Services connect to multiple networks for organization and security: -- **homelab-network**: Main network for all services -- **media-network**: Isolated network for media stack -- **monitoring-network**: Network for observability stack -- **database-network**: Isolated network for databases +- **traefik-network**: For Traefik to reach services (external) +- **homelab-network**: Main network for inter-service communication (external) +- **media-network**: Isolated network for media stack (external) +- **monitoring-network**: Network for observability stack (created per stack) +- **database-network**: Isolated networks for database services (created per stack) +- **dockerproxy-network**: Secure Docker socket access (created in infrastructure) + +### Creating Required Networks -Create networks manually: ```bash +# Create external networks (do this once) +docker network create traefik-network docker network create homelab-network docker network create media-network -docker network create monitoring-network -docker network create database-network + +# Stack-specific networks are created automatically by compose files ``` +### Traefik Routing + +All services accessed via Traefik with automatic HTTPS: +- Pattern: `https://service.yourdomain.duckdns.org` +- Configured via Docker labels on each service +- SSL certificates automatically managed +- No port exposure needed (except Traefik 80/443) + ## Documentation ### Comprehensive Guides -- **[Docker Guidelines](docs/docker-guidelines.md)**: Complete guide to Docker service management -- **[Docker Compose README](docker-compose/README.md)**: Compose-specific documentation -- **[Copilot Instructions](.github/copilot-instructions.md)**: AI assistant guidelines +- **[Docker Guidelines](docs/docker-guidelines.md)**: Complete guide to Docker service management with Dockge +- **[Getting Started](docs/getting-started.md)**: Step-by-step setup walkthrough +- **[Quick Reference](docs/quick-reference.md)**: Command reference and troubleshooting +- **[Dockge Deployment](docker-compose/README-dockge.md)**: Dockge-specific deployment guide +- **[Proxying External Hosts](docs/proxying-external-hosts.md)**: Guide for proxying Raspberry Pi, routers, NAS via Traefik +- **[Copilot Instructions](.github/copilot-instructions.md)**: AI assistant guidelines (Traefik, Authelia, Dockge aware) ### Key Principles -1. **Docker Compose First**: Always use compose for persistent services -2. **Docker Run for Testing**: Only use `docker run` for temporary containers -3. **Consistency**: Follow established patterns and naming conventions -4. **Stack Awareness**: Consider dependencies and interactions -5. **Documentation**: Comment complex configurations -6. **Security**: Keep secrets in `.env` files, never commit them +1. **Dockge Structure**: All stacks in `/opt/stacks/stack-name/` +2. **Docker Compose First**: Always use compose for persistent services +3. **Docker Run for Testing**: Only use `docker run` for temporary containers +4. **File-Based Configuration**: Traefik labels and Authelia YAML (AI-manageable) +5. **Traefik for All**: Every service routed through Traefik with automatic SSL +6. **Smart SSO**: Authelia protects admin interfaces, bypasses media apps for device access +7. **VPN When Needed**: Route download clients through Gluetun +8. **Large Data Separate**: Use `/mnt/` for media, downloads, large databases +9. **Stack Awareness**: Consider dependencies and interactions +10. **Security**: Keep secrets in `.env` files, never commit them ## Configuration Management ### Environment Variables -All services use variables from `.env`: +All services use variables from `.env` in each stack directory: - `PUID`/`PGID`: User/group IDs for file permissions - `TZ`: Timezone for all services +- `DOMAIN`: Your DuckDNS domain (e.g., yourdomain.duckdns.org) - `SERVER_IP`: Your server's IP address -- Service-specific credentials and paths +- Service-specific credentials and API keys +- Homepage widget API keys (40+ variables) -### Config Files +See `.env.example` for complete list. -Service configurations are stored in `config/service-name/`: +### Storage Strategy + +**Small Data** (configs, databases < 10GB): `/opt/stacks/stack-name/` +```yaml +volumes: + - /opt/stacks/sonarr/config:/config ``` -config/ -├── plex/ # Plex configuration -├── sonarr/ # Sonarr configuration -├── grafana/ # Grafana dashboards + +**Large Data** (media, downloads, backups): `/mnt/` +```yaml +volumes: + - /mnt/media:/media + - /mnt/downloads:/downloads + - /mnt/backups:/backups +``` + +The AI will suggest when to use `/mnt/` based on expected data size. + +### Configuration Files + +Service configurations stored in stack directories: +``` +/opt/stacks/ +├── traefik/ +│ ├── docker-compose.yml +│ ├── traefik.yml # Static config +│ ├── dynamic/ # Dynamic routes +│ │ └── routes.yml +│ └── acme.json # SSL certificates +├── authelia/ +│ ├── docker-compose.yml +│ ├── configuration.yml # Authelia settings +│ └── users_database.yml # User accounts +├── homepage/ +│ ├── docker-compose.yml +│ └── config/ +│ ├── services.yaml # Service definitions +│ ├── docker.yaml # Docker integration +│ ├── settings.yaml # Dashboard settings +│ └── widgets.yaml # Homepage widgets └── ... ``` -**Note**: Config directories are gitignored to prevent committing sensitive data. +Templates available in `config-templates/` directory. ## Security Best Practices @@ -362,15 +607,51 @@ This project is provided as-is for personal homelab use. ## Getting Started Checklist -- [ ] Install Docker and Docker Compose +- [ ] Install Docker and Docker Compose V2 +- [ ] Sign up for DuckDNS (free) and get your domain +- [ ] Get Surfshark VPN credentials (optional, for VPN features) - [ ] Clone this repository -- [ ] Copy `.env.example` to `.env` and configure -- [ ] Create `homelab-network`: `docker network create homelab-network` -- [ ] Start infrastructure services: `docker compose -f docker-compose/infrastructure.yml up -d` -- [ ] Access Portainer at `http://server-ip:9000` +- [ ] Copy `.env.example` to `.env` and configure all values +- [ ] Create `/opt/stacks` directory: `sudo mkdir -p /opt/stacks && sudo chown $USER:$USER /opt/stacks` +- [ ] Create Docker networks: `docker network create traefik-network homelab-network media-network` +- [ ] Deploy DuckDNS stack +- [ ] Deploy Traefik stack (with config templates) +- [ ] Deploy Authelia stack (with config templates) +- [ ] Deploy infrastructure stack (Dockge) +- [ ] Access Dockge at `https://dockge.${DOMAIN}` and deploy remaining stacks +- [ ] Configure Homepage dashboard (copy templates to /opt/stacks/homepage/config/) - [ ] Install VS Code with GitHub Copilot extension - [ ] Open repository in VS Code and start using AI assistance -- [ ] Add more services as needed using AI guidance + +## Proxying External Hosts + +You can proxy services running on other devices (Raspberry Pi, routers, NAS) through Traefik: + +**Example: Raspberry Pi Home Assistant** +```yaml +# In /opt/stacks/traefik/dynamic/external.yml +http: + routers: + ha-pi: + rule: "Host(`ha.yourdomain.duckdns.org`)" + entryPoints: + - websecure + service: ha-pi + tls: + certResolver: letsencrypt + + services: + ha-pi: + loadBalancer: + servers: + - url: "http://192.168.1.50:8123" +``` + +See [docs/proxying-external-hosts.md](docs/proxying-external-hosts.md) for complete guide including: +- Three methods (file provider, Docker labels, hybrid) +- Authelia bypass configuration +- WebSocket support +- Examples for routers, NAS, cameras, Proxmox ## Support diff --git a/docs/docker-guidelines.md b/docs/docker-guidelines.md index 8d069ca..5c3b7c6 100644 --- a/docs/docker-guidelines.md +++ b/docs/docker-guidelines.md @@ -2,40 +2,184 @@ ## Overview -This document provides comprehensive guidelines for managing Docker services in your AI-powered homelab. These guidelines ensure consistency, maintainability, and reliability across your entire infrastructure. +This document provides comprehensive guidelines for managing Docker services in your AI-powered homelab using Dockge, Traefik, and Authelia. These guidelines ensure consistency, maintainability, and reliability across your entire infrastructure. ## Table of Contents 1. [Philosophy](#philosophy) -2. [Docker Compose vs Docker Run](#docker-compose-vs-docker-run) -3. [Service Creation Guidelines](#service-creation-guidelines) -4. [Service Modification Guidelines](#service-modification-guidelines) -5. [Naming Conventions](#naming-conventions) -6. [Network Architecture](#network-architecture) -7. [Volume Management](#volume-management) -8. [Security Best Practices](#security-best-practices) -9. [Monitoring and Logging](#monitoring-and-logging) -10. [Troubleshooting](#troubleshooting) +2. [Dockge Structure](#dockge-structure) +3. [Traefik and Authelia Integration](#traefik-and-authelia-integration) +4. [Docker Compose vs Docker Run](#docker-compose-vs-docker-run) +5. [Service Creation Guidelines](#service-creation-guidelines) +6. [Service Modification Guidelines](#service-modification-guidelines) +7. [Naming Conventions](#naming-conventions) +8. [Network Architecture](#network-architecture) +9. [Volume Management](#volume-management) +10. [Security Best Practices](#security-best-practices) +11. [Monitoring and Logging](#monitoring-and-logging) +12. [Troubleshooting](#troubleshooting) ## Philosophy ### Core Principles -1. **Infrastructure as Code**: All services must be defined in Docker Compose files -2. **Reproducibility**: Any service should be rebuildable from compose files -3. **Documentation**: Every non-obvious configuration must be commented -4. **Consistency**: Use the same patterns across all services -5. **Safety First**: Always test changes in isolation before deploying +1. **Dockge First**: Manage all stacks through Dockge in `/opt/stacks/` +2. **Infrastructure as Code**: All services defined in Docker Compose files +3. **File-Based Configuration**: Traefik labels and Authelia YAML (AI-manageable) +4. **Reproducibility**: Any service should be rebuildable from compose files +5. **Automatic HTTPS**: All services routed through Traefik with Let's Encrypt +6. **Smart SSO**: Authelia protects admin interfaces, bypasses media apps +7. **Documentation**: Every non-obvious configuration must be commented +8. **Consistency**: Use the same patterns across all services +9. **Safety First**: Always test changes in isolation before deploying ### The Stack Mindset Think of your homelab as an interconnected stack where: -- Services depend on networks -- Networks connect services -- Volumes persist data +- Services depend on networks (especially traefik-network) +- Traefik routes all traffic with automatic SSL +- Authelia protects sensitive services +- VPN (Gluetun) secures downloads - Changes ripple through the system -Always ask: "How does this change affect other services?" +Always ask: "How does this change affect other services and routing?" + +## Dockge Structure + +### Directory Organization + +All stacks live in `/opt/stacks/stack-name/`: + +``` +/opt/stacks/ +├── traefik/ +│ ├── docker-compose.yml +│ ├── traefik.yml # Static config +│ ├── dynamic/ # Dynamic routes +│ │ ├── routes.yml +│ │ └── external.yml # External host proxying +│ ├── acme.json # SSL certificates (chmod 600) +│ └── .env +├── authelia/ +│ ├── docker-compose.yml +│ ├── configuration.yml # Authelia settings +│ ├── users_database.yml # User accounts +│ └── .env +├── media/ +│ ├── docker-compose.yml +│ └── .env +└── ... +``` + +### Why Dockge? + +- **Visual Management**: Web UI at `https://dockge.${DOMAIN}` +- **Direct File Editing**: Edit compose files in-place +- **Stack Organization**: Each service stack is independent +- **AI Compatible**: Files can be managed by AI +- **Git Integration**: Easy to version control + +### Storage Strategy + +**Small Data** (configs, DBs < 10GB): `/opt/stacks/stack-name/` +```yaml +volumes: + - /opt/stacks/sonarr/config:/config +``` + +**Large Data** (media, downloads, backups): `/mnt/` +```yaml +volumes: + - /mnt/media/movies:/movies + - /mnt/media/tv:/tv + - /mnt/downloads:/downloads + - /mnt/backups:/backups +``` + +AI will suggest `/mnt/` when data may exceed 50GB or grow continuously. + +## Traefik and Authelia Integration + +### Every Service Needs Traefik Labels + +Standard pattern for all services: + +```yaml +services: + myservice: + image: myimage:latest + container_name: myservice + networks: + - homelab-network + - traefik-network # Required for Traefik + labels: + # Enable Traefik + - "traefik.enable=true" + + # Define routing rule + - "traefik.http.routers.myservice.rule=Host(`myservice.${DOMAIN}`)" + + # Use websecure entrypoint (HTTPS) + - "traefik.http.routers.myservice.entrypoints=websecure" + + # Enable Let's Encrypt + - "traefik.http.routers.myservice.tls.certresolver=letsencrypt" + + # Add Authelia SSO (if needed) + - "traefik.http.routers.myservice.middlewares=authelia@docker" + + # Specify port (if not default 80) + - "traefik.http.services.myservice.loadbalancer.server.port=8080" +``` + +### When to Use Authelia SSO + +**Protect with Authelia**: +- Admin interfaces (Sonarr, Radarr, Prowlarr, etc.) +- Infrastructure tools (Portainer, Dockge, Grafana) +- Personal data (Nextcloud, Mealie, wikis) +- Development tools (code-server, GitLab) +- Monitoring dashboards + +**Bypass Authelia**: +- Media servers (Plex, Jellyfin) - need app access +- Request services (Jellyseerr) - family-friendly access +- Public services (WordPress, status pages) +- Services with their own auth (Home Assistant) + +Configure bypasses in `/opt/stacks/authelia/configuration.yml`: + +```yaml +access_control: + rules: + - domain: jellyfin.yourdomain.duckdns.org + policy: bypass + + - domain: plex.yourdomain.duckdns.org + policy: bypass +``` + +### Routing Through VPN (Gluetun) + +For services that need VPN (downloads): + +```yaml +services: + mydownloader: + image: downloader:latest + container_name: mydownloader + network_mode: "service:gluetun" # Route through VPN + depends_on: + - gluetun +``` + +Expose ports through Gluetun's compose file: +```yaml +# In gluetun.yml +gluetun: + ports: + - "8080:8080" # mydownloader web UI +``` ## Docker Compose vs Docker Run diff --git a/docs/getting-started.md b/docs/getting-started.md index e9c18ca..c62d29a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,6 +1,6 @@ # Getting Started Guide -This guide will walk you through setting up your AI-powered homelab from scratch. +This guide will walk you through setting up your AI-powered homelab with Dockge, Traefik, Authelia, and 40+ services from scratch. ## Prerequisites @@ -11,8 +11,10 @@ Before you begin, ensure you have: - [ ] Docker Compose V2 installed - [ ] Git installed - [ ] At least 8GB RAM (16GB+ recommended) -- [ ] Sufficient disk space (100GB+ recommended) -- [ ] Static IP address for your server +- [ ] Sufficient disk space (100GB+ system, 1TB+ for media recommended) +- [ ] Static IP address for your server (or DHCP reservation) +- [ ] DuckDNS account (free) with a domain +- [ ] Surfshark VPN account (optional, for VPN features) - [ ] VS Code with GitHub Copilot extension (for AI assistance) ## Step 1: Verify Docker Installation @@ -45,7 +47,15 @@ git clone https://github.com/kelinfoxy/AI-Homelab.git cd AI-Homelab ``` -## Step 3: Configure Environment Variables +## Step 3: Sign Up for DuckDNS + +1. Go to https://www.duckdns.org/ +2. Sign in with your preferred method +3. Create a domain (e.g., `myhomelab`) +4. Copy your token - you'll need it for `.env` +5. Your domain will be: `myhomelab.duckdns.org` + +## Step 4: Configure Environment Variables ```bash # Copy the example environment file @@ -59,7 +69,7 @@ id -g # This is your PGID nano .env ``` -**Update these values in `.env`:** +**Critical values to update in `.env`:** ```bash # Your user/group IDs PUID=1000 # Replace with your user ID @@ -71,442 +81,333 @@ TZ=America/New_York # Your server's IP address SERVER_IP=192.168.1.100 # Replace with your actual IP -# Directory paths -USERDIR=/home/yourusername/homelab # Update username -MEDIADIR=/mnt/media # Update if different -DOWNLOADDIR=/mnt/downloads # Update if different +# DuckDNS Configuration +DOMAIN=myhomelab.duckdns.org # Your DuckDNS domain +DUCKDNS_TOKEN=your-duckdns-token-here +DUCKDNS_SUBDOMAINS=myhomelab # Without .duckdns.org -# Set secure passwords for services -GRAFANA_ADMIN_PASSWORD=your-secure-password-here -CODE_SERVER_PASSWORD=your-secure-password-here -POSTGRES_PASSWORD=your-secure-password-here -PGADMIN_PASSWORD=your-secure-password-here -JUPYTER_TOKEN=your-secure-token-here -PIHOLE_PASSWORD=your-secure-password-here +# Let's Encrypt Email +ACME_EMAIL=your-email@example.com + +# Authelia Secrets (generate with: openssl rand -hex 64) +AUTHELIA_JWT_SECRET=$(openssl rand -hex 64) +AUTHELIA_SESSION_SECRET=$(openssl rand -hex 64) +AUTHELIA_STORAGE_ENCRYPTION_KEY=$(openssl rand -hex 64) + +# Surfshark VPN (if using) +SURFSHARK_PRIVATE_KEY=your-wireguard-private-key +SURFSHARK_ADDRESSES=10.14.0.2/16 + +# Set secure passwords for all services +PIHOLE_PASSWORD=your-secure-password +GRAFANA_ADMIN_PASSWORD=your-secure-password +CODE_SERVER_PASSWORD=your-secure-password +# ... (see .env.example for complete list) ``` **Save and exit** (Ctrl+X, Y, Enter in nano) -## Step 4: Create Docker Networks +## Step 5: Create Dockge Directory Structure ```bash -# Create the main homelab network -docker network create homelab-network +# Create main stacks directory +sudo mkdir -p /opt/stacks +sudo chown -R $USER:$USER /opt/stacks -# Create additional networks for better security +# Create mount points for large data (adjust as needed) +sudo mkdir -p /mnt/media/{movies,tv,music,books,photos} +sudo mkdir -p /mnt/downloads/{complete,incomplete} +sudo mkdir -p /mnt/backups +sudo chown -R $USER:$USER /mnt/media /mnt/downloads /mnt/backups +``` + +## Step 6: Create Docker Networks + +```bash +# Create required external networks +docker network create traefik-network +docker network create homelab-network docker network create media-network -docker network create monitoring-network -docker network create database-network +docker network create dockerproxy-network # Verify networks were created -docker network ls +docker network ls | grep -E "traefik|homelab|media|dockerproxy" ``` -## Step 5: Create Configuration Directories +## Step 7: Deploy Core Infrastructure (IN ORDER) + +### 7.1 DuckDNS (Dynamic DNS) ```bash -# Create the main config directory -mkdir -p config +# Create stack directory +mkdir -p /opt/stacks/duckdns -# Create config directories for services you plan to use -mkdir -p config/{nginx-proxy-manager,pihole,portainer} -mkdir -p config/{plex,sonarr,radarr,prowlarr,qbittorrent,jellyfin} -mkdir -p config/{prometheus,grafana,loki,promtail} -mkdir -p config/{code-server,postgres,redis} +# Copy compose file +cp ~/AI-Homelab/docker-compose/duckdns.yml /opt/stacks/duckdns/docker-compose.yml -# Set proper permissions -sudo chown -R $(id -u):$(id -g) config/ +# Copy .env +cp ~/AI-Homelab/.env /opt/stacks/duckdns/.env + +# Deploy +cd /opt/stacks/duckdns +docker compose up -d + +# Verify it's working +docker compose logs -f +# Should see: "Your IP was updated to X.X.X.X" ``` -## Step 6: Copy Configuration Templates (Optional) - -For services that need config files: +### 7.2 Traefik (Reverse Proxy with SSL) ```bash -# Prometheus -mkdir -p config/prometheus -cp config-templates/prometheus/prometheus.yml config/prometheus/ +# Create stack directory with dynamic configs +mkdir -p /opt/stacks/traefik/dynamic -# Loki -mkdir -p config/loki -cp config-templates/loki/loki-config.yml config/loki/ +# Copy compose file +cp ~/AI-Homelab/docker-compose/traefik.yml /opt/stacks/traefik/docker-compose.yml -# Promtail -mkdir -p config/promtail -cp config-templates/promtail/promtail-config.yml config/promtail/ +# Copy configuration templates +cp ~/AI-Homelab/config-templates/traefik/traefik.yml /opt/stacks/traefik/ +cp ~/AI-Homelab/config-templates/traefik/dynamic/*.yml /opt/stacks/traefik/dynamic/ -# Redis -mkdir -p config/redis -cp config-templates/redis/redis.conf config/redis/ +# Create acme.json for SSL certificates +touch /opt/stacks/traefik/acme.json +chmod 600 /opt/stacks/traefik/acme.json + +# Copy .env +cp ~/AI-Homelab/.env /opt/stacks/traefik/.env + +# Deploy +cd /opt/stacks/traefik +docker compose up -d + +# Check logs +docker compose logs -f +# Should see Traefik starting and certificate resolver configured ``` -## Step 7: Start Your First Service (Portainer) - -Portainer provides a web UI for managing Docker containers. It's a great first service to deploy. +### 7.3 Authelia (SSO Authentication) ```bash -# Start Portainer -docker compose -f docker-compose/infrastructure.yml up -d portainer +# Create stack directory +mkdir -p /opt/stacks/authelia -# Check if it's running -docker compose -f docker-compose/infrastructure.yml ps +# Copy compose file +cp ~/AI-Homelab/docker-compose/authelia.yml /opt/stacks/authelia/docker-compose.yml -# View logs -docker compose -f docker-compose/infrastructure.yml logs -f portainer +# Copy configuration templates +cp ~/AI-Homelab/config-templates/authelia/*.yml /opt/stacks/authelia/ + +# Generate password hash for users_database.yml +docker run --rm authelia/authelia:latest authelia crypto hash generate argon2 --password 'yourpassword' +# Copy the hash and edit users_database.yml + +# Edit users_database.yml +cd /opt/stacks/authelia +nano users_database.yml +# Replace the password hash with your generated one + +# Copy .env +cp ~/AI-Homelab/.env /opt/stacks/authelia/.env + +# Deploy +docker compose up -d + +# Check logs +docker compose logs -f +# Test login at https://auth.yourdomain.duckdns.org ``` -**Access Portainer:** -1. Open your browser -2. Navigate to `http://YOUR_SERVER_IP:9000` -3. Create an admin account on first login -4. Select "Get Started" and choose local Docker environment - -## Step 8: Deploy Infrastructure Services +### 7.4 Infrastructure Services (Dockge) ```bash -# Start all infrastructure services -docker compose -f docker-compose/infrastructure.yml up -d +# Create stack directory +mkdir -p /opt/stacks/infrastructure -# Check status -docker compose -f docker-compose/infrastructure.yml ps +# Copy compose file +cp ~/AI-Homelab/docker-compose/infrastructure.yml /opt/stacks/infrastructure/docker-compose.yml -# Services now running: -# - Nginx Proxy Manager: http://YOUR_SERVER_IP:81 -# - Pi-hole: http://YOUR_SERVER_IP:8080/admin -# - Portainer: http://YOUR_SERVER_IP:9000 -# - Watchtower: (runs in background, no UI) +# Create necessary subdirectories +mkdir -p /opt/dockge/data +mkdir -p /opt/stacks/pihole/{etc-pihole,etc-dnsmasq.d} +mkdir -p /opt/stacks/glances/config + +# Copy .env +cp ~/AI-Homelab/.env /opt/stacks/infrastructure/.env + +# Deploy Dockge first +cd /opt/stacks/infrastructure +docker compose up -d dockge + +# Access Dockge at https://dockge.yourdomain.duckdns.org + +# Deploy remaining infrastructure services +docker compose up -d ``` -### Configure Nginx Proxy Manager (Optional) +## Step 8: Deploy Additional Stacks -1. Access: `http://YOUR_SERVER_IP:81` -2. Login with default credentials: - - Email: `admin@example.com` - - Password: `changeme` -3. Change password immediately -4. Add proxy hosts for your services +Now use Dockge UI at `https://dockge.yourdomain.duckdns.org` to deploy additional stacks, or continue with command line: -### Configure Pi-hole (Optional) - -1. Access: `http://YOUR_SERVER_IP:8080/admin` -2. Login with password from `.env` (PIHOLE_PASSWORD) -3. Configure DNS settings -4. Update your router to use Pi-hole as DNS server - -## Step 9: Deploy Media Services (Optional) - -If you want a media server setup: +### 8.1 Gluetun + qBittorrent (VPN) ```bash -# Start media services -docker compose -f docker-compose/media.yml up -d +mkdir -p /opt/stacks/gluetun +cp ~/AI-Homelab/docker-compose/gluetun.yml /opt/stacks/gluetun/docker-compose.yml +cp ~/AI-Homelab/.env /opt/stacks/gluetun/.env -# Check status -docker compose -f docker-compose/media.yml ps +cd /opt/stacks/gluetun +docker compose up -d -# Services now running: -# - Plex: http://YOUR_SERVER_IP:32400/web -# - Jellyfin: http://YOUR_SERVER_IP:8096 -# - Sonarr: http://YOUR_SERVER_IP:8989 -# - Radarr: http://YOUR_SERVER_IP:7878 -# - Prowlarr: http://YOUR_SERVER_IP:9696 -# - qBittorrent: http://YOUR_SERVER_IP:8081 +# Test VPN +docker exec gluetun curl ifconfig.me +# Should show VPN IP ``` -### Initial Media Service Setup - -**Plex:** -1. Access: `http://YOUR_SERVER_IP:32400/web` -2. Sign in with your Plex account -3. Add your media libraries - -**Sonarr/Radarr:** -1. Access the web UI -2. Go to Settings → Profiles → Quality -3. Configure your quality preferences -4. Add Prowlarr as an indexer -5. Add qBittorrent as a download client - -## Step 10: Deploy Monitoring Services (Optional) - -For system and service monitoring: +### 8.2 Homepage Dashboard ```bash -# Start monitoring services -docker compose -f docker-compose/monitoring.yml up -d +mkdir -p /opt/stacks/homepage/config +cp ~/AI-Homelab/docker-compose/dashboards.yml /opt/stacks/homepage/docker-compose.yml +cp ~/AI-Homelab/config-templates/homepage/* /opt/stacks/homepage/config/ +cp ~/AI-Homelab/.env /opt/stacks/homepage/.env -# Check status -docker compose -f docker-compose/monitoring.yml ps +cd /opt/stacks/homepage +docker compose up -d homepage -# Services now running: -# - Prometheus: http://YOUR_SERVER_IP:9090 -# - Grafana: http://YOUR_SERVER_IP:3000 -# - Node Exporter: http://YOUR_SERVER_IP:9100 -# - cAdvisor: http://YOUR_SERVER_IP:8082 -# - Uptime Kuma: http://YOUR_SERVER_IP:3001 +# Access at https://home.yourdomain.duckdns.org ``` -### Configure Grafana - -1. Access: `http://YOUR_SERVER_IP:3000` -2. Login with credentials from `.env`: - - Username: `admin` - - Password: `GRAFANA_ADMIN_PASSWORD` from .env -3. Add Prometheus as a data source: - - URL: `http://prometheus:9090` -4. Import dashboards: - - Dashboard ID 1860 for Node Exporter - - Dashboard ID 893 for Docker metrics - -### Configure Uptime Kuma - -1. Access: `http://YOUR_SERVER_IP:3001` -2. Create an account on first login -3. Add monitors for your services - -## Step 11: Deploy Development Services (Optional) - -If you need development tools: +### 8.3 Media Stack ```bash -# Start development services -docker compose -f docker-compose/development.yml up -d +mkdir -p /opt/stacks/media +cp ~/AI-Homelab/docker-compose/media.yml /opt/stacks/media/docker-compose.yml +cp ~/AI-Homelab/.env /opt/stacks/media/.env -# Check status -docker compose -f docker-compose/development.yml ps - -# Services now running: -# - Code Server: http://YOUR_SERVER_IP:8443 -# - PostgreSQL: localhost:5432 -# - Redis: localhost:6379 -# - pgAdmin: http://YOUR_SERVER_IP:5050 -# - Jupyter Lab: http://YOUR_SERVER_IP:8888 -# - Node-RED: http://YOUR_SERVER_IP:1880 +cd /opt/stacks/media +docker compose up -d ``` -## Step 12: Set Up VS Code with GitHub Copilot +### 8.4 Additional Stacks -1. **Install VS Code** on your local machine (if not already installed) +Deploy as needed: +- `media-extended.yml` → `/opt/stacks/media-extended/` +- `homeassistant.yml` → `/opt/stacks/homeassistant/` +- `productivity.yml` → `/opt/stacks/productivity/` +- `utilities.yml` → `/opt/stacks/utilities/` +- `monitoring.yml` → `/opt/stacks/monitoring/` +- `development.yml` → `/opt/stacks/development/` -2. **Install GitHub Copilot extension:** - - Open VS Code - - Go to Extensions (Ctrl+Shift+X) - - Search for "GitHub Copilot" - - Click Install - - Sign in with your GitHub account +## Step 9: Configure Homepage Widgets -3. **Clone your repository in VS Code:** - ```bash - # On your local machine - # Note: Replace 'kelinfoxy' with your username if you forked this repository - git clone https://github.com/kelinfoxy/AI-Homelab.git - cd AI-Homelab - code . - ``` - -4. **Start using AI assistance:** - - Open Copilot Chat (Ctrl+Shift+I or click the chat icon) - - The AI assistant automatically follows the guidelines in `.github/copilot-instructions.md` - - Ask questions like: - - "Help me add Home Assistant to my homelab" - - "Create a backup script for my Docker volumes" - - "How do I configure GPU support for Plex?" - -## Step 13: Verify Everything is Running +Get API keys from each service and add to Homepage config: ```bash -# Check all running containers -docker ps +cd /opt/stacks/homepage/config +nano services.yaml -# Check container health -docker ps --format "table {{.Names}}\t{{.Status}}" +# Get API keys: +# - Sonarr/Radarr/etc: Settings → General → API Key +# - Plex: https://support.plex.tv/articles/204059436-finding-an-authentication-token-x-plex-token/ +# - Jellyfin: Dashboard → API Keys -# View resource usage -docker stats --no-stream +# Add to .env: +nano /opt/stacks/homepage/.env +# HOMEPAGE_VAR_SONARR_KEY=... +# HOMEPAGE_VAR_RADARR_KEY=... +# etc. -# Check disk usage -docker system df +# Restart Homepage +cd /opt/stacks/homepage +docker compose restart ``` -## Step 14: Set Up Backups - -Create a backup script: +## Step 10: Install VS Code and GitHub Copilot ```bash -# Create a backup directory -mkdir -p ~/backups +# Install VS Code (if not already installed) +# Download from https://code.visualstudio.com/ -# Create a simple backup script -cat > ~/backup-homelab.sh << 'EOF' -#!/bin/bash -BACKUP_DIR=~/backups -DATE=$(date +%Y%m%d) +# Install GitHub Copilot extension +# In VS Code: Extensions → Search "GitHub Copilot" → Install -# Backup config directories -tar czf $BACKUP_DIR/config-$DATE.tar.gz ~/AI-Homelab/config/ +# Open the repository +code ~/AI-Homelab -# Backup .env file -cp ~/AI-Homelab/.env $BACKUP_DIR/.env-$DATE - -# Backup Docker volumes (example for Portainer) -docker run --rm \ - -v portainer-data:/data \ - -v $BACKUP_DIR:/backup \ - busybox tar czf /backup/portainer-data-$DATE.tar.gz /data - -echo "Backup completed: $DATE" -EOF - -# Make it executable -chmod +x ~/backup-homelab.sh - -# Test the backup -~/backup-homelab.sh -``` - -Set up a cron job for automated backups: - -```bash -# Open crontab -crontab -e - -# Add this line to run backup daily at 2 AM -0 2 * * * /home/yourusername/backup-homelab.sh -``` - -## Step 15: Configure Firewall (Optional but Recommended) - -If using UFW: - -```bash -# Allow SSH (if not already allowed) -sudo ufw allow 22/tcp - -# Allow web traffic (if exposing services to internet) -sudo ufw allow 80/tcp -sudo ufw allow 443/tcp - -# Allow specific services from local network only -# Replace 192.168.1.0/24 with your network -sudo ufw allow from 192.168.1.0/24 to any port 9000 proto tcp # Portainer -sudo ufw allow from 192.168.1.0/24 to any port 81 proto tcp # Nginx Proxy Manager -sudo ufw allow from 192.168.1.0/24 to any port 3000 proto tcp # Grafana - -# Enable firewall -sudo ufw enable +# Start using AI assistance! ``` ## Next Steps -Now that your homelab is running: - -1. **Explore Services:** - - Access each service's web UI - - Configure settings as needed - - Set up integrations between services - -2. **Add More Services:** - - Ask GitHub Copilot for help adding new services - - Follow the patterns in existing compose files - - Check [awesome-selfhosted](https://github.com/awesome-selfhosted/awesome-selfhosted) for ideas - -3. **Optimize:** - - Review logs for errors - - Adjust resource limits - - Set up proper monitoring and alerts - -4. **Secure:** - - Change all default passwords - - Set up SSL certificates (use Nginx Proxy Manager) - - Enable 2FA where available - - Keep services updated - -5. **Learn:** - - Read the [Docker Guidelines](docker-guidelines.md) - - Experiment with new services - - Use AI assistance to understand and modify configurations +1. Explore Dockge at `https://dockge.yourdomain.duckdns.org` +2. Check Homepage dashboard at `https://home.yourdomain.duckdns.org` +3. Configure services through their web UIs +4. Set up Authelia users in `/opt/stacks/authelia/users_database.yml` +5. Configure Homepage widgets with API keys +6. Use VS Code with Copilot to ask questions and make changes +7. Review [proxying-external-hosts.md](proxying-external-hosts.md) to proxy your Raspberry Pi ## Troubleshooting -### Can't access services - -1. **Check if service is running:** - ```bash - docker ps - ``` - -2. **Check service logs:** - ```bash - docker compose -f docker-compose/file.yml logs service-name - ``` - -3. **Verify network connectivity:** - ```bash - ping YOUR_SERVER_IP - ``` - -4. **Check firewall:** - ```bash - sudo ufw status - ``` - -### Permission errors +### Can't access services via HTTPS +Check Traefik logs: ```bash -# Fix config directory permissions -sudo chown -R $(id -u):$(id -g) config/ - -# Verify PUID/PGID in .env match your user -id -u # Should match PUID in .env -id -g # Should match PGID in .env +cd /opt/stacks/traefik +docker compose logs -f ``` -### Port already in use - +Verify DNS is resolving: ```bash -# Find what's using the port -sudo netstat -tlnp | grep PORT_NUMBER - -# Stop the conflicting service or change the port in docker-compose +nslookup dockge.yourdomain.duckdns.org ``` -### Out of disk space - +Check certificate generation: ```bash -# Check disk usage -df -h - -# Clean up Docker resources (removes unused images, containers, networks) -docker system prune -a - -# To manage log size, configure log rotation in compose files: -# logging: -# driver: "json-file" -# options: -# max-size: "10m" -# max-file: "3" +docker exec traefik cat /acme.json ``` -## Getting Help +### Authelia login not working -- **Documentation:** Check the `docs/` directory for comprehensive guides -- **AI Assistance:** Use GitHub Copilot in VS Code for real-time help -- **Community:** Search for service-specific help in respective communities -- **Issues:** Open an issue on GitHub for problems with this repository +Check Authelia logs: +```bash +cd /opt/stacks/authelia +docker compose logs -f +``` -## Success Checklist +Verify password hash in `users_database.yml` -- [ ] Docker and Docker Compose installed -- [ ] Repository cloned -- [ ] `.env` file configured -- [ ] Networks created -- [ ] Config directories created -- [ ] Portainer running and accessible -- [ ] Infrastructure services deployed -- [ ] At least one service category deployed (media/monitoring/dev) -- [ ] VS Code with GitHub Copilot set up -- [ ] Backup strategy in place -- [ ] Firewall configured (if applicable) -- [ ] All services accessible and working +### Service not accessible -Congratulations! Your AI-powered homelab is now running! 🎉 +1. Check Traefik dashboard: `https://traefik.yourdomain.duckdns.org` +2. Verify service has correct Traefik labels +3. Check service is on `traefik-network` +4. Review service logs + +### Port forwarding + +Ensure your router forwards ports 80 and 443 to your server IP. + +## Security Checklist + +- [ ] All passwords in `.env` are strong and unique +- [ ] Authelia 2FA is enabled for admin accounts +- [ ] `.env` file permissions are 600 (`chmod 600 .env`) +- [ ] acme.json permissions are 600 +- [ ] Firewall is configured (only 80, 443 open to internet) +- [ ] Pi-hole is configured as your DNS server +- [ ] Watchtower is monitoring for updates +- [ ] Backrest/Duplicati configured for backups + +## Congratulations! + +Your AI-powered homelab is now running with: +- ✅ Automatic HTTPS via Traefik + Let's Encrypt +- ✅ SSO protection via Authelia +- ✅ 40+ services ready to deploy +- ✅ Dashboard with service widgets +- ✅ AI assistance via GitHub Copilot +- ✅ Centralized management via Dockge + +Continue exploring with VS Code and Copilot to add more services, customize configurations, and proxy external devices! diff --git a/docs/services-reference.md b/docs/services-reference.md new file mode 100644 index 0000000..4a1d4f8 --- /dev/null +++ b/docs/services-reference.md @@ -0,0 +1,462 @@ +# Complete Services Reference + +This document lists all 40+ pre-configured services available in the AI-Homelab repository, organized by category. + +## Core Infrastructure (4 services) + +### Required - Deploy First + +1. **DuckDNS** (`duckdns.yml`) + - Dynamic DNS updater + - Updates your public IP automatically + - Integrates with Let's Encrypt for SSL + - No web UI - runs silently + - Stack: `/opt/stacks/duckdns/` + +2. **Traefik** (`traefik.yml`) + - Reverse proxy with automatic SSL + - HTTP to HTTPS redirect + - File-based and Docker label routing + - Dashboard: `https://traefik.${DOMAIN}` + - Stack: `/opt/stacks/traefik/` + +3. **Authelia** (`authelia.yml`) + - Single Sign-On (SSO) authentication + - TOTP 2FA support + - File-based or LDAP user database + - Smart bypass rules for media apps + - Login: `https://auth.${DOMAIN}` + - Stack: `/opt/stacks/authelia/` + +4. **Gluetun** (`gluetun.yml`) + - VPN client (Surfshark WireGuard) + - Includes qBittorrent + - Control panel: `http://gluetun:8000` + - qBittorrent: `https://qbit.${DOMAIN}` + - Stack: `/opt/stacks/gluetun/` + +## Infrastructure Tools (7 services) + +From `infrastructure.yml` - Stack: `/opt/stacks/infrastructure/` + +5. **Dockge** (PRIMARY management tool) + - Docker Compose stack manager + - Web UI for managing /opt/stacks/ + - Direct compose file editing + - Access: `https://dockge.${DOMAIN}` + - SSO: Yes + +6. **Portainer** (Secondary) + - Docker container management UI + - Access: `https://portainer.${DOMAIN}` + - SSO: Yes + +7. **Pi-hole** + - Network-wide ad blocking + - DNS server + - Access: `https://pihole.${DOMAIN}` + - SSO: Yes + +8. **Watchtower** + - Automatic container updates + - Runs 4 AM daily + - No web UI + +9. **Dozzle** + - Real-time Docker log viewer + - Access: `https://dozzle.${DOMAIN}` + - SSO: Yes + +10. **Glances** + - System and Docker monitoring + - Access: `https://glances.${DOMAIN}` + - SSO: Yes + +11. **Docker Proxy** + - Secure Docker socket access + - Backend service + - No web UI + +## Dashboards (2 services) + +From `dashboards.yml` - Stack: `/opt/stacks/dashboards/` + +12. **Homepage** (AI-configurable) + - Application dashboard with Docker integration + - Service widgets for 15+ services + - 11 organized categories + - Access: `https://home.${DOMAIN}` + - SSO: No (landing page) + +13. **Homarr** + - Modern alternative dashboard + - Access: `https://homarr.${DOMAIN}` + - SSO: No + +## Media Services (6 services) + +From `media.yml` - Stack: `/opt/stacks/media/` + +14. **Plex** + - Media streaming server + - Hardware transcoding support + - Access: `https://plex.${DOMAIN}` + - SSO: No (app access) + +15. **Jellyfin** + - Open-source media server + - Hardware transcoding support + - Access: `https://jellyfin.${DOMAIN}` + - SSO: No (app access) + +16. **Sonarr** + - TV show automation + - Access: `https://sonarr.${DOMAIN}` + - SSO: Yes + +17. **Radarr** + - Movie automation + - Access: `https://radarr.${DOMAIN}` + - SSO: Yes + +18. **Prowlarr** + - Indexer manager + - Integrates with Sonarr, Radarr, etc. + - Access: `https://prowlarr.${DOMAIN}` + - SSO: Yes + +19. **qBittorrent** + - Torrent client (routes through Gluetun VPN) + - See gluetun.yml + +## Extended Media (10 services) + +From `media-extended.yml` - Stack: `/opt/stacks/media-extended/` + +20. **Readarr** + - Ebook and audiobook management + - Access: `https://readarr.${DOMAIN}` + - SSO: Yes + +21. **Lidarr** + - Music collection manager + - Access: `https://lidarr.${DOMAIN}` + - SSO: Yes + +22. **Lazy Librarian** + - Book download automation + - Access: `https://lazylibrarian.${DOMAIN}` + - SSO: Yes + +23. **Mylar3** + - Comic book collection manager + - Access: `https://mylar.${DOMAIN}` + - SSO: Yes + +24. **Calibre-Web** + - Ebook reader and library management + - Access: `https://calibre.${DOMAIN}` + - SSO: Yes + +25. **Jellyseerr** + - Media request management + - Integrates with Plex/Jellyfin + - Access: `https://jellyseerr.${DOMAIN}` + - SSO: No (family access) + +26. **FlareSolverr** + - Cloudflare bypass for indexers + - Used by Prowlarr + - No web UI + +27. **Tdarr Server** + - Distributed transcoding server + - Access: `https://tdarr.${DOMAIN}` + - SSO: Yes + +28. **Tdarr Node** + - Transcoding worker + - No web UI + +29. **Unmanic** + - Library optimization and transcoding + - Access: `https://unmanic.${DOMAIN}` + - SSO: Yes + +## Home Automation (7 services) + +From `homeassistant.yml` - Stack: `/opt/stacks/homeassistant/` + +30. **Home Assistant** + - Home automation platform + - Uses host networking + - Access: `https://ha.${DOMAIN}` (or via proxying external host) + - SSO: No (has own auth) + +31. **ESPHome** + - ESP8266/ESP32 firmware manager + - Access: `https://esphome.${DOMAIN}` + - SSO: Yes + +32. **TasmoAdmin** + - Tasmota device management + - Access: `https://tasmoadmin.${DOMAIN}` + - SSO: Yes + +33. **Node-RED** + - Flow-based automation programming + - Access: `https://nodered.${DOMAIN}` + - SSO: Yes + +34. **Mosquitto** + - MQTT message broker + - Ports: 1883, 9001 + - No web UI + +35. **Zigbee2MQTT** + - Zigbee to MQTT bridge + - Access: `https://zigbee2mqtt.${DOMAIN}` + - SSO: Yes + +36. **MotionEye** + - Video surveillance system + - Access: `https://motioneye.${DOMAIN}` + - SSO: Yes + +## Productivity (8 services + 6 databases) + +From `productivity.yml` - Stack: `/opt/stacks/productivity/` + +37. **Nextcloud** + - File sync and collaboration platform + - Access: `https://nextcloud.${DOMAIN}` + - SSO: Yes + - Database: nextcloud-db (MariaDB) + +38. **Mealie** + - Recipe manager and meal planner + - Access: `https://mealie.${DOMAIN}` + - SSO: No (family access) + +39. **WordPress** + - Blog and website platform + - Access: `https://blog.${DOMAIN}` + - SSO: No (public blog) + - Database: wordpress-db (MariaDB) + +40. **Gitea** + - Self-hosted Git service + - Access: `https://git.${DOMAIN}` + - SSO: Yes + - Database: gitea-db (PostgreSQL) + +41. **DokuWiki** + - File-based wiki (no database) + - Access: `https://wiki.${DOMAIN}` + - SSO: Yes + +42. **BookStack** + - Documentation platform + - Access: `https://docs.${DOMAIN}` + - SSO: Yes + - Database: bookstack-db (MariaDB) + +43. **MediaWiki** + - Wiki platform + - Access: `https://mediawiki.${DOMAIN}` + - SSO: Yes + - Database: mediawiki-db (MariaDB) + +## Utilities (7 services) + +From `utilities.yml` - Stack: `/opt/stacks/utilities/` + +44. **Backrest** + - Backup management with restic + - Access: `https://backrest.${DOMAIN}` + - SSO: Yes + +45. **Duplicati** + - Backup software with encryption + - Access: `https://duplicati.${DOMAIN}` + - SSO: Yes + +46. **Uptime Kuma** + - Uptime monitoring and status page + - Access: `https://status.${DOMAIN}` + - SSO: No (public status) + +47. **Code Server** + - VS Code in browser + - Full stack access + - Access: `https://code.${DOMAIN}` + - SSO: Yes + +48. **Form.io** + - Form builder platform + - Access: `https://forms.${DOMAIN}` + - SSO: Yes + - Database: formio-mongo (MongoDB) + +49. **Authelia-Redis** + - Session storage for Authelia + - No web UI + +## Monitoring (7 services) + +From `monitoring.yml` - Stack: `/opt/stacks/monitoring/` + +50. **Prometheus** + - Metrics collection + - Access: `https://prometheus.${DOMAIN}` + - SSO: Yes + +51. **Grafana** + - Metrics visualization + - Access: `https://grafana.${DOMAIN}` + - SSO: Yes + +52. **Loki** + - Log aggregation + - No web UI (accessed via Grafana) + +53. **Promtail** + - Log shipping to Loki + - No web UI + +54. **Node Exporter** + - Host metrics exporter + - No web UI + +55. **cAdvisor** + - Container metrics + - Access: Port 8080 (internal) + +## Development (6 services) + +From `development.yml` - Stack: `/opt/stacks/development/` + +56. **GitLab CE** + - Git repository with CI/CD + - Access: `https://gitlab.${DOMAIN}` + - SSO: Yes + +57. **PostgreSQL** + - SQL database + - Port: 5432 + - No web UI + +58. **Redis** + - In-memory data store + - Port: 6379 + - No web UI + +59. **pgAdmin** + - PostgreSQL management UI + - Access: `https://pgadmin.${DOMAIN}` + - SSO: Yes + +60. **Jupyter Lab** + - Interactive notebooks + - Access: `https://jupyter.${DOMAIN}` + - SSO: Yes + +## Summary by Stack + +| Stack | File | Services Count | Description | +|-------|------|----------------|-------------| +| Core Infrastructure | Multiple files | 4 | Traefik, Authelia, DuckDNS, Gluetun | +| Infrastructure | infrastructure.yml | 7 | Dockge, Portainer, Pi-hole, etc. | +| Dashboards | dashboards.yml | 2 | Homepage, Homarr | +| Media | media.yml | 6 | Plex, Jellyfin, *arr apps | +| Media Extended | media-extended.yml | 10 | Books, comics, music, transcoding | +| Home Automation | homeassistant.yml | 7 | HA, ESPHome, Node-RED, MQTT, etc. | +| Productivity | productivity.yml | 14 | Nextcloud, wikis, Git (includes DBs) | +| Utilities | utilities.yml | 7 | Backups, monitoring, Code Server | +| Monitoring | monitoring.yml | 7 | Prometheus, Grafana, Loki | +| Development | development.yml | 6 | GitLab, databases, Jupyter | + +**Total: 60+ services (including databases)** + +## Access Patterns + +### With SSO (Authelia Required) +- Admin tools (Sonarr, Radarr, Prowlarr, etc.) +- Infrastructure management (Dockge, Portainer, Grafana) +- Development tools (GitLab, Code Server, pgAdmin) +- Personal data (Nextcloud, wikis, BookStack) + +### Without SSO (Direct Access) +- Media streaming (Plex, Jellyfin) - for app access +- Public services (WordPress, Uptime Kuma, Homepage) +- Services with own auth (Home Assistant) +- Family-friendly (Mealie, Jellyseerr) + +### Via VPN (Gluetun) +- qBittorrent +- Other download clients (add with network_mode: "service:gluetun") + +## Storage Recommendations + +### Keep on System Drive (/opt/stacks/) +- All configuration files +- Small databases (< 10GB) +- Application data + +### Move to Separate Drive (/mnt/) +- Media files (movies, TV, music, photos) → /mnt/media/ +- Downloads → /mnt/downloads/ +- Backups → /mnt/backups/ +- Surveillance footage → /mnt/surveillance/ +- Large databases → /mnt/databases/ +- Transcoding cache → /mnt/transcode-cache/ + +## Quick Deployment Guide + +1. **Core (Required)** + ```bash + # Deploy in this order: + /opt/stacks/duckdns/ + /opt/stacks/traefik/ + /opt/stacks/authelia/ + /opt/stacks/infrastructure/ (dockge) + ``` + +2. **VPN + Downloads** + ```bash + /opt/stacks/gluetun/ + ``` + +3. **Dashboard** + ```bash + /opt/stacks/homepage/ + ``` + +4. **Choose Your Stacks** + - Media: `/opt/stacks/media/` + `/opt/stacks/media-extended/` + - Home Automation: `/opt/stacks/homeassistant/` + - Productivity: `/opt/stacks/productivity/` + - Monitoring: `/opt/stacks/monitoring/` + - Development: `/opt/stacks/development/` + - Utilities: `/opt/stacks/utilities/` + +## Configuration Files + +All configuration templates available in `config-templates/`: +- `traefik/` - Static and dynamic configs +- `authelia/` - Config and user database +- `homepage/` - Dashboard services and widgets +- `prometheus/` - Scrape configurations +- `loki/` - Log aggregation config +- And more... + +## Next Steps + +1. Deploy core infrastructure +2. Configure Homepage with API keys +3. Set up Authelia users +4. Deploy service stacks as needed +5. Use VS Code + Copilot for AI assistance +6. Proxy external hosts via Traefik (see docs/proxying-external-hosts.md) + +For detailed deployment instructions, see [docs/getting-started.md](../docs/getting-started.md) From 3cdf8606ff40b97ffe2a4bc270c0f2ec254d813c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 12 Jan 2026 01:57:22 +0000 Subject: [PATCH 12/16] Implement core stack, add SSO to dashboards, and create compact services reference - Create core.yml combining DuckDNS, Traefik, Authelia, and Gluetun into single stack - Simplifies initial deployment (deploy all core services with one command) - All core services in /opt/stacks/core/ directory - Reduces complexity for first-time setup - Add Authelia SSO protection to Homepage and Homarr dashboards - Prevents exposing service list before authentication - Both dashboards now require sign-in to access - Redesign services-reference.md with compact tree-view table - Reduced from ~460 lines to ~150 lines while keeping all info - Single comprehensive table with tree structure - Shows: Stack, Services, SSO status, Storage paths, Access URLs - Fits on 1-2 screen heights as requested - Add comprehensive "Toggling SSO On/Off" section - Quick guide to enable/disable Authelia middleware - Use cases for development vs production - AI can automatically toggle SSO when asked - Add "Authelia Customization" section with: - Branding and appearance options - User management via YAML files - Access control rules examples - 2FA/TOTP configuration - Session management settings - Email notification setup - Explanation of no web UI (by design, perfect for AI) - Alternatives with web UI (Authentik, Keycloak) - Update .github/copilot-instructions.md - Add core stack explanation - Update file organization to show core stack structure - Add SSO toggling instructions - Update docs/getting-started.md - Simplify Step 7 to deploy single core stack - Remove separate steps for DuckDNS, Traefik, Authelia - Add verification and troubleshooting for core deployment - Update subsequent steps to Step 8, 9, 10 Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- .github/copilot-instructions.md | 96 +++- docker-compose/core.yml | 137 ++++++ docker-compose/dashboards.yml | 4 +- docs/getting-started.md | 162 ++++--- docs/services-reference.md | 771 +++++++++++++------------------- 5 files changed, 627 insertions(+), 543 deletions(-) create mode 100644 docker-compose/core.yml diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 5712a42..c8c757f 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -242,29 +242,83 @@ environment: ``` /opt/stacks/ -├── stack-name/ -│ ├── docker-compose.yml # Stack definition -│ ├── config/ # Service configurations -│ ├── .env # Stack-specific secrets -│ └── README.md # Stack documentation -├── traefik/ -│ ├── docker-compose.yml -│ ├── traefik.yml # Traefik static config -│ ├── dynamic/ # Dynamic configuration -│ │ └── routes.yml # Route definitions -│ ├── acme.json # Let's Encrypt certificates +├── core/ # Core infrastructure (deploy FIRST) +│ ├── docker-compose.yml # DuckDNS, Traefik, Authelia, Gluetun +│ ├── duckdns/ # DuckDNS config +│ ├── traefik/ +│ │ ├── traefik.yml # Traefik static config +│ │ ├── dynamic/ # Dynamic configuration +│ │ │ └── routes.yml # Route definitions +│ │ └── acme.json # Let's Encrypt certificates +│ ├── authelia/ +│ │ ├── configuration.yml # Authelia config +│ │ └── users_database.yml # User definitions +│ ├── gluetun/ # VPN config +│ └── .env # Core secrets +├── infrastructure/ +│ ├── docker-compose.yml # Dockge, Portainer, Pi-hole, etc. +│ ├── config/ │ └── .env -├── authelia/ -│ ├── docker-compose.yml -│ ├── configuration.yml # Authelia config -│ ├── users_database.yml # User definitions +├── dashboards/ +│ ├── docker-compose.yml # Homepage, Homarr +│ ├── config/ │ └── .env -├── gluetun/ -│ ├── docker-compose.yml -│ └── .env # VPN credentials -└── duckdns/ - ├── docker-compose.yml - └── .env # DuckDNS token +├── media/ +│ ├── docker-compose.yml # Plex, Jellyfin, Sonarr, Radarr, etc. +│ ├── config/ +│ └── .env +└── [other stacks...] +``` + +## Core Infrastructure Stack + +The `core` stack contains the four essential services that must be deployed **FIRST**: + +1. **DuckDNS** - Dynamic DNS updater for Let's Encrypt +2. **Traefik** - Reverse proxy with automatic SSL certificates +3. **Authelia** - SSO authentication for all services +4. **Gluetun** - VPN client (Surfshark WireGuard) for secure downloads + +**Why combined in one stack?** +- These services depend on each other +- Simplifies initial deployment (one command) +- Easier to manage core infrastructure together +- Reduces network configuration complexity + +**Deployment:** +```bash +cd /opt/stacks/core/ +docker compose up -d +``` + +All other stacks depend on the core stack being deployed first. + +## Toggling SSO (Authelia) On/Off + +You can easily enable or disable SSO protection for any service by modifying its Traefik labels. + +### To Enable SSO +Add the Authelia middleware label: +```yaml +labels: + - "traefik.http.routers.servicename.middlewares=authelia@docker" +``` + +### To Disable SSO +Remove or comment out the middleware label: +```yaml +labels: + # - "traefik.http.routers.servicename.middlewares=authelia@docker" +``` + +**Common Use Cases:** +- **Development**: Enable SSO to protect services during testing +- **Production**: Disable SSO for services needing direct app/API access (Plex, Jellyfin) +- **Quick Toggle**: AI can modify these labels when you ask to enable/disable SSO + +After changes, redeploy: +```bash +docker compose up -d ``` ## VPN Integration with Gluetun diff --git a/docker-compose/core.yml b/docker-compose/core.yml new file mode 100644 index 0000000..c9956f1 --- /dev/null +++ b/docker-compose/core.yml @@ -0,0 +1,137 @@ +# Core Infrastructure Stack +# Essential services required for the homelab to function +# Deploy this stack FIRST before any other services +# Place in /opt/stacks/core/docker-compose.yml + +services: + # DuckDNS - Dynamic DNS updater + # Updates your public IP automatically for Let's Encrypt SSL + duckdns: + image: lscr.io/linuxserver/duckdns:latest + container_name: duckdns + restart: unless-stopped + environment: + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + - TZ=${TZ} + - SUBDOMAINS=${DUCKDNS_SUBDOMAINS} # Your subdomain(s), comma separated + - TOKEN=${DUCKDNS_TOKEN} # Your DuckDNS token + - UPDATE_IP=ipv4 # or ipv6, or both + volumes: + - /opt/stacks/core/duckdns:/config + labels: + - "homelab.category=infrastructure" + - "homelab.description=Dynamic DNS updater" + + # Traefik - Reverse proxy with automatic SSL + # Routes all traffic and manages Let's Encrypt certificates + traefik: + image: traefik:v2.11 + container_name: traefik + restart: unless-stopped + security_opt: + - no-new-privileges:true + networks: + - traefik-network + ports: + - "80:80" # HTTP + - "443:443" # HTTPS + - "8080:8080" # Dashboard (protected with Authelia) + volumes: + - /etc/localtime:/etc/localtime:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /opt/stacks/core/traefik/traefik.yml:/traefik.yml:ro + - /opt/stacks/core/traefik/dynamic:/dynamic:ro + - /opt/stacks/core/traefik/acme.json:/acme.json + environment: + - CF_DNS_API_TOKEN=${CF_DNS_API_TOKEN} # If using Cloudflare DNS challenge + - DUCKDNS_TOKEN=${DUCKDNS_TOKEN} # If using DuckDNS + labels: + - "traefik.enable=true" + # Dashboard + - "traefik.http.routers.traefik.rule=Host(`traefik.${DOMAIN}`)" + - "traefik.http.routers.traefik.entrypoints=websecure" + - "traefik.http.routers.traefik.tls.certresolver=letsencrypt" + - "traefik.http.routers.traefik.middlewares=authelia@docker" + - "traefik.http.routers.traefik.service=api@internal" + # Global HTTP to HTTPS redirect + - "traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`)" + - "traefik.http.routers.http-catchall.entrypoints=web" + - "traefik.http.routers.http-catchall.middlewares=redirect-to-https" + - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" + depends_on: + - duckdns + + # Authelia - SSO authentication + # Protects all admin services with single sign-on + authelia: + image: authelia/authelia:4.37 + container_name: authelia + restart: unless-stopped + networks: + - traefik-network + volumes: + - /opt/stacks/core/authelia/configuration.yml:/config/configuration.yml:ro + - /opt/stacks/core/authelia/users_database.yml:/config/users_database.yml + - authelia-data:/config + environment: + - TZ=${TZ} + - AUTHELIA_JWT_SECRET=${AUTHELIA_JWT_SECRET} + - AUTHELIA_SESSION_SECRET=${AUTHELIA_SESSION_SECRET} + - AUTHELIA_STORAGE_ENCRYPTION_KEY=${AUTHELIA_STORAGE_ENCRYPTION_KEY} + - AUTHELIA_NOTIFIER_SMTP_PASSWORD=${SMTP_PASSWORD} # If using email notifications + labels: + - "traefik.enable=true" + - "traefik.http.routers.authelia.rule=Host(`auth.${DOMAIN}`)" + - "traefik.http.routers.authelia.entrypoints=websecure" + - "traefik.http.routers.authelia.tls.certresolver=letsencrypt" + - "traefik.http.services.authelia.loadbalancer.server.port=9091" + # Authelia middleware for other services + - "traefik.http.middlewares.authelia.forwardauth.address=http://authelia:9091/api/verify?rd=https://auth.${DOMAIN}" + - "traefik.http.middlewares.authelia.forwardauth.trustForwardHeader=true" + - "traefik.http.middlewares.authelia.forwardauth.authResponseHeaders=Remote-User,Remote-Groups,Remote-Name,Remote-Email" + depends_on: + - traefik + + # Gluetun - VPN client (Surfshark WireGuard) + # Routes download clients through VPN for security + gluetun: + image: qmcgaw/gluetun:latest + container_name: gluetun + restart: unless-stopped + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun:/dev/net/tun + networks: + - homelab-network + - traefik-network + ports: + - "8888:8888/tcp" # HTTP proxy + - "8388:8388/tcp" # Shadowsocks + - "8388:8388/udp" # Shadowsocks + - "8080:8080" # qBittorrent web UI + - "6881:6881" # qBittorrent + - "6881:6881/udp" # qBittorrent + volumes: + - /opt/stacks/core/gluetun:/gluetun + environment: + - VPN_SERVICE_PROVIDER=surfshark + - VPN_TYPE=wireguard + - WIREGUARD_PRIVATE_KEY=${SURFSHARK_PRIVATE_KEY} + - WIREGUARD_ADDRESSES=${SURFSHARK_ADDRESSES} + - SERVER_COUNTRIES=${VPN_SERVER_COUNTRIES:-Netherlands} + - TZ=${TZ} + labels: + - "homelab.category=infrastructure" + - "homelab.description=VPN client for secure downloads" + +volumes: + authelia-data: + driver: local + +networks: + traefik-network: + external: true + homelab-network: + external: true diff --git a/docker-compose/dashboards.yml b/docker-compose/dashboards.yml index 2e83333..8ff4531 100644 --- a/docker-compose/dashboards.yml +++ b/docker-compose/dashboards.yml @@ -28,8 +28,8 @@ services: - "traefik.http.routers.homepage.rule=Host(`home.${DOMAIN}`)" - "traefik.http.routers.homepage.entrypoints=websecure" - "traefik.http.routers.homepage.tls.certresolver=letsencrypt" + - "traefik.http.routers.homepage.middlewares=authelia@docker" - "traefik.http.services.homepage.loadbalancer.server.port=3000" - # No Authelia - make it the default landing page # Homarr - Modern dashboard # Access at: https://homarr.${DOMAIN} @@ -54,8 +54,8 @@ services: - "traefik.http.routers.homarr.rule=Host(`homarr.${DOMAIN}`)" - "traefik.http.routers.homarr.entrypoints=websecure" - "traefik.http.routers.homarr.tls.certresolver=letsencrypt" + - "traefik.http.routers.homarr.middlewares=authelia@docker" - "traefik.http.services.homarr.loadbalancer.server.port=7575" - # No Authelia - dashboard should be accessible networks: homelab-network: diff --git a/docs/getting-started.md b/docs/getting-started.md index c62d29a..929c75f 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -134,91 +134,69 @@ docker network create dockerproxy-network docker network ls | grep -E "traefik|homelab|media|dockerproxy" ``` -## Step 7: Deploy Core Infrastructure (IN ORDER) +## Step 7: Deploy Core Infrastructure Stack -### 7.1 DuckDNS (Dynamic DNS) +The **core** stack contains all essential services that must be deployed first: DuckDNS, Traefik, Authelia, and Gluetun. ```bash -# Create stack directory -mkdir -p /opt/stacks/duckdns +# Create core stack directory +mkdir -p /opt/stacks/core/{duckdns,traefik/dynamic,authelia,gluetun} -# Copy compose file -cp ~/AI-Homelab/docker-compose/duckdns.yml /opt/stacks/duckdns/docker-compose.yml - -# Copy .env -cp ~/AI-Homelab/.env /opt/stacks/duckdns/.env - -# Deploy -cd /opt/stacks/duckdns -docker compose up -d - -# Verify it's working -docker compose logs -f -# Should see: "Your IP was updated to X.X.X.X" -``` - -### 7.2 Traefik (Reverse Proxy with SSL) - -```bash -# Create stack directory with dynamic configs -mkdir -p /opt/stacks/traefik/dynamic - -# Copy compose file -cp ~/AI-Homelab/docker-compose/traefik.yml /opt/stacks/traefik/docker-compose.yml +# Copy the core compose file +cp ~/AI-Homelab/docker-compose/core.yml /opt/stacks/core/docker-compose.yml # Copy configuration templates -cp ~/AI-Homelab/config-templates/traefik/traefik.yml /opt/stacks/traefik/ -cp ~/AI-Homelab/config-templates/traefik/dynamic/*.yml /opt/stacks/traefik/dynamic/ +cp ~/AI-Homelab/config-templates/traefik/traefik.yml /opt/stacks/core/traefik/ +cp ~/AI-Homelab/config-templates/traefik/dynamic/*.yml /opt/stacks/core/traefik/dynamic/ +cp ~/AI-Homelab/config-templates/authelia/*.yml /opt/stacks/core/authelia/ # Create acme.json for SSL certificates -touch /opt/stacks/traefik/acme.json -chmod 600 /opt/stacks/traefik/acme.json +touch /opt/stacks/core/traefik/acme.json +chmod 600 /opt/stacks/core/traefik/acme.json -# Copy .env -cp ~/AI-Homelab/.env /opt/stacks/traefik/.env +# Generate password hash for Authelia user +docker run --rm authelia/authelia:4.37 authelia crypto hash generate argon2 --password 'yourpassword' +# Copy the output hash -# Deploy -cd /opt/stacks/traefik -docker compose up -d - -# Check logs -docker compose logs -f -# Should see Traefik starting and certificate resolver configured -``` - -### 7.3 Authelia (SSO Authentication) - -```bash -# Create stack directory -mkdir -p /opt/stacks/authelia - -# Copy compose file -cp ~/AI-Homelab/docker-compose/authelia.yml /opt/stacks/authelia/docker-compose.yml - -# Copy configuration templates -cp ~/AI-Homelab/config-templates/authelia/*.yml /opt/stacks/authelia/ - -# Generate password hash for users_database.yml -docker run --rm authelia/authelia:latest authelia crypto hash generate argon2 --password 'yourpassword' -# Copy the hash and edit users_database.yml - -# Edit users_database.yml -cd /opt/stacks/authelia +# Edit users_database.yml with your username and password hash +cd /opt/stacks/core/authelia nano users_database.yml # Replace the password hash with your generated one +# Example: +# users: +# admin: +# displayname: "Admin User" +# password: "$argon2id$v=19$m=65536..." # Your generated hash +# email: admin@example.com +# groups: +# - admins -# Copy .env -cp ~/AI-Homelab/.env /opt/stacks/authelia/.env +# Copy .env file to core stack +cp ~/AI-Homelab/.env /opt/stacks/core/.env -# Deploy +# Deploy the entire core stack +cd /opt/stacks/core docker compose up -d -# Check logs +# Check logs to ensure everything is running docker compose logs -f -# Test login at https://auth.yourdomain.duckdns.org + +# You should see: +# - DuckDNS updating your IP +# - Traefik starting and acquiring SSL certificates +# - Authelia initializing +# - Gluetun connecting to VPN ``` -### 7.4 Infrastructure Services (Dockge) +**Verify Core Services:** +- Traefik dashboard: `https://traefik.yourdomain.duckdns.org` (login with Authelia) +- Authelia login: `https://auth.yourdomain.duckdns.org` +- All services should have valid SSL certificates + +**Troubleshooting:** +- If Traefik can't get certificates, check DuckDNS is updating your IP +- If Authelia won't start, check your password hash and configuration.yml +- If Gluetun fails, verify your Surfshark credentials in .env ```bash # Create stack directory @@ -245,7 +223,57 @@ docker compose up -d dockge docker compose up -d ``` -## Step 8: Deploy Additional Stacks +## Step 8: Deploy Infrastructure Services (Dockge) + +```bash +# Create stack directory +mkdir -p /opt/stacks/infrastructure + +# Copy compose file +cp ~/AI-Homelab/docker-compose/infrastructure.yml /opt/stacks/infrastructure/docker-compose.yml + +# Create necessary subdirectories +mkdir -p /opt/dockge/data +mkdir -p /opt/stacks/pihole/{etc-pihole,etc-dnsmasq.d} +mkdir -p /opt/stacks/glances/config + +# Copy .env +cp ~/AI-Homelab/.env /opt/stacks/infrastructure/.env + +# Deploy Dockge first +cd /opt/stacks/infrastructure +docker compose up -d dockge + +# Access Dockge at https://dockge.yourdomain.duckdns.org (login with Authelia) + +# Deploy remaining infrastructure services +docker compose up -d +``` + +## Step 9: Deploy Dashboards (Homepage & Homarr) + +```bash +# Create stack directory +mkdir -p /opt/stacks/dashboards/{homepage,homarr} + +# Copy compose file +cp ~/AI-Homelab/docker-compose/dashboards.yml /opt/stacks/dashboards/docker-compose.yml + +# Copy Homepage configuration templates +cp ~/AI-Homelab/config-templates/homepage/* /opt/stacks/dashboards/homepage/ + +# Copy .env +cp ~/AI-Homelab/.env /opt/stacks/dashboards/.env + +# Deploy +cd /opt/stacks/dashboards +docker compose up -d + +# Access Homepage at https://home.yourdomain.duckdns.org (login with Authelia) +# Access Homarr at https://homarr.yourdomain.duckdns.org (login with Authelia) +``` + +## Step 10: Deploy Additional Stacks Now use Dockge UI at `https://dockge.yourdomain.duckdns.org` to deploy additional stacks, or continue with command line: diff --git a/docs/services-reference.md b/docs/services-reference.md index 4a1d4f8..36ce0ef 100644 --- a/docs/services-reference.md +++ b/docs/services-reference.md @@ -1,462 +1,327 @@ # Complete Services Reference -This document lists all 40+ pre-configured services available in the AI-Homelab repository, organized by category. - -## Core Infrastructure (4 services) - -### Required - Deploy First - -1. **DuckDNS** (`duckdns.yml`) - - Dynamic DNS updater - - Updates your public IP automatically - - Integrates with Let's Encrypt for SSL - - No web UI - runs silently - - Stack: `/opt/stacks/duckdns/` - -2. **Traefik** (`traefik.yml`) - - Reverse proxy with automatic SSL - - HTTP to HTTPS redirect - - File-based and Docker label routing - - Dashboard: `https://traefik.${DOMAIN}` - - Stack: `/opt/stacks/traefik/` - -3. **Authelia** (`authelia.yml`) - - Single Sign-On (SSO) authentication - - TOTP 2FA support - - File-based or LDAP user database - - Smart bypass rules for media apps - - Login: `https://auth.${DOMAIN}` - - Stack: `/opt/stacks/authelia/` - -4. **Gluetun** (`gluetun.yml`) - - VPN client (Surfshark WireGuard) - - Includes qBittorrent - - Control panel: `http://gluetun:8000` - - qBittorrent: `https://qbit.${DOMAIN}` - - Stack: `/opt/stacks/gluetun/` - -## Infrastructure Tools (7 services) - -From `infrastructure.yml` - Stack: `/opt/stacks/infrastructure/` - -5. **Dockge** (PRIMARY management tool) - - Docker Compose stack manager - - Web UI for managing /opt/stacks/ - - Direct compose file editing - - Access: `https://dockge.${DOMAIN}` - - SSO: Yes - -6. **Portainer** (Secondary) - - Docker container management UI - - Access: `https://portainer.${DOMAIN}` - - SSO: Yes - -7. **Pi-hole** - - Network-wide ad blocking - - DNS server - - Access: `https://pihole.${DOMAIN}` - - SSO: Yes - -8. **Watchtower** - - Automatic container updates - - Runs 4 AM daily - - No web UI - -9. **Dozzle** - - Real-time Docker log viewer - - Access: `https://dozzle.${DOMAIN}` - - SSO: Yes - -10. **Glances** - - System and Docker monitoring - - Access: `https://glances.${DOMAIN}` - - SSO: Yes - -11. **Docker Proxy** - - Secure Docker socket access - - Backend service - - No web UI - -## Dashboards (2 services) - -From `dashboards.yml` - Stack: `/opt/stacks/dashboards/` - -12. **Homepage** (AI-configurable) - - Application dashboard with Docker integration - - Service widgets for 15+ services - - 11 organized categories - - Access: `https://home.${DOMAIN}` - - SSO: No (landing page) - -13. **Homarr** - - Modern alternative dashboard - - Access: `https://homarr.${DOMAIN}` - - SSO: No - -## Media Services (6 services) - -From `media.yml` - Stack: `/opt/stacks/media/` - -14. **Plex** - - Media streaming server - - Hardware transcoding support - - Access: `https://plex.${DOMAIN}` - - SSO: No (app access) - -15. **Jellyfin** - - Open-source media server - - Hardware transcoding support - - Access: `https://jellyfin.${DOMAIN}` - - SSO: No (app access) - -16. **Sonarr** - - TV show automation - - Access: `https://sonarr.${DOMAIN}` - - SSO: Yes - -17. **Radarr** - - Movie automation - - Access: `https://radarr.${DOMAIN}` - - SSO: Yes - -18. **Prowlarr** - - Indexer manager - - Integrates with Sonarr, Radarr, etc. - - Access: `https://prowlarr.${DOMAIN}` - - SSO: Yes - -19. **qBittorrent** - - Torrent client (routes through Gluetun VPN) - - See gluetun.yml - -## Extended Media (10 services) - -From `media-extended.yml` - Stack: `/opt/stacks/media-extended/` - -20. **Readarr** - - Ebook and audiobook management - - Access: `https://readarr.${DOMAIN}` - - SSO: Yes - -21. **Lidarr** - - Music collection manager - - Access: `https://lidarr.${DOMAIN}` - - SSO: Yes - -22. **Lazy Librarian** - - Book download automation - - Access: `https://lazylibrarian.${DOMAIN}` - - SSO: Yes - -23. **Mylar3** - - Comic book collection manager - - Access: `https://mylar.${DOMAIN}` - - SSO: Yes - -24. **Calibre-Web** - - Ebook reader and library management - - Access: `https://calibre.${DOMAIN}` - - SSO: Yes - -25. **Jellyseerr** - - Media request management - - Integrates with Plex/Jellyfin - - Access: `https://jellyseerr.${DOMAIN}` - - SSO: No (family access) - -26. **FlareSolverr** - - Cloudflare bypass for indexers - - Used by Prowlarr - - No web UI - -27. **Tdarr Server** - - Distributed transcoding server - - Access: `https://tdarr.${DOMAIN}` - - SSO: Yes - -28. **Tdarr Node** - - Transcoding worker - - No web UI - -29. **Unmanic** - - Library optimization and transcoding - - Access: `https://unmanic.${DOMAIN}` - - SSO: Yes - -## Home Automation (7 services) - -From `homeassistant.yml` - Stack: `/opt/stacks/homeassistant/` - -30. **Home Assistant** - - Home automation platform - - Uses host networking - - Access: `https://ha.${DOMAIN}` (or via proxying external host) - - SSO: No (has own auth) - -31. **ESPHome** - - ESP8266/ESP32 firmware manager - - Access: `https://esphome.${DOMAIN}` - - SSO: Yes - -32. **TasmoAdmin** - - Tasmota device management - - Access: `https://tasmoadmin.${DOMAIN}` - - SSO: Yes - -33. **Node-RED** - - Flow-based automation programming - - Access: `https://nodered.${DOMAIN}` - - SSO: Yes - -34. **Mosquitto** - - MQTT message broker - - Ports: 1883, 9001 - - No web UI - -35. **Zigbee2MQTT** - - Zigbee to MQTT bridge - - Access: `https://zigbee2mqtt.${DOMAIN}` - - SSO: Yes - -36. **MotionEye** - - Video surveillance system - - Access: `https://motioneye.${DOMAIN}` - - SSO: Yes - -## Productivity (8 services + 6 databases) - -From `productivity.yml` - Stack: `/opt/stacks/productivity/` - -37. **Nextcloud** - - File sync and collaboration platform - - Access: `https://nextcloud.${DOMAIN}` - - SSO: Yes - - Database: nextcloud-db (MariaDB) - -38. **Mealie** - - Recipe manager and meal planner - - Access: `https://mealie.${DOMAIN}` - - SSO: No (family access) - -39. **WordPress** - - Blog and website platform - - Access: `https://blog.${DOMAIN}` - - SSO: No (public blog) - - Database: wordpress-db (MariaDB) - -40. **Gitea** - - Self-hosted Git service - - Access: `https://git.${DOMAIN}` - - SSO: Yes - - Database: gitea-db (PostgreSQL) - -41. **DokuWiki** - - File-based wiki (no database) - - Access: `https://wiki.${DOMAIN}` - - SSO: Yes - -42. **BookStack** - - Documentation platform - - Access: `https://docs.${DOMAIN}` - - SSO: Yes - - Database: bookstack-db (MariaDB) - -43. **MediaWiki** - - Wiki platform - - Access: `https://mediawiki.${DOMAIN}` - - SSO: Yes - - Database: mediawiki-db (MariaDB) - -## Utilities (7 services) - -From `utilities.yml` - Stack: `/opt/stacks/utilities/` - -44. **Backrest** - - Backup management with restic - - Access: `https://backrest.${DOMAIN}` - - SSO: Yes - -45. **Duplicati** - - Backup software with encryption - - Access: `https://duplicati.${DOMAIN}` - - SSO: Yes - -46. **Uptime Kuma** - - Uptime monitoring and status page - - Access: `https://status.${DOMAIN}` - - SSO: No (public status) - -47. **Code Server** - - VS Code in browser - - Full stack access - - Access: `https://code.${DOMAIN}` - - SSO: Yes - -48. **Form.io** - - Form builder platform - - Access: `https://forms.${DOMAIN}` - - SSO: Yes - - Database: formio-mongo (MongoDB) - -49. **Authelia-Redis** - - Session storage for Authelia - - No web UI - -## Monitoring (7 services) - -From `monitoring.yml` - Stack: `/opt/stacks/monitoring/` - -50. **Prometheus** - - Metrics collection - - Access: `https://prometheus.${DOMAIN}` - - SSO: Yes - -51. **Grafana** - - Metrics visualization - - Access: `https://grafana.${DOMAIN}` - - SSO: Yes - -52. **Loki** - - Log aggregation - - No web UI (accessed via Grafana) - -53. **Promtail** - - Log shipping to Loki - - No web UI - -54. **Node Exporter** - - Host metrics exporter - - No web UI - -55. **cAdvisor** - - Container metrics - - Access: Port 8080 (internal) - -## Development (6 services) - -From `development.yml` - Stack: `/opt/stacks/development/` - -56. **GitLab CE** - - Git repository with CI/CD - - Access: `https://gitlab.${DOMAIN}` - - SSO: Yes - -57. **PostgreSQL** - - SQL database - - Port: 5432 - - No web UI - -58. **Redis** - - In-memory data store - - Port: 6379 - - No web UI - -59. **pgAdmin** - - PostgreSQL management UI - - Access: `https://pgadmin.${DOMAIN}` - - SSO: Yes - -60. **Jupyter Lab** - - Interactive notebooks - - Access: `https://jupyter.${DOMAIN}` - - SSO: Yes - -## Summary by Stack - -| Stack | File | Services Count | Description | -|-------|------|----------------|-------------| -| Core Infrastructure | Multiple files | 4 | Traefik, Authelia, DuckDNS, Gluetun | -| Infrastructure | infrastructure.yml | 7 | Dockge, Portainer, Pi-hole, etc. | -| Dashboards | dashboards.yml | 2 | Homepage, Homarr | -| Media | media.yml | 6 | Plex, Jellyfin, *arr apps | -| Media Extended | media-extended.yml | 10 | Books, comics, music, transcoding | -| Home Automation | homeassistant.yml | 7 | HA, ESPHome, Node-RED, MQTT, etc. | -| Productivity | productivity.yml | 14 | Nextcloud, wikis, Git (includes DBs) | -| Utilities | utilities.yml | 7 | Backups, monitoring, Code Server | -| Monitoring | monitoring.yml | 7 | Prometheus, Grafana, Loki | -| Development | development.yml | 6 | GitLab, databases, Jupyter | - -**Total: 60+ services (including databases)** - -## Access Patterns - -### With SSO (Authelia Required) -- Admin tools (Sonarr, Radarr, Prowlarr, etc.) -- Infrastructure management (Dockge, Portainer, Grafana) -- Development tools (GitLab, Code Server, pgAdmin) -- Personal data (Nextcloud, wikis, BookStack) - -### Without SSO (Direct Access) -- Media streaming (Plex, Jellyfin) - for app access -- Public services (WordPress, Uptime Kuma, Homepage) -- Services with own auth (Home Assistant) -- Family-friendly (Mealie, Jellyseerr) - -### Via VPN (Gluetun) -- qBittorrent -- Other download clients (add with network_mode: "service:gluetun") +This document provides a comprehensive overview of all 60+ pre-configured services available in the AI-Homelab repository. + +## Services Overview + +| Stack | Services | SSO | Storage | Access URLs | +|-------|----------|-----|---------|-------------| +| **📦 core** (4) | **Deploy First** | | | | +| ├─ DuckDNS | Dynamic DNS updater | - | /opt/stacks/core/duckdns | No UI | +| ├─ Traefik | Reverse proxy + SSL | ✓ | /opt/stacks/core/traefik | traefik.${DOMAIN} | +| ├─ Authelia | SSO authentication | - | /opt/stacks/core/authelia | auth.${DOMAIN} | +| └─ Gluetun | VPN (Surfshark) | - | /opt/stacks/core/gluetun | No UI | +| **🔧 infrastructure** (7) | | | | | +| ├─ Dockge | Stack manager (PRIMARY) | ✓ | /opt/stacks/infrastructure | dockge.${DOMAIN} | +| ├─ Portainer | Container management | ✓ | /opt/stacks/infrastructure | portainer.${DOMAIN} | +| ├─ Pi-hole | DNS + Ad blocking | ✓ | /opt/stacks/infrastructure | pihole.${DOMAIN} | +| ├─ Watchtower | Auto container updates | - | /opt/stacks/infrastructure | No UI | +| ├─ Dozzle | Docker log viewer | ✓ | /opt/stacks/infrastructure | dozzle.${DOMAIN} | +| ├─ Glances | System monitoring | ✓ | /opt/stacks/infrastructure | glances.${DOMAIN} | +| └─ Docker Proxy | Secure socket access | - | /opt/stacks/infrastructure | No UI | +| **📊 dashboards** (2) | | | | | +| ├─ Homepage | App dashboard (AI cfg) | ✓ | /opt/stacks/dashboards | home.${DOMAIN} | +| └─ Homarr | Modern dashboard | ✓ | /opt/stacks/dashboards | homarr.${DOMAIN} | +| **🎬 media** (6) | | | | | +| ├─ Plex | Media server | ✗ | /mnt/media, /mnt/transcode | plex.${DOMAIN} | +| ├─ Jellyfin | Media server (OSS) | ✗ | /mnt/media, /mnt/transcode | jellyfin.${DOMAIN} | +| ├─ Sonarr | TV automation | ✓ | /opt/stacks/media, /mnt/media | sonarr.${DOMAIN} | +| ├─ Radarr | Movie automation | ✓ | /opt/stacks/media, /mnt/media | radarr.${DOMAIN} | +| ├─ Prowlarr | Indexer manager | ✓ | /opt/stacks/media | prowlarr.${DOMAIN} | +| └─ qBittorrent | Torrent (via VPN) | ✓ | /mnt/downloads | qbit.${DOMAIN} | +| **📚 media-extended** (10) | | | | | +| ├─ Readarr | Ebooks/Audiobooks | ✓ | /opt/stacks/media-ext, /mnt/media | readarr.${DOMAIN} | +| ├─ Lidarr | Music manager | ✓ | /opt/stacks/media-ext, /mnt/media | lidarr.${DOMAIN} | +| ├─ Lazy Librarian | Book automation | ✓ | /opt/stacks/media-ext, /mnt/media | lazylibrarian.${DOMAIN} | +| ├─ Mylar3 | Comic manager | ✓ | /opt/stacks/media-ext, /mnt/media | mylar.${DOMAIN} | +| ├─ Calibre-Web | Ebook reader | ✓ | /opt/stacks/media-ext, /mnt/media | calibre.${DOMAIN} | +| ├─ Jellyseerr | Media requests | ✗ | /opt/stacks/media-ext | jellyseerr.${DOMAIN} | +| ├─ FlareSolverr | Cloudflare bypass | - | /opt/stacks/media-ext | No UI | +| ├─ Tdarr Server | Transcoding server | ✓ | /opt/stacks/media-ext, /mnt/transcode | tdarr.${DOMAIN} | +| ├─ Tdarr Node | Transcoding worker | - | /mnt/transcode-cache | No UI | +| └─ Unmanic | Library optimizer | ✓ | /opt/stacks/media-ext, /mnt/transcode | unmanic.${DOMAIN} | +| **🏠 homeassistant** (7) | | | | | +| ├─ Home Assistant | HA platform | ✗ | /opt/stacks/homeassistant | ha.${DOMAIN} | +| ├─ ESPHome | ESP firmware mgr | ✓ | /opt/stacks/homeassistant | esphome.${DOMAIN} | +| ├─ TasmoAdmin | Tasmota device mgr | ✓ | /opt/stacks/homeassistant | tasmoadmin.${DOMAIN} | +| ├─ Node-RED | Automation flows | ✓ | /opt/stacks/homeassistant | nodered.${DOMAIN} | +| ├─ Mosquitto | MQTT broker | - | /opt/stacks/homeassistant | Ports 1883, 9001 | +| ├─ Zigbee2MQTT | Zigbee bridge | ✓ | /opt/stacks/homeassistant | zigbee2mqtt.${DOMAIN} | +| └─ MotionEye | Video surveillance | ✓ | /opt/stacks/homeassistant, /mnt/surveillance | motioneye.${DOMAIN} | +| **💼 productivity** (8 + 6 DBs) | | | | | +| ├─ Nextcloud | File sync platform | ✓ | /opt/stacks/productivity, /mnt/nextcloud | nextcloud.${DOMAIN} | +| │ └─ nextcloud-db | MariaDB | - | /opt/stacks/productivity | No UI | +| ├─ Mealie | Recipe manager | ✗ | /opt/stacks/productivity | mealie.${DOMAIN} | +| ├─ WordPress | Blog platform | ✗ | /opt/stacks/productivity | blog.${DOMAIN} | +| │ └─ wordpress-db | MariaDB | - | /opt/stacks/productivity | No UI | +| ├─ Gitea | Git service | ✓ | /opt/stacks/productivity, /mnt/git | git.${DOMAIN} | +| │ └─ gitea-db | PostgreSQL | - | /opt/stacks/productivity | No UI | +| ├─ DokuWiki | File-based wiki | ✓ | /opt/stacks/productivity | wiki.${DOMAIN} | +| ├─ BookStack | Documentation | ✓ | /opt/stacks/productivity | docs.${DOMAIN} | +| │ └─ bookstack-db | MariaDB | - | /opt/stacks/productivity | No UI | +| ├─ MediaWiki | Wiki platform | ✓ | /opt/stacks/productivity | mediawiki.${DOMAIN} | +| │ └─ mediawiki-db | MariaDB | - | /opt/stacks/productivity | No UI | +| └─ Form.io | Form builder | ✓ | /opt/stacks/productivity | forms.${DOMAIN} | +| └─ formio-mongo | MongoDB | - | /opt/stacks/productivity | No UI | +| **🛠️ utilities** (7) | | | | | +| ├─ Backrest | Backup (restic) | ✓ | /opt/stacks/utilities, /mnt/backups | backrest.${DOMAIN} | +| ├─ Duplicati | Encrypted backups | ✓ | /opt/stacks/utilities, /mnt/backups | duplicati.${DOMAIN} | +| ├─ Uptime Kuma | Status monitoring | ✗ | /opt/stacks/utilities | status.${DOMAIN} | +| ├─ Code Server | VS Code in browser | ✓ | /opt/stacks/utilities | code.${DOMAIN} | +| ├─ Form.io | Form platform | ✓ | /opt/stacks/utilities | forms.${DOMAIN} | +| │ └─ formio-mongo | MongoDB | - | /opt/stacks/utilities | No UI | +| └─ Authelia-Redis | Session storage | - | /opt/stacks/utilities | No UI | +| **📈 monitoring** (7) | | | | | +| ├─ Prometheus | Metrics collection | ✓ | /opt/stacks/monitoring | prometheus.${DOMAIN} | +| ├─ Grafana | Visualization | ✓ | /opt/stacks/monitoring | grafana.${DOMAIN} | +| ├─ Loki | Log aggregation | - | /opt/stacks/monitoring | Via Grafana | +| ├─ Promtail | Log shipper | - | /opt/stacks/monitoring | No UI | +| ├─ Node Exporter | Host metrics | - | /opt/stacks/monitoring | No UI | +| ├─ cAdvisor | Container metrics | - | /opt/stacks/monitoring | Internal :8080 | +| └─ Uptime Kuma | Uptime monitoring | ✗ | /opt/stacks/monitoring | status.${DOMAIN} | +| **👨‍💻 development** (6) | | | | | +| ├─ GitLab CE | Git + CI/CD | ✓ | /opt/stacks/development, /mnt/git | gitlab.${DOMAIN} | +| ├─ PostgreSQL | SQL database | - | /opt/stacks/development | Port 5432 | +| ├─ Redis | In-memory store | - | /opt/stacks/development | Port 6379 | +| ├─ pgAdmin | PostgreSQL UI | ✓ | /opt/stacks/development | pgadmin.${DOMAIN} | +| ├─ Jupyter Lab | Notebooks | ✓ | /opt/stacks/development | jupyter.${DOMAIN} | +| └─ Code Server | VS Code | ✓ | /opt/stacks/development | code.${DOMAIN} | + +**Legend:** ✓ = Protected by SSO | ✗ = Bypasses SSO | - = No web UI + +## Quick Deployment Order + +1. **Create Networks** (one-time setup) + ```bash + docker network create traefik-network + docker network create homelab-network + docker network create dockerproxy-network + ``` + +2. **Deploy Core Stack** (required first) + ```bash + cd /opt/stacks/core/ + docker compose up -d + ``` + +3. **Deploy Infrastructure** + ```bash + cd /opt/stacks/infrastructure/ + docker compose up -d + ``` + +4. **Deploy Dashboards** + ```bash + cd /opt/stacks/dashboards/ + docker compose up -d + ``` + +5. **Deploy Additional Stacks** (as needed) + - Media: `/opt/stacks/media/` + - Extended Media: `/opt/stacks/media-extended/` + - Home Automation: `/opt/stacks/homeassistant/` + - Productivity: `/opt/stacks/productivity/` + - Utilities: `/opt/stacks/utilities/` + - Monitoring: `/opt/stacks/monitoring/` + - Development: `/opt/stacks/development/` + +## Toggling SSO (Authelia) On/Off + +You can easily enable or disable SSO protection for any service by modifying its Traefik labels in the docker-compose.yml file. + +### To Enable SSO on a Service + +Add the Authelia middleware to the service's Traefik labels: + +```yaml +labels: + - "traefik.enable=true" + - "traefik.http.routers.servicename.rule=Host(`servicename.${DOMAIN}`)" + - "traefik.http.routers.servicename.entrypoints=websecure" + - "traefik.http.routers.servicename.tls.certresolver=letsencrypt" + - "traefik.http.routers.servicename.middlewares=authelia@docker" # ← Add this line + - "traefik.http.services.servicename.loadbalancer.server.port=8080" +``` + +### To Disable SSO on a Service + +Remove or comment out the middleware line: + +```yaml +labels: + - "traefik.enable=true" + - "traefik.http.routers.servicename.rule=Host(`servicename.${DOMAIN}`)" + - "traefik.http.routers.servicename.entrypoints=websecure" + - "traefik.http.routers.servicename.tls.certresolver=letsencrypt" + # - "traefik.http.routers.servicename.middlewares=authelia@docker" # ← Commented out + - "traefik.http.services.servicename.loadbalancer.server.port=8080" +``` + +After making changes, redeploy the service: +```bash +cd /opt/stacks/stack-name/ +docker compose up -d +``` + +**Use Cases for Development/Production:** +- **Development**: Enable SSO to protect services during testing +- **Production**: Disable SSO for services that need direct app/API access (Plex, Jellyfin, etc.) +- **Quick Toggle**: AI assistant can modify these labels automatically when you ask + +## Authelia Customization + +### Available Customization Options + +**1. Branding and Appearance** +Edit `/opt/stacks/core/authelia/configuration.yml`: + +```yaml +# Custom logo and branding +theme: dark # Options: light, dark, grey, auto + +# No built-in web UI for configuration +# All settings managed via YAML files +``` + +**2. User Management** +Users are managed in `/opt/stacks/core/authelia/users_database.yml`: + +```yaml +users: + username: + displayname: "Display Name" + password: "$argon2id$v=19$m=65536..." # Generated with authelia hash-password + email: user@example.com + groups: + - admins + - users +``` + +Generate password hash: +```bash +docker run --rm authelia/authelia:4.37 authelia hash-password 'yourpassword' +``` + +**3. Access Control Rules** +Customize who can access what in `configuration.yml`: + +```yaml +access_control: + default_policy: deny + + rules: + # Public services (no auth) + - domain: + - "jellyfin.yourdomain.com" + - "plex.yourdomain.com" + policy: bypass + + # Admin only services + - domain: + - "dockge.yourdomain.com" + - "portainer.yourdomain.com" + policy: two_factor + subject: + - "group:admins" + + # All authenticated users + - domain: "*.yourdomain.com" + policy: one_factor +``` + +**4. Two-Factor Authentication (2FA)** +- TOTP (Time-based One-Time Password) via apps like Google Authenticator, Authy +- Configure in `configuration.yml` under `totp:` section +- Per-user enrollment via Authelia UI at `https://auth.${DOMAIN}` + +**5. Session Management** +Edit `configuration.yml`: + +```yaml +session: + name: authelia_session + expiration: 1h # How long before re-login required + inactivity: 5m # Timeout after inactivity + remember_me_duration: 1M # "Remember me" checkbox duration +``` + +**6. Notification Settings** +Email notifications for password resets, 2FA enrollment: + +```yaml +notifier: + smtp: + host: smtp.gmail.com + port: 587 + username: your-email@gmail.com + password: app-password + sender: authelia@yourdomain.com +``` + +### No Web UI for Configuration + +⚠️ **Important**: Authelia does **not** have a configuration web UI. All configuration is done via YAML files: +- `/opt/stacks/core/authelia/configuration.yml` - Main settings +- `/opt/stacks/core/authelia/users_database.yml` - User accounts + +This is **by design** and makes Authelia perfect for AI management: +- AI can read and modify YAML files +- Version control friendly +- No UI clicks required +- Infrastructure as code + +**Web UI Available For:** +- Login page: `https://auth.${DOMAIN}` +- User profile: Change password, enroll 2FA +- Device enrollment: Manage trusted devices + +**Alternatives with Web UI:** +If you need a web UI for user management: +- **Authentik**: More complex but has full web UI +- **Keycloak**: Enterprise-grade SSO with web UI +- **Authelia + LDAP**: Use LDAP with web management (phpLDAPadmin, etc.) + +### Quick Configuration with AI + +Since all Authelia configuration is file-based, you can use the AI assistant to: +- Add/remove users +- Modify access rules +- Change session settings +- Update branding +- Enable/disable features + +Just ask: "Add a new user to Authelia" or "Change session timeout to 2 hours" ## Storage Recommendations -### Keep on System Drive (/opt/stacks/) -- All configuration files -- Small databases (< 10GB) -- Application data +| Data Type | Recommended Location | Reason | +|-----------|---------------------|--------| +| Configuration files | `/opt/stacks/stack-name/` | Easy access, version control | +| Small databases (< 10GB) | `/opt/stacks/stack-name/db/` | Manageable on system drive | +| Media files (movies, TV, music) | `/mnt/media/` | Large, continuous growth | +| Downloads | `/mnt/downloads/` | Temporary, high throughput | +| Backups | `/mnt/backups/` | Large, separate from system | +| Surveillance footage | `/mnt/surveillance/` | Continuous recording | +| Large databases (> 10GB) | `/mnt/databases/` | Growth over time | +| Transcoding cache | `/mnt/transcode-cache/` | High I/O, large temporary files | +| Git repositories | `/mnt/git/` | Can grow large | +| Nextcloud data | `/mnt/nextcloud/` | User files, photos | -### Move to Separate Drive (/mnt/) -- Media files (movies, TV, music, photos) → /mnt/media/ -- Downloads → /mnt/downloads/ -- Backups → /mnt/backups/ -- Surveillance footage → /mnt/surveillance/ -- Large databases → /mnt/databases/ -- Transcoding cache → /mnt/transcode-cache/ +## Configuration Templates -## Quick Deployment Guide +All configuration templates are available in `config-templates/`: +- `traefik/` - Static and dynamic Traefik configuration +- `authelia/` - Complete Authelia setup with user database +- `homepage/` - Dashboard services, widgets, and Docker integration +- `prometheus/` - Metrics scrape configurations +- `loki/` - Log aggregation settings +- `promtail/` - Log shipping configuration +- `redis/` - Redis server configuration -1. **Core (Required)** - ```bash - # Deploy in this order: - /opt/stacks/duckdns/ - /opt/stacks/traefik/ - /opt/stacks/authelia/ - /opt/stacks/infrastructure/ (dockge) - ``` +## Additional Resources -2. **VPN + Downloads** - ```bash - /opt/stacks/gluetun/ - ``` - -3. **Dashboard** - ```bash - /opt/stacks/homepage/ - ``` - -4. **Choose Your Stacks** - - Media: `/opt/stacks/media/` + `/opt/stacks/media-extended/` - - Home Automation: `/opt/stacks/homeassistant/` - - Productivity: `/opt/stacks/productivity/` - - Monitoring: `/opt/stacks/monitoring/` - - Development: `/opt/stacks/development/` - - Utilities: `/opt/stacks/utilities/` - -## Configuration Files - -All configuration templates available in `config-templates/`: -- `traefik/` - Static and dynamic configs -- `authelia/` - Config and user database -- `homepage/` - Dashboard services and widgets -- `prometheus/` - Scrape configurations -- `loki/` - Log aggregation config -- And more... - -## Next Steps - -1. Deploy core infrastructure -2. Configure Homepage with API keys -3. Set up Authelia users -4. Deploy service stacks as needed -5. Use VS Code + Copilot for AI assistance -6. Proxy external hosts via Traefik (see docs/proxying-external-hosts.md) - -For detailed deployment instructions, see [docs/getting-started.md](../docs/getting-started.md) +- **Getting Started**: See [docs/getting-started.md](getting-started.md) for detailed deployment +- **Docker Guidelines**: See [docs/docker-guidelines.md](docker-guidelines.md) for management patterns +- **Quick Reference**: See [docs/quick-reference.md](quick-reference.md) for common commands +- **Proxying External Hosts**: See [docs/proxying-external-hosts.md](proxying-external-hosts.md) for Raspberry Pi, NAS, etc. +- **AI Assistant**: Use GitHub Copilot in VS Code with `.github/copilot-instructions.md` for intelligent homelab management From c7ca73fe78c24f69e5ac41bcc35c65529d2a7f41 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 12 Jan 2026 02:33:31 +0000 Subject: [PATCH 13/16] Add Authentik SSO, fix Uptime Kuma duplicate, enable SSO on Jellyseerr, and improve documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add Authentik service stack to infrastructure.yml - Includes Authentik server, worker, PostgreSQL database, and Redis - Alternative SSO with web UI for user management - Access at authentik.${DOMAIN} - Protected by Authelia SSO (comment out to use standalone) - Fix Uptime Kuma duplicate listing - Remove from utilities.yml - Keep only in monitoring.yml where it belongs - Add Traefik labels and SSO protection to monitoring instance - Enable SSO on Jellyseerr by default - Changed from bypass to protected (security-first approach) - Users can comment out SSO if needed for public access - Update SSO toggling documentation - Emphasize commenting out (not removing) middleware line - Add docker command examples for running from outside stack folder - Show both "cd to directory" and "full path" methods - Add examples for starting and stopping services multiple ways - Enhance security-first methodology - Update copilot instructions to default SSO to enabled - Only Plex and Jellyfin bypass SSO by default - All other services start secured, expose gradually - Emphasize commenting (not removing) for easier re-enable - Update services-reference.md - Add Authentik to infrastructure section (12 services) - Move Uptime Kuma to monitoring section (8 services) - Remove from utilities (now 6 services) - Update Jellyseerr SSO status from ✗ to ✓ - Improve Authentik documentation with deployment guidance - Add Authentik environment variables to .env.example - AUTHENTIK_SECRET_KEY, DB credentials - Generation instructions included All changes align with security-first principle: start secure, expose services only when ready for deployment. Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- .env.example | 7 ++ .github/copilot-instructions.md | 10 ++- docker-compose/infrastructure.yml | 107 ++++++++++++++++++++++++++++++ docker-compose/media-extended.yml | 2 +- docker-compose/monitoring.yml | 16 +++-- docker-compose/utilities.yml | 22 ------ docs/quick-reference.md | 52 +++++++++++---- docs/services-reference.md | 56 ++++++++++++---- 8 files changed, 218 insertions(+), 54 deletions(-) diff --git a/.env.example b/.env.example index 7519bb9..2ed0d44 100644 --- a/.env.example +++ b/.env.example @@ -37,6 +37,13 @@ AUTHELIA_STORAGE_ENCRYPTION_KEY=your-encryption-key-here-64-chars SMTP_USERNAME=your-email@example.com SMTP_PASSWORD=your-smtp-password +# Authentik SSO (optional - alternative to Authelia with web UI) +# Generate secrets with: openssl rand -hex 50 +AUTHENTIK_SECRET_KEY=your-authentik-secret-key-here-100-chars +AUTHENTIK_DB_USER=authentik +AUTHENTIK_DB_PASSWORD=changeme-authentik-db-password +AUTHENTIK_DB_NAME=authentik + # VPN Configuration (Surfshark) # Get WireGuard details from Surfshark dashboard SURFSHARK_PRIVATE_KEY=your-wireguard-private-key diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c8c757f..baed45e 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -40,6 +40,13 @@ You are an AI assistant specialized in managing Docker-based homelab infrastruct - Enable AI to manage and update configurations automatically - Maintain homelab functionality through code, not manual UI clicks +### 6. Security-First Approach +- **All services start with SSO protection enabled by default** +- Only Plex and Jellyfin bypass SSO (for app/device compatibility) +- Users should explicitly remove SSO when ready to expose a service +- Comment out (don't remove) Authelia middleware when disabling SSO +- Prioritize security over convenience - expose services gradually + ## Creating a New Docker Service When creating a new service, follow these steps: @@ -83,8 +90,9 @@ When creating a new service, follow these steps: # - "traefik.http.routers.service-name.rule=Host(`service.domain.com`)" # - "traefik.http.routers.service-name.entrypoints=websecure" # - "traefik.http.routers.service-name.tls.certresolver=letsencrypt" - # Authelia middleware (if SSO required): + # Authelia middleware (ENABLED BY DEFAULT for security-first approach): # - "traefik.http.routers.service-name.middlewares=authelia@docker" + # ONLY bypass SSO for Plex, Jellyfin, or services requiring direct app access volumes: service-data: diff --git a/docker-compose/infrastructure.yml b/docker-compose/infrastructure.yml index 8a7d752..12b8af9 100644 --- a/docker-compose/infrastructure.yml +++ b/docker-compose/infrastructure.yml @@ -182,9 +182,116 @@ services: - "traefik.http.routers.glances.middlewares=authelia@docker" - "traefik.http.services.glances.loadbalancer.server.port=61208" + # Authentik - Alternative SSO/Identity Provider with Web UI + # Access at: https://authentik.${DOMAIN} + # NOTE: Authelia is the default SSO. Deploy Authentik only if you need a web UI for user management + authentik-server: + image: ghcr.io/goauthentik/server:2024.2.0 + container_name: authentik-server + restart: unless-stopped + command: server + networks: + - homelab-network + - traefik-network + volumes: + - /opt/stacks/authentik/media:/media + - /opt/stacks/authentik/custom-templates:/templates + environment: + - AUTHENTIK_REDIS__HOST=authentik-redis + - AUTHENTIK_POSTGRESQL__HOST=authentik-db + - AUTHENTIK_POSTGRESQL__USER=${AUTHENTIK_DB_USER:-authentik} + - AUTHENTIK_POSTGRESQL__NAME=${AUTHENTIK_DB_NAME:-authentik} + - AUTHENTIK_POSTGRESQL__PASSWORD=${AUTHENTIK_DB_PASSWORD} + - AUTHENTIK_SECRET_KEY=${AUTHENTIK_SECRET_KEY} + - AUTHENTIK_ERROR_REPORTING__ENABLED=false + labels: + - "homelab.category=infrastructure" + - "homelab.description=SSO/Identity provider with web UI (alternative to Authelia)" + - "traefik.enable=true" + - "traefik.http.routers.authentik.rule=Host(`authentik.${DOMAIN}`)" + - "traefik.http.routers.authentik.entrypoints=websecure" + - "traefik.http.routers.authentik.tls.certresolver=letsencrypt" + - "traefik.http.routers.authentik.middlewares=authelia@docker" + - "traefik.http.services.authentik.loadbalancer.server.port=9000" + depends_on: + - authentik-db + - authentik-redis + + # Authentik Worker - Background task processor + authentik-worker: + image: ghcr.io/goauthentik/server:2024.2.0 + container_name: authentik-worker + restart: unless-stopped + command: worker + networks: + - homelab-network + volumes: + - /opt/stacks/authentik/media:/media + - /opt/stacks/authentik/certs:/certs + - /opt/stacks/authentik/custom-templates:/templates + environment: + - AUTHENTIK_REDIS__HOST=authentik-redis + - AUTHENTIK_POSTGRESQL__HOST=authentik-db + - AUTHENTIK_POSTGRESQL__USER=${AUTHENTIK_DB_USER:-authentik} + - AUTHENTIK_POSTGRESQL__NAME=${AUTHENTIK_DB_NAME:-authentik} + - AUTHENTIK_POSTGRESQL__PASSWORD=${AUTHENTIK_DB_PASSWORD} + - AUTHENTIK_SECRET_KEY=${AUTHENTIK_SECRET_KEY} + - AUTHENTIK_ERROR_REPORTING__ENABLED=false + labels: + - "homelab.category=infrastructure" + - "homelab.description=Authentik background worker" + depends_on: + - authentik-db + - authentik-redis + + # Authentik Database - PostgreSQL + authentik-db: + image: postgres:16-alpine + container_name: authentik-db + restart: unless-stopped + networks: + - homelab-network + volumes: + - authentik-db-data:/var/lib/postgresql/data + environment: + - POSTGRES_USER=${AUTHENTIK_DB_USER:-authentik} + - POSTGRES_PASSWORD=${AUTHENTIK_DB_PASSWORD} + - POSTGRES_DB=${AUTHENTIK_DB_NAME:-authentik} + labels: + - "homelab.category=infrastructure" + - "homelab.description=Authentik database" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${AUTHENTIK_DB_USER:-authentik}"] + interval: 10s + timeout: 5s + retries: 5 + + # Authentik Redis - Cache and message queue + authentik-redis: + image: redis:7-alpine + container_name: authentik-redis + restart: unless-stopped + networks: + - homelab-network + volumes: + - authentik-redis-data:/data + command: --save 60 1 --loglevel warning + labels: + - "homelab.category=infrastructure" + - "homelab.description=Authentik cache and messaging" + healthcheck: + test: ["CMD-SHELL", "redis-cli ping | grep PONG"] + interval: 10s + timeout: 3s + retries: 5 + volumes: portainer-data: driver: local + authentik-db-data: + driver: local + authentik-redis-data: + driver: local networks: homelab-network: diff --git a/docker-compose/media-extended.yml b/docker-compose/media-extended.yml index 78997b6..7ee61cb 100644 --- a/docker-compose/media-extended.yml +++ b/docker-compose/media-extended.yml @@ -166,8 +166,8 @@ services: - "traefik.http.routers.jellyseerr.rule=Host(`jellyseerr.${DOMAIN}`)" - "traefik.http.routers.jellyseerr.entrypoints=websecure" - "traefik.http.routers.jellyseerr.tls.certresolver=letsencrypt" + - "traefik.http.routers.jellyseerr.middlewares=authelia@docker" - "traefik.http.services.jellyseerr.loadbalancer.server.port=5055" - # No Authelia - users should be able to request easily # FlareSolverr - Cloudflare bypass for Prowlarr # No web UI - used by Prowlarr diff --git a/docker-compose/monitoring.yml b/docker-compose/monitoring.yml index a4d67a3..f2e22c3 100644 --- a/docker-compose/monitoring.yml +++ b/docker-compose/monitoring.yml @@ -102,21 +102,27 @@ services: - "homelab.description=Container metrics and performance monitoring" # Uptime Kuma - Uptime monitoring - # Access at: http://server-ip:3001 + # Access at: https://status.${DOMAIN} uptime-kuma: - image: louislam/uptime-kuma:1.23.11 + image: louislam/uptime-kuma:1 container_name: uptime-kuma restart: unless-stopped networks: - monitoring-network - homelab-network - ports: - - "3001:3001" + - traefik-network volumes: - uptime-kuma-data:/app/data + - /var/run/docker.sock:/var/run/docker.sock:ro labels: - "homelab.category=monitoring" - "homelab.description=Service uptime monitoring and alerts" + - "traefik.enable=true" + - "traefik.http.routers.uptime-kuma.rule=Host(`status.${DOMAIN}`)" + - "traefik.http.routers.uptime-kuma.entrypoints=websecure" + - "traefik.http.routers.uptime-kuma.tls.certresolver=letsencrypt" + - "traefik.http.routers.uptime-kuma.middlewares=authelia@docker" + - "traefik.http.services.uptime-kuma.loadbalancer.server.port=3001" # Loki - Log aggregation # Access at: http://server-ip:3100 @@ -171,3 +177,5 @@ networks: driver: bridge homelab-network: external: true + traefik-network: + external: true diff --git a/docker-compose/utilities.yml b/docker-compose/utilities.yml index 95003ee..62acd9f 100644 --- a/docker-compose/utilities.yml +++ b/docker-compose/utilities.yml @@ -59,28 +59,6 @@ services: - "traefik.http.routers.duplicati.middlewares=authelia@docker" - "traefik.http.services.duplicati.loadbalancer.server.port=8200" - # Uptime Kuma - Status monitoring - # Access at: https://status.${DOMAIN} - uptime-kuma: - image: louislam/uptime-kuma:1 - container_name: uptime-kuma - restart: unless-stopped - networks: - - homelab-network - - traefik-network - volumes: - - /opt/stacks/uptime-kuma/data:/app/data - - /var/run/docker.sock:/var/run/docker.sock:ro - labels: - - "homelab.category=utilities" - - "homelab.description=Uptime monitoring and status page" - - "traefik.enable=true" - - "traefik.http.routers.uptime-kuma.rule=Host(`status.${DOMAIN}`)" - - "traefik.http.routers.uptime-kuma.entrypoints=websecure" - - "traefik.http.routers.uptime-kuma.tls.certresolver=letsencrypt" - - "traefik.http.services.uptime-kuma.loadbalancer.server.port=3001" - # No Authelia - public status page - # Code Server - VS Code in browser # Access at: https://code.${DOMAIN} code-server: diff --git a/docs/quick-reference.md b/docs/quick-reference.md index 2215f99..416808e 100644 --- a/docs/quick-reference.md +++ b/docs/quick-reference.md @@ -11,25 +11,53 @@ ### Service Management ```bash -# Start all services in a compose file -docker compose -f docker-compose/infrastructure.yml up -d +# Start all services in a compose file (from stack directory) +cd /opt/stacks/stack-name/ +docker compose up -d -# Start specific service -docker compose -f docker-compose/infrastructure.yml up -d service-name +# Start all services (from anywhere, using full path) +docker compose -f /opt/stacks/stack-name/docker-compose.yml up -d -# Stop all services -docker compose -f docker-compose/infrastructure.yml down +# Start specific service (from stack directory) +cd /opt/stacks/stack-name/ +docker compose up -d service-name -# Stop specific service -docker compose -f docker-compose/infrastructure.yml stop service-name +# Start specific service (from anywhere) +docker compose -f /opt/stacks/stack-name/docker-compose.yml up -d service-name -# Restart service -docker compose -f docker-compose/file.yml restart service-name +# Stop all services (from stack directory) +cd /opt/stacks/stack-name/ +docker compose down -# Remove service and volumes -docker compose -f docker-compose/file.yml down -v +# Stop all services (from anywhere) +docker compose -f /opt/stacks/stack-name/docker-compose.yml down + +# Stop specific service (from stack directory) +cd /opt/stacks/stack-name/ +docker compose stop service-name + +# Stop specific service (from anywhere) +docker compose -f /opt/stacks/stack-name/docker-compose.yml stop service-name + +# Restart service (from stack directory) +cd /opt/stacks/stack-name/ +docker compose restart service-name + +# Restart service (from anywhere) +docker compose -f /opt/stacks/stack-name/docker-compose.yml restart service-name + +# Remove service and volumes (from stack directory) +cd /opt/stacks/stack-name/ +docker compose down -v + +# Remove service and volumes (from anywhere) +docker compose -f /opt/stacks/stack-name/docker-compose.yml down -v ``` +**Note:** There's more than one way to manage containers - use whichever is most convenient: +- Navigate to `/opt/stacks/stack-name/` and use short commands +- Use full paths with `-f` flag from anywhere in the system + ### Monitoring ```bash diff --git a/docs/services-reference.md b/docs/services-reference.md index 36ce0ef..9ebf56a 100644 --- a/docs/services-reference.md +++ b/docs/services-reference.md @@ -11,9 +11,13 @@ This document provides a comprehensive overview of all 60+ pre-configured servic | ├─ Traefik | Reverse proxy + SSL | ✓ | /opt/stacks/core/traefik | traefik.${DOMAIN} | | ├─ Authelia | SSO authentication | - | /opt/stacks/core/authelia | auth.${DOMAIN} | | └─ Gluetun | VPN (Surfshark) | - | /opt/stacks/core/gluetun | No UI | -| **🔧 infrastructure** (7) | | | | | +| **🔧 infrastructure** (12) | | | | | | ├─ Dockge | Stack manager (PRIMARY) | ✓ | /opt/stacks/infrastructure | dockge.${DOMAIN} | | ├─ Portainer | Container management | ✓ | /opt/stacks/infrastructure | portainer.${DOMAIN} | +| ├─ Authentik Server | SSO with web UI | ✓ | /opt/stacks/authentik | authentik.${DOMAIN} | +| │ ├─ authentik-worker | Background tasks | - | /opt/stacks/authentik | No UI | +| │ ├─ authentik-db | PostgreSQL | - | /opt/stacks/authentik | No UI | +| │ └─ authentik-redis | Cache/messaging | - | /opt/stacks/authentik | No UI | | ├─ Pi-hole | DNS + Ad blocking | ✓ | /opt/stacks/infrastructure | pihole.${DOMAIN} | | ├─ Watchtower | Auto container updates | - | /opt/stacks/infrastructure | No UI | | ├─ Dozzle | Docker log viewer | ✓ | /opt/stacks/infrastructure | dozzle.${DOMAIN} | @@ -35,7 +39,7 @@ This document provides a comprehensive overview of all 60+ pre-configured servic | ├─ Lazy Librarian | Book automation | ✓ | /opt/stacks/media-ext, /mnt/media | lazylibrarian.${DOMAIN} | | ├─ Mylar3 | Comic manager | ✓ | /opt/stacks/media-ext, /mnt/media | mylar.${DOMAIN} | | ├─ Calibre-Web | Ebook reader | ✓ | /opt/stacks/media-ext, /mnt/media | calibre.${DOMAIN} | -| ├─ Jellyseerr | Media requests | ✗ | /opt/stacks/media-ext | jellyseerr.${DOMAIN} | +| ├─ Jellyseerr | Media requests | ✓ | /opt/stacks/media-ext | jellyseerr.${DOMAIN} | | ├─ FlareSolverr | Cloudflare bypass | - | /opt/stacks/media-ext | No UI | | ├─ Tdarr Server | Transcoding server | ✓ | /opt/stacks/media-ext, /mnt/transcode | tdarr.${DOMAIN} | | ├─ Tdarr Node | Transcoding worker | - | /mnt/transcode-cache | No UI | @@ -63,22 +67,21 @@ This document provides a comprehensive overview of all 60+ pre-configured servic | │ └─ mediawiki-db | MariaDB | - | /opt/stacks/productivity | No UI | | └─ Form.io | Form builder | ✓ | /opt/stacks/productivity | forms.${DOMAIN} | | └─ formio-mongo | MongoDB | - | /opt/stacks/productivity | No UI | -| **🛠️ utilities** (7) | | | | | +| **🛠️ utilities** (6) | | | | | | ├─ Backrest | Backup (restic) | ✓ | /opt/stacks/utilities, /mnt/backups | backrest.${DOMAIN} | | ├─ Duplicati | Encrypted backups | ✓ | /opt/stacks/utilities, /mnt/backups | duplicati.${DOMAIN} | -| ├─ Uptime Kuma | Status monitoring | ✗ | /opt/stacks/utilities | status.${DOMAIN} | | ├─ Code Server | VS Code in browser | ✓ | /opt/stacks/utilities | code.${DOMAIN} | | ├─ Form.io | Form platform | ✓ | /opt/stacks/utilities | forms.${DOMAIN} | | │ └─ formio-mongo | MongoDB | - | /opt/stacks/utilities | No UI | | └─ Authelia-Redis | Session storage | - | /opt/stacks/utilities | No UI | -| **📈 monitoring** (7) | | | | | +| **📈 monitoring** (8) | | | | | | ├─ Prometheus | Metrics collection | ✓ | /opt/stacks/monitoring | prometheus.${DOMAIN} | | ├─ Grafana | Visualization | ✓ | /opt/stacks/monitoring | grafana.${DOMAIN} | | ├─ Loki | Log aggregation | - | /opt/stacks/monitoring | Via Grafana | | ├─ Promtail | Log shipper | - | /opt/stacks/monitoring | No UI | | ├─ Node Exporter | Host metrics | - | /opt/stacks/monitoring | No UI | | ├─ cAdvisor | Container metrics | - | /opt/stacks/monitoring | Internal :8080 | -| └─ Uptime Kuma | Uptime monitoring | ✗ | /opt/stacks/monitoring | status.${DOMAIN} | +| └─ Uptime Kuma | Uptime monitoring | ✓ | /opt/stacks/monitoring | status.${DOMAIN} | | **👨‍💻 development** (6) | | | | | | ├─ GitLab CE | Git + CI/CD | ✓ | /opt/stacks/development, /mnt/git | gitlab.${DOMAIN} | | ├─ PostgreSQL | SQL database | - | /opt/stacks/development | Port 5432 | @@ -145,7 +148,7 @@ labels: ### To Disable SSO on a Service -Remove or comment out the middleware line: +Comment out (don't remove) the middleware line: ```yaml labels: @@ -153,19 +156,37 @@ labels: - "traefik.http.routers.servicename.rule=Host(`servicename.${DOMAIN}`)" - "traefik.http.routers.servicename.entrypoints=websecure" - "traefik.http.routers.servicename.tls.certresolver=letsencrypt" - # - "traefik.http.routers.servicename.middlewares=authelia@docker" # ← Commented out + # - "traefik.http.routers.servicename.middlewares=authelia@docker" # ← Commented out (not removed) - "traefik.http.services.servicename.loadbalancer.server.port=8080" ``` After making changes, redeploy the service: + ```bash +# From inside the stack directory cd /opt/stacks/stack-name/ docker compose up -d + +# Or from anywhere, using the full path +docker compose -f /opt/stacks/stack-name/docker-compose.yml up -d +``` + +**Stopping a Service:** + +```bash +# From inside the stack directory +cd /opt/stacks/stack-name/ +docker compose down + +# Or from anywhere, using the full path +docker compose -f /opt/stacks/stack-name/docker-compose.yml down ``` **Use Cases for Development/Production:** -- **Development**: Enable SSO to protect services during testing -- **Production**: Disable SSO for services that need direct app/API access (Plex, Jellyfin, etc.) +- **Security First**: All services start with SSO enabled by default for maximum security +- **Development**: Keep SSO enabled to protect services during testing +- **Production**: Disable SSO only for services needing direct app/API access (Plex, Jellyfin) +- **Gradual Exposure**: Comment out SSO only when ready to expose a service - **Quick Toggle**: AI assistant can modify these labels automatically when you ask ## Authelia Customization @@ -264,20 +285,27 @@ notifier: - `/opt/stacks/core/authelia/configuration.yml` - Main settings - `/opt/stacks/core/authelia/users_database.yml` - User accounts -This is **by design** and makes Authelia perfect for AI management: +This is **by design** and makes Authelia perfect for AI management and security-first approach: - AI can read and modify YAML files - Version control friendly - No UI clicks required - Infrastructure as code +- Secure by default **Web UI Available For:** - Login page: `https://auth.${DOMAIN}` - User profile: Change password, enroll 2FA - Device enrollment: Manage trusted devices -**Alternatives with Web UI:** -If you need a web UI for user management: -- **Authentik**: More complex but has full web UI +**Alternative with Web UI: Authentik** +If you need a web UI for user management, Authentik is included in the infrastructure stack: +- **Authentik**: Full-featured SSO with web UI for user/group management +- Access at: `https://authentik.${DOMAIN}` +- Includes PostgreSQL database and Redis cache +- More complex but offers GUI-based configuration +- Deploy only if you need web-based user management + +**Other Alternatives:** - **Keycloak**: Enterprise-grade SSO with web UI - **Authelia + LDAP**: Use LDAP with web management (phpLDAPadmin, etc.) From 32974a582053cb126e0b3e706b8b137b00c052e5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 12 Jan 2026 03:47:53 +0000 Subject: [PATCH 14/16] Add Bitwarden, setup script, remove redundant files, update disk requirements, and add LinuxServer preference - Add Vaultwarden (Bitwarden) password manager to utilities.yml - Self-hosted password manager with web UI - SMTP configuration for email notifications - Admin token for management - Access at bitwarden.${DOMAIN} - Protected by Authelia SSO - Create automated first-run setup script (scripts/setup-homelab.sh) - Installs Docker Engine and Compose V2 - Configures user groups (sudo, docker) - Enables SSH for remote management - Detects NVIDIA GPU and provides manual driver installation instructions - Creates directory structure and Docker networks - Comprehensive instructions for post-setup deployment - Remove redundant compose files (now in core.yml) - Deleted authelia.yml, duckdns.yml, gluetun.yml, traefik.yml - All services consolidated into unified core.yml stack - Eliminates confusion and duplication - Update disk space requirements across documentation - Changed from "100GB+ system, 1TB+ media" to: - "120GB+ system drive (NVMe or SSD highly recommended)" - "2TB+ for media & additional disks for services like Nextcloud" - Updated in README.md and getting-started.md - Add preference for LinuxServer.io images - Updated copilot-instructions.md - LinuxServer images support PUID/PGID for proper file permissions - Preference noted in consistency guidelines - Update core stack documentation - Emphasize unified core.yml deployment - Add both deployment methods (cd to directory vs full path) - Update getting-started.md with correct deployment steps - Note removal of separate stack files - Add Bitwarden environment variables to .env.example - BITWARDEN_ADMIN_TOKEN, SIGNUPS_ALLOWED, INVITATIONS_ALLOWED - SMTP configuration for email notifications - Generation instructions included - Update services-reference.md - Add Vaultwarden to utilities section (now 7 services) - Update service count and access URLs All documentation now consistent with unified core stack approach and includes all requested features. Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- .env.example | 11 ++ .github/copilot-instructions.md | 10 +- README.md | 39 +++--- docker-compose/authelia.yml | 39 ------ docker-compose/duckdns.yml | 21 --- docker-compose/gluetun.yml | 81 ------------ docker-compose/traefik.yml | 43 ------ docker-compose/utilities.yml | 32 +++++ docs/getting-started.md | 33 +---- docs/services-reference.md | 3 +- scripts/README.md | 74 +++++++++++ scripts/setup-homelab.sh | 227 ++++++++++++++++++++++++++++++++ 12 files changed, 380 insertions(+), 233 deletions(-) delete mode 100644 docker-compose/authelia.yml delete mode 100644 docker-compose/duckdns.yml delete mode 100644 docker-compose/gluetun.yml delete mode 100644 docker-compose/traefik.yml create mode 100644 scripts/README.md create mode 100755 scripts/setup-homelab.sh diff --git a/.env.example b/.env.example index 2ed0d44..244b41e 100644 --- a/.env.example +++ b/.env.example @@ -174,6 +174,17 @@ JUPYTER_TOKEN=changeme # Pi-hole PIHOLE_PASSWORD=changeme +# Bitwarden (Vaultwarden) Password Manager +# Admin token: openssl rand -base64 48 +BITWARDEN_ADMIN_TOKEN=changeme-bitwarden-admin-token +BITWARDEN_SIGNUPS_ALLOWED=true # Set to false after creating accounts +BITWARDEN_INVITATIONS_ALLOWED=true +SMTP_HOST=smtp.gmail.com +SMTP_FROM=bitwarden@yourdomain.com +SMTP_PORT=587 +SMTP_SECURITY=starttls +# SMTP_USERNAME and SMTP_PASSWORD defined above + # Watchtower Notifications (optional) # WATCHTOWER_NOTIFICATION_URL= diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index baed45e..39f0fd6 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -26,6 +26,7 @@ You are an AI assistant specialized in managing Docker-based homelab infrastruct - Use the same network naming patterns - Maintain uniform volume mount structures - Apply consistent environment variable patterns +- **Prefer LinuxServer.io images** when available (they support PUID/PGID for proper file permissions) ### 4. Stack-Aware Changes - Before making changes, consider the impact on the entire server stack @@ -280,7 +281,7 @@ environment: ## Core Infrastructure Stack -The `core` stack contains the four essential services that must be deployed **FIRST**: +The `core` stack (located at `/opt/stacks/core/docker-compose.yml`) contains the four essential services that must be deployed **FIRST**: 1. **DuckDNS** - Dynamic DNS updater for Let's Encrypt 2. **Traefik** - Reverse proxy with automatic SSL certificates @@ -292,15 +293,22 @@ The `core` stack contains the four essential services that must be deployed **FI - Simplifies initial deployment (one command) - Easier to manage core infrastructure together - Reduces network configuration complexity +- All core services in `/opt/stacks/core/` directory **Deployment:** ```bash +# From within the directory cd /opt/stacks/core/ docker compose up -d + +# Or from anywhere with full path +docker compose -f /opt/stacks/core/docker-compose.yml up -d ``` All other stacks depend on the core stack being deployed first. +**Note:** The separate `authelia.yml`, `duckdns.yml`, `gluetun.yml`, and `traefik.yml` files have been removed to eliminate redundancy. All these services are now in the unified `core.yml` stack. + ## Toggling SSO (Authelia) On/Off You can easily enable or disable SSO protection for any service by modifying its Traefik labels. diff --git a/README.md b/README.md index b0f9f4a..aa1785c 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ The infrastructure uses Traefik for reverse proxy with automatic SSL, Authelia f - VS Code with GitHub Copilot extension (for AI assistance) - A domain from DuckDNS (free) - Surfshark VPN account (optional, for VPN features) +- Sufficient disk space: 120GB+ system drive (NVMe or SSD highly recommended), 2TB+ for media & additional disks for services like Nextcloud that require lots of space ### Quick Setup (Dockge Structure) @@ -62,31 +63,29 @@ The infrastructure uses Traefik for reverse proxy with automatic SSL, Authelia f docker network create media-network ``` -5. **Deploy core infrastructure (in order):** +5. **Deploy core infrastructure stack:** ```bash - # 1. DuckDNS (Dynamic DNS) - mkdir -p /opt/stacks/duckdns && cp docker-compose/duckdns.yml /opt/stacks/duckdns/docker-compose.yml - cd /opt/stacks/duckdns && docker compose up -d + # Deploy the unified core stack (DuckDNS, Traefik, Authelia, Gluetun) + mkdir -p /opt/stacks/core + cp docker-compose/core.yml /opt/stacks/core/docker-compose.yml + cp -r config-templates/traefik /opt/stacks/core/ + cp -r config-templates/authelia /opt/stacks/core/ - # 2. Traefik (Reverse Proxy with SSL) - mkdir -p /opt/stacks/traefik/dynamic - cp docker-compose/traefik.yml /opt/stacks/traefik/docker-compose.yml - cp config-templates/traefik/* /opt/stacks/traefik/ - cd /opt/stacks/traefik && docker compose up -d + # From within the directory + cd /opt/stacks/core && docker compose up -d - # 3. Authelia (SSO) - mkdir -p /opt/stacks/authelia - cp docker-compose/authelia.yml /opt/stacks/authelia/docker-compose.yml - cp config-templates/authelia/* /opt/stacks/authelia/ - cd /opt/stacks/authelia && docker compose up -d - - # 4. Dockge (Stack Manager) - mkdir -p /opt/stacks/infrastructure - cp docker-compose/infrastructure.yml /opt/stacks/infrastructure/docker-compose.yml - cd /opt/stacks/infrastructure && docker compose up -d dockge + # OR from anywhere with full path + docker compose -f /opt/stacks/core/docker-compose.yml up -d ``` -6. **Access Dockge:** +6. **Deploy infrastructure stack (includes Dockge):** + ```bash + mkdir -p /opt/stacks/infrastructure + cp docker-compose/infrastructure.yml /opt/stacks/infrastructure/docker-compose.yml + cd /opt/stacks/infrastructure && docker compose up -d + ``` + +7. **Access Dockge:** Open `https://dockge.yourdomain.duckdns.org` (use Authelia login) Now deploy remaining stacks through Dockge's UI! diff --git a/docker-compose/authelia.yml b/docker-compose/authelia.yml deleted file mode 100644 index 0dd329c..0000000 --- a/docker-compose/authelia.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Authelia SSO Stack -# Single Sign-On authentication for all services -# Place in /opt/stacks/authelia/docker-compose.yml - -services: - authelia: - image: authelia/authelia:4.37 - container_name: authelia - restart: unless-stopped - networks: - - traefik-network - volumes: - - /opt/stacks/authelia/configuration.yml:/config/configuration.yml:ro - - /opt/stacks/authelia/users_database.yml:/config/users_database.yml - - authelia-data:/config - environment: - - TZ=${TZ} - - AUTHELIA_JWT_SECRET=${AUTHELIA_JWT_SECRET} - - AUTHELIA_SESSION_SECRET=${AUTHELIA_SESSION_SECRET} - - AUTHELIA_STORAGE_ENCRYPTION_KEY=${AUTHELIA_STORAGE_ENCRYPTION_KEY} - - AUTHELIA_NOTIFIER_SMTP_PASSWORD=${SMTP_PASSWORD} # If using email notifications - labels: - - "traefik.enable=true" - - "traefik.http.routers.authelia.rule=Host(`auth.${DOMAIN}`)" - - "traefik.http.routers.authelia.entrypoints=websecure" - - "traefik.http.routers.authelia.tls.certresolver=letsencrypt" - - "traefik.http.services.authelia.loadbalancer.server.port=9091" - # Authelia middleware for other services - - "traefik.http.middlewares.authelia.forwardauth.address=http://authelia:9091/api/verify?rd=https://auth.${DOMAIN}" - - "traefik.http.middlewares.authelia.forwardauth.trustForwardHeader=true" - - "traefik.http.middlewares.authelia.forwardauth.authResponseHeaders=Remote-User,Remote-Groups,Remote-Name,Remote-Email" - -volumes: - authelia-data: - driver: local - -networks: - traefik-network: - external: true diff --git a/docker-compose/duckdns.yml b/docker-compose/duckdns.yml deleted file mode 100644 index 16dec0a..0000000 --- a/docker-compose/duckdns.yml +++ /dev/null @@ -1,21 +0,0 @@ -# DuckDNS Dynamic DNS Stack -# Updates your DuckDNS domain with current public IP -# Place in /opt/stacks/duckdns/docker-compose.yml - -services: - duckdns: - image: lscr.io/linuxserver/duckdns:latest - container_name: duckdns - restart: unless-stopped - environment: - - PUID=${PUID:-1000} - - PGID=${PGID:-1000} - - TZ=${TZ} - - SUBDOMAINS=${DUCKDNS_SUBDOMAINS} # Your subdomain(s), comma separated - - TOKEN=${DUCKDNS_TOKEN} # Your DuckDNS token - - UPDATE_IP=ipv4 # or ipv6, or both - volumes: - - /opt/stacks/duckdns/config:/config - labels: - - "homelab.category=infrastructure" - - "homelab.description=Dynamic DNS updater" diff --git a/docker-compose/gluetun.yml b/docker-compose/gluetun.yml deleted file mode 100644 index 534d376..0000000 --- a/docker-compose/gluetun.yml +++ /dev/null @@ -1,81 +0,0 @@ -# Gluetun VPN Stack -# VPN client for routing services through Surfshark (or other VPN providers) -# Place in /opt/stacks/gluetun/docker-compose.yml -# Services that need VPN use: network_mode: "service:gluetun" - -services: - gluetun: - image: qmcgaw/gluetun:latest - container_name: gluetun - restart: unless-stopped - cap_add: - - NET_ADMIN - devices: - - /dev/net/tun:/dev/net/tun - networks: - - gluetun-network - - traefik-network - ports: - # qBittorrent ports (service runs through Gluetun) - - "8080:8080" # qBittorrent WebUI - - "6881:6881" # qBittorrent TCP - - "6881:6881/udp" # qBittorrent UDP - environment: - - VPN_SERVICE_PROVIDER=surfshark - - VPN_TYPE=wireguard # or openvpn - - WIREGUARD_PRIVATE_KEY=${SURFSHARK_PRIVATE_KEY} - - WIREGUARD_ADDRESSES=${SURFSHARK_ADDRESSES} - - SERVER_COUNTRIES=${VPN_COUNTRY:-Netherlands} # Preferred VPN server country - - TZ=${TZ} - # For OpenVPN instead of WireGuard: - # - OPENVPN_USER=${SURFSHARK_USERNAME} - # - OPENVPN_PASSWORD=${SURFSHARK_PASSWORD} - volumes: - - /opt/stacks/gluetun/config:/gluetun - labels: - - "homelab.category=infrastructure" - - "homelab.description=VPN client for secure routing (Surfshark)" - - # qBittorrent - Torrent client routing through VPN - # Access at: https://qbit.yourdomain.duckdns.org - qbittorrent: - image: lscr.io/linuxserver/qbittorrent:4.6.2 - container_name: qbittorrent - network_mode: "service:gluetun" # Routes all traffic through VPN - depends_on: - - gluetun - volumes: - - /opt/stacks/qbittorrent/config:/config - - /mnt/downloads:/downloads # Large downloads on separate drive - environment: - - PUID=${PUID:-1000} - - PGID=${PGID:-1000} - - TZ=${TZ} - - WEBUI_PORT=8080 - labels: - - "homelab.category=media" - - "homelab.description=Torrent download client (via VPN)" - # Traefik labels (applied to Gluetun since qBittorrent uses its network) - # Configure these on the Gluetun container instead: - - # Traefik routing for qBittorrent (via Gluetun) - # Since qBittorrent uses Gluetun's network, we add a sidecar label container - qbit-labels: - image: alpine:latest - container_name: qbit-labels - command: tail -f /dev/null - networks: - - traefik-network - labels: - - "traefik.enable=true" - - "traefik.http.routers.qbittorrent.rule=Host(`qbit.${DOMAIN}`)" - - "traefik.http.routers.qbittorrent.entrypoints=websecure" - - "traefik.http.routers.qbittorrent.tls.certresolver=letsencrypt" - - "traefik.http.routers.qbittorrent.middlewares=authelia@docker" - - "traefik.http.services.qbittorrent.loadbalancer.server.url=http://gluetun:8080" - -networks: - gluetun-network: - driver: bridge - traefik-network: - external: true diff --git a/docker-compose/traefik.yml b/docker-compose/traefik.yml deleted file mode 100644 index adde6b1..0000000 --- a/docker-compose/traefik.yml +++ /dev/null @@ -1,43 +0,0 @@ -# Traefik Reverse Proxy Stack -# Main reverse proxy with Let's Encrypt SSL automation -# Place in /opt/stacks/traefik/docker-compose.yml - -services: - traefik: - image: traefik:v2.11 - container_name: traefik - restart: unless-stopped - security_opt: - - no-new-privileges:true - networks: - - traefik-network - ports: - - "80:80" # HTTP - - "443:443" # HTTPS - - "8080:8080" # Dashboard (protect with Authelia) - volumes: - - /etc/localtime:/etc/localtime:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - - /opt/stacks/traefik/traefik.yml:/traefik.yml:ro - - /opt/stacks/traefik/dynamic:/dynamic:ro - - /opt/stacks/traefik/acme.json:/acme.json - environment: - - CF_DNS_API_TOKEN=${CF_DNS_API_TOKEN} # If using Cloudflare DNS challenge - - DUCKDNS_TOKEN=${DUCKDNS_TOKEN} # If using DuckDNS - labels: - - "traefik.enable=true" - # Dashboard - - "traefik.http.routers.traefik.rule=Host(`traefik.${DOMAIN}`)" - - "traefik.http.routers.traefik.entrypoints=websecure" - - "traefik.http.routers.traefik.tls.certresolver=letsencrypt" - - "traefik.http.routers.traefik.middlewares=authelia@docker" - - "traefik.http.routers.traefik.service=api@internal" - # Global HTTP to HTTPS redirect - - "traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`)" - - "traefik.http.routers.http-catchall.entrypoints=web" - - "traefik.http.routers.http-catchall.middlewares=redirect-to-https" - - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" - -networks: - traefik-network: - external: true diff --git a/docker-compose/utilities.yml b/docker-compose/utilities.yml index 62acd9f..1eea30a 100644 --- a/docker-compose/utilities.yml +++ b/docker-compose/utilities.yml @@ -126,6 +126,38 @@ services: - "homelab.category=utilities" - "homelab.description=Form.io database" + # Bitwarden (Vaultwarden) - Password manager + # Access at: https://bitwarden.${DOMAIN} + vaultwarden: + image: vaultwarden/server:latest + container_name: vaultwarden + restart: unless-stopped + networks: + - homelab-network + - traefik-network + volumes: + - /opt/stacks/vaultwarden/data:/data + environment: + - DOMAIN=https://bitwarden.${DOMAIN} + - SIGNUPS_ALLOWED=${BITWARDEN_SIGNUPS_ALLOWED:-true} + - INVITATIONS_ALLOWED=${BITWARDEN_INVITATIONS_ALLOWED:-true} + - ADMIN_TOKEN=${BITWARDEN_ADMIN_TOKEN} + - SMTP_HOST=${SMTP_HOST} + - SMTP_FROM=${SMTP_FROM} + - SMTP_PORT=${SMTP_PORT:-587} + - SMTP_SECURITY=${SMTP_SECURITY:-starttls} + - SMTP_USERNAME=${SMTP_USERNAME} + - SMTP_PASSWORD=${SMTP_PASSWORD} + labels: + - "homelab.category=utilities" + - "homelab.description=Self-hosted password manager (Bitwarden)" + - "traefik.enable=true" + - "traefik.http.routers.vaultwarden.rule=Host(`bitwarden.${DOMAIN}`)" + - "traefik.http.routers.vaultwarden.entrypoints=websecure" + - "traefik.http.routers.vaultwarden.tls.certresolver=letsencrypt" + - "traefik.http.routers.vaultwarden.middlewares=authelia@docker" + - "traefik.http.services.vaultwarden.loadbalancer.server.port=80" + # Authelia Redis - Session storage for Authelia # No web UI - backend service authelia-redis: diff --git a/docs/getting-started.md b/docs/getting-started.md index 929c75f..7ba1052 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -11,7 +11,7 @@ Before you begin, ensure you have: - [ ] Docker Compose V2 installed - [ ] Git installed - [ ] At least 8GB RAM (16GB+ recommended) -- [ ] Sufficient disk space (100GB+ system, 1TB+ for media recommended) +- [ ] Sufficient disk space: 120GB+ system drive (NVMe or SSD highly recommended), 2TB+ for media & additional disks for services like Nextcloud that require lots of space - [ ] Static IP address for your server (or DHCP reservation) - [ ] DuckDNS account (free) with a domain - [ ] Surfshark VPN account (optional, for VPN features) @@ -174,10 +174,14 @@ nano users_database.yml # Copy .env file to core stack cp ~/AI-Homelab/.env /opt/stacks/core/.env -# Deploy the entire core stack +# Deploy the entire core stack - use either method: +# Method 1: From within directory cd /opt/stacks/core docker compose up -d +# Method 2: From anywhere with full path +docker compose -f /opt/stacks/core/docker-compose.yml up -d + # Check logs to ensure everything is running docker compose logs -f @@ -198,31 +202,6 @@ docker compose logs -f - If Authelia won't start, check your password hash and configuration.yml - If Gluetun fails, verify your Surfshark credentials in .env -```bash -# Create stack directory -mkdir -p /opt/stacks/infrastructure - -# Copy compose file -cp ~/AI-Homelab/docker-compose/infrastructure.yml /opt/stacks/infrastructure/docker-compose.yml - -# Create necessary subdirectories -mkdir -p /opt/dockge/data -mkdir -p /opt/stacks/pihole/{etc-pihole,etc-dnsmasq.d} -mkdir -p /opt/stacks/glances/config - -# Copy .env -cp ~/AI-Homelab/.env /opt/stacks/infrastructure/.env - -# Deploy Dockge first -cd /opt/stacks/infrastructure -docker compose up -d dockge - -# Access Dockge at https://dockge.yourdomain.duckdns.org - -# Deploy remaining infrastructure services -docker compose up -d -``` - ## Step 8: Deploy Infrastructure Services (Dockge) ```bash diff --git a/docs/services-reference.md b/docs/services-reference.md index 9ebf56a..9eb0113 100644 --- a/docs/services-reference.md +++ b/docs/services-reference.md @@ -67,7 +67,8 @@ This document provides a comprehensive overview of all 60+ pre-configured servic | │ └─ mediawiki-db | MariaDB | - | /opt/stacks/productivity | No UI | | └─ Form.io | Form builder | ✓ | /opt/stacks/productivity | forms.${DOMAIN} | | └─ formio-mongo | MongoDB | - | /opt/stacks/productivity | No UI | -| **🛠️ utilities** (6) | | | | | +| **🛠️ utilities** (7) | | | | | +| ├─ Vaultwarden | Password manager | ✓ | /opt/stacks/utilities | bitwarden.${DOMAIN} | | ├─ Backrest | Backup (restic) | ✓ | /opt/stacks/utilities, /mnt/backups | backrest.${DOMAIN} | | ├─ Duplicati | Encrypted backups | ✓ | /opt/stacks/utilities, /mnt/backups | duplicati.${DOMAIN} | | ├─ Code Server | VS Code in browser | ✓ | /opt/stacks/utilities | code.${DOMAIN} | diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..59afdf3 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,74 @@ +# AI-Homelab Setup Scripts + +## setup-homelab.sh + +Automated first-run setup script for preparing a fresh Debian installation for AI-Homelab deployment. + +### What It Does + +1. **System Update** - Updates all system packages +2. **Install Dependencies** - Installs required packages (curl, git, etc.) +3. **Install Docker** - Adds Docker repository and installs Docker Engine with Compose V2 +4. **Configure User Groups** - Adds user to sudo and docker groups +5. **Configure SSH** - Enables and starts SSH server for remote access +6. **Detect NVIDIA GPU** - Checks for NVIDIA graphics card and provides manual driver installation instructions +7. **Create Directories** - Sets up `/opt/stacks`, `/opt/dockge`, `/mnt/media`, `/mnt/downloads` +8. **Create Docker Networks** - Creates homelab-network, traefik-network, and media-network + +### Usage + +```bash +# Download the repository +git clone https://github.com/kelinfoxy/AI-Homelab.git +cd AI-Homelab + +# Make the script executable (if needed) +chmod +x scripts/setup-homelab.sh + +# Run with sudo +sudo ./scripts/setup-homelab.sh +``` + +### After Running + +1. Log out and log back in for group changes to take effect +2. Edit `.env` file with your configuration +3. Deploy the core infrastructure stack +4. Deploy the infrastructure stack (includes Dockge) +5. Access Dockge to manage remaining stacks + +### NVIDIA GPU Support + +If an NVIDIA GPU is detected, the script will provide instructions for manual driver installation: + +1. Identify your GPU model from the output +2. Visit https://www.nvidia.com/Download/index.aspx +3. Download the official driver for your GPU +4. Run the installer: `sudo bash NVIDIA-Linux-x86_64-XXX.XX.run` +5. Install container toolkit: + ```bash + sudo apt-get install -y nvidia-container-toolkit + sudo nvidia-ctk runtime configure --runtime=docker + sudo systemctl restart docker + ``` + +This manual approach avoids driver conflicts that often occur with automated installation methods. + +### Requirements + +- Fresh Debian installation (Debian 11 or 12) +- Root access (via sudo) +- Internet connection + +### Tested On + +- Debian 11 (Bullseye) +- Debian 12 (Bookworm) + +### Notes + +- The script is idempotent - safe to run multiple times +- Creates directories with proper ownership +- Configures Docker networks automatically +- SSH is enabled for remote management +- NVIDIA driver installation requires manual intervention for reliability diff --git a/scripts/setup-homelab.sh b/scripts/setup-homelab.sh new file mode 100755 index 0000000..963d0f9 --- /dev/null +++ b/scripts/setup-homelab.sh @@ -0,0 +1,227 @@ +#!/bin/bash +# AI-Homelab First-Run Setup Script +# This script prepares a fresh Debian installation for homelab deployment +# Run as: sudo ./setup-homelab.sh + +set -e # Exit on error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Log functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check if running as root +if [ "$EUID" -ne 0 ]; then + log_error "Please run as root (use: sudo ./setup-homelab.sh)" + exit 1 +fi + +# Get the actual user who invoked sudo +ACTUAL_USER="${SUDO_USER:-$USER}" +if [ "$ACTUAL_USER" = "root" ]; then + log_error "Please run this script with sudo, not as root user" + exit 1 +fi + +log_info "Setting up AI-Homelab for user: $ACTUAL_USER" +echo "" + +# Step 1: System Update +log_info "Step 1/8: Updating system packages..." +apt-get update && apt-get upgrade -y +log_success "System updated successfully" +echo "" + +# Step 2: Install Required Packages +log_info "Step 2/8: Installing required packages..." +apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg \ + lsb-release \ + software-properties-common \ + git \ + openssh-server \ + sudo \ + pciutils \ + net-tools + +log_success "Required packages installed" +echo "" + +# Step 3: Install Docker +log_info "Step 3/8: Installing Docker..." +if command -v docker &> /dev/null; then + log_warning "Docker is already installed ($(docker --version))" +else + # Add Docker's official GPG key + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + + # Add the repository to Apt sources + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null + + # Update and install Docker + apt-get update + apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + + log_success "Docker installed successfully ($(docker --version))" +fi +echo "" + +# Step 4: Configure User Groups +log_info "Step 4/8: Configuring user groups..." + +# Add user to sudo group if not already +if groups "$ACTUAL_USER" | grep -q '\bsudo\b'; then + log_warning "User $ACTUAL_USER is already in sudo group" +else + usermod -aG sudo "$ACTUAL_USER" + log_success "User $ACTUAL_USER added to sudo group" +fi + +# Add user to docker group +if groups "$ACTUAL_USER" | grep -q '\bdocker\b'; then + log_warning "User $ACTUAL_USER is already in docker group" +else + usermod -aG docker "$ACTUAL_USER" + log_success "User $ACTUAL_USER added to docker group" +fi +echo "" + +# Step 5: Configure SSH +log_info "Step 5/8: Configuring SSH server..." +systemctl enable ssh +systemctl start ssh + +# Check if SSH is running +if systemctl is-active --quiet ssh; then + SSH_PORT=$(grep "^Port" /etc/ssh/sshd_config | awk '{print $2}') + SSH_PORT=${SSH_PORT:-22} + log_success "SSH server is running on port $SSH_PORT" +else + log_warning "SSH server failed to start, check configuration" +fi +echo "" + +# Step 6: Detect and Install NVIDIA Drivers (if applicable) +log_info "Step 6/8: Checking for NVIDIA GPU..." + +# Detect NVIDIA GPU +if lspci | grep -i nvidia > /dev/null; then + log_info "NVIDIA GPU detected:" + lspci | grep -i nvidia + echo "" + + log_warning "NVIDIA GPU found, but driver installation requires manual intervention." + log_info "For best results, please follow these steps manually:" + echo "" + echo " 1. Identify your GPU model from the output above" + echo " 2. Visit: https://www.nvidia.com/Download/index.aspx" + echo " 3. Download the official driver for your GPU" + echo " 4. Run the downloaded installer (example): sudo bash NVIDIA-Linux-x86_64-XXX.XX.run" + echo "" + log_info "After installing NVIDIA drivers, run:" + echo " sudo apt-get install -y nvidia-container-toolkit" + echo " sudo nvidia-ctk runtime configure --runtime=docker" + echo " sudo systemctl restart docker" + echo "" + log_warning "Skipping automatic NVIDIA driver installation to avoid conflicts" + echo "" +else + log_info "No NVIDIA GPU detected, skipping driver installation" + echo "" +fi + +# Step 7: Create Directory Structure +log_info "Step 7/8: Creating directory structure..." +mkdir -p /opt/stacks +mkdir -p /opt/dockge/data +mkdir -p /mnt/media +mkdir -p /mnt/downloads + +# Set ownership +chown -R "$ACTUAL_USER:$ACTUAL_USER" /opt/stacks +chown -R "$ACTUAL_USER:$ACTUAL_USER" /opt/dockge +chown -R "$ACTUAL_USER:$ACTUAL_USER" /mnt/media +chown -R "$ACTUAL_USER:$ACTUAL_USER" /mnt/downloads + +log_success "Directory structure created" +echo "" + +# Step 8: Create Docker Networks +log_info "Step 8/8: Creating Docker networks..." +su - "$ACTUAL_USER" -c "docker network create homelab-network 2>/dev/null || true" +su - "$ACTUAL_USER" -c "docker network create traefik-network 2>/dev/null || true" +su - "$ACTUAL_USER" -c "docker network create media-network 2>/dev/null || true" +log_success "Docker networks created" +echo "" + +# Final Summary +echo "" +echo "==========================================" +log_success "AI-Homelab setup completed successfully!" +echo "==========================================" +echo "" +log_info "Next steps:" +echo "" +echo " 1. Log out and log back in for group changes to take effect" +echo " (or run: newgrp docker)" +echo "" +echo " 2. Navigate to your AI-Homelab repository:" +echo " cd ~/AI-Homelab" +echo "" +echo " 3. Edit the .env file with your configuration:" +echo " cp .env.example .env" +echo " nano .env" +echo "" +echo " 4. Deploy the core infrastructure stack:" +echo " mkdir -p /opt/stacks/core" +echo " cp docker-compose/core.yml /opt/stacks/core/docker-compose.yml" +echo " cp -r config-templates/traefik /opt/stacks/core/" +echo " cp -r config-templates/authelia /opt/stacks/core/" +echo " cd /opt/stacks/core && docker compose up -d" +echo "" +echo " 5. Deploy the infrastructure stack (includes Dockge):" +echo " mkdir -p /opt/stacks/infrastructure" +echo " cp docker-compose/infrastructure.yml /opt/stacks/infrastructure/docker-compose.yml" +echo " cd /opt/stacks/infrastructure && docker compose up -d" +echo "" +echo " 6. Access Dockge at: https://dockge.yourdomain.duckdns.org" +echo " (Use your configured domain and Authelia credentials)" +echo "" +echo "==========================================" + +if lspci | grep -i nvidia > /dev/null; then + echo "" + log_warning "REMINDER: Manual NVIDIA driver installation required" + echo " See instructions above in Step 6" + echo "==========================================" +fi + +echo "" +log_info "Setup complete! Please log out and log back in." From afe057c5432a3e39a43aa6957abc3b831fdf954c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 12 Jan 2026 03:49:06 +0000 Subject: [PATCH 15/16] Fix Vaultwarden SSO - disable for browser extension and mobile app compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove Authelia middleware from Vaultwarden - Add comment explaining SSO bypass for app compatibility - Update services-reference.md to show SSO disabled (✗) Vaultwarden requires direct access for browser extensions and mobile apps to function properly, similar to Jellyfin and Plex. Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- docker-compose/utilities.yml | 4 +++- docs/services-reference.md | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-compose/utilities.yml b/docker-compose/utilities.yml index 1eea30a..0ba3b32 100644 --- a/docker-compose/utilities.yml +++ b/docker-compose/utilities.yml @@ -128,6 +128,7 @@ services: # Bitwarden (Vaultwarden) - Password manager # Access at: https://bitwarden.${DOMAIN} + # Note: SSO disabled for browser extension and mobile app compatibility vaultwarden: image: vaultwarden/server:latest container_name: vaultwarden @@ -155,7 +156,8 @@ services: - "traefik.http.routers.vaultwarden.rule=Host(`bitwarden.${DOMAIN}`)" - "traefik.http.routers.vaultwarden.entrypoints=websecure" - "traefik.http.routers.vaultwarden.tls.certresolver=letsencrypt" - - "traefik.http.routers.vaultwarden.middlewares=authelia@docker" + # SSO disabled for browser extension and mobile app compatibility + # - "traefik.http.routers.vaultwarden.middlewares=authelia@docker" - "traefik.http.services.vaultwarden.loadbalancer.server.port=80" # Authelia Redis - Session storage for Authelia diff --git a/docs/services-reference.md b/docs/services-reference.md index 9eb0113..a2dcd9c 100644 --- a/docs/services-reference.md +++ b/docs/services-reference.md @@ -68,7 +68,7 @@ This document provides a comprehensive overview of all 60+ pre-configured servic | └─ Form.io | Form builder | ✓ | /opt/stacks/productivity | forms.${DOMAIN} | | └─ formio-mongo | MongoDB | - | /opt/stacks/productivity | No UI | | **🛠️ utilities** (7) | | | | | -| ├─ Vaultwarden | Password manager | ✓ | /opt/stacks/utilities | bitwarden.${DOMAIN} | +| ├─ Vaultwarden | Password manager | ✗ | /opt/stacks/utilities | bitwarden.${DOMAIN} | | ├─ Backrest | Backup (restic) | ✓ | /opt/stacks/utilities, /mnt/backups | backrest.${DOMAIN} | | ├─ Duplicati | Encrypted backups | ✓ | /opt/stacks/utilities, /mnt/backups | duplicati.${DOMAIN} | | ├─ Code Server | VS Code in browser | ✓ | /opt/stacks/utilities | code.${DOMAIN} | From b5eac0b1e99d5c0f3c854275bb500c6833bb9fbc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 12 Jan 2026 04:17:57 +0000 Subject: [PATCH 16/16] Create deployment script and restructure Quick Setup with automated workflow - Create deploy-homelab.sh script for automated deployment - Validates prerequisites (Docker, .env file) - Creates required directories and networks - Deploys core stack (DuckDNS, Traefik, Authelia, Gluetun) - Deploys infrastructure stack (Dockge, Portainer, etc.) - Waits for Dockge web UI to be ready - Automatically opens browser to Dockge when accessible - Comprehensive error checking and user feedback - Restructure README Quick Setup section - Step 1: Clone repository - Step 2: (Optional) Run setup-homelab.sh for fresh Debian installations - Step 3: Create and configure .env file - Step 4: Run deploy-homelab.sh (automated deployment with browser launch) - Step 5: Deploy additional stacks through Dockge web UI - Include manual deployment alternative for advanced users - Update scripts/README.md documentation - Document both setup-homelab.sh and deploy-homelab.sh - Clear usage instructions for each script - Browser detection details - Troubleshooting section - Manual deployment alternatives Key improvements: - User only needs to run 2 scripts (setup optional, deploy required) - Automated browser launch when Dockge is ready - No manual directory creation or network setup needed - Smart wait logic ensures UI is accessible before opening browser - Better user experience with clear progress indicators Co-authored-by: kelinfoxy <67766943+kelinfoxy@users.noreply.github.com> --- README.md | 82 ++++++++----- scripts/README.md | 122 ++++++++++++++++++- scripts/deploy-homelab.sh | 239 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 412 insertions(+), 31 deletions(-) create mode 100755 scripts/deploy-homelab.sh diff --git a/README.md b/README.md index aa1785c..2586784 100644 --- a/README.md +++ b/README.md @@ -43,52 +43,80 @@ The infrastructure uses Traefik for reverse proxy with automatic SSL, Authelia f cd AI-Homelab ``` -2. **Create environment file:** +2. **(Optional) Run first-run setup script:** + + **For fresh Debian installations only**, this automated script will: + - Update system and install Docker Engine + Compose V2 + - Configure user groups (sudo, docker) and SSH access + - Detect NVIDIA GPU and provide driver installation guidance + - Create directory structure and Docker networks + + ```bash + sudo ./scripts/setup-homelab.sh + ``` + + After completion, log out and log back in for group changes to take effect. + + **Skip this step if Docker is already installed and configured.** + +3. **Create and configure environment file:** ```bash cp .env.example .env - # Edit .env with your values (domain, API keys, passwords) - nano .env + nano .env # Edit with your domain, API keys, and passwords ``` + + **Required variables:** + - `DOMAIN` - Your DuckDNS domain (e.g., yourhomelab.duckdns.org) + - `DUCKDNS_TOKEN` - Token from DuckDNS.org + - `TZ` - Your timezone (e.g., America/New_York) + - Authelia user credentials + - API keys for services you plan to use -3. **Create required directories:** +4. **Run deployment script:** + + This automated script will: + - Create required directories + - Verify Docker networks exist + - Deploy core infrastructure (DuckDNS, Traefik, Authelia, Gluetun) + - Deploy infrastructure stack with Dockge + - Wait for Dockge to be ready + - Automatically open Dockge in your browser + ```bash - sudo mkdir -p /opt/stacks - sudo chown -R $USER:$USER /opt/stacks + ./scripts/deploy-homelab.sh ``` - -4. **Create Docker networks:** + + The script will automatically open `https://dockge.yourdomain.duckdns.org` when ready. + + **Manual deployment alternative:** ```bash - docker network create homelab-network - docker network create traefik-network - docker network create media-network - ``` - -5. **Deploy core infrastructure stack:** - ```bash - # Deploy the unified core stack (DuckDNS, Traefik, Authelia, Gluetun) + # Deploy core stack mkdir -p /opt/stacks/core cp docker-compose/core.yml /opt/stacks/core/docker-compose.yml cp -r config-templates/traefik /opt/stacks/core/ cp -r config-templates/authelia /opt/stacks/core/ - - # From within the directory + cp .env /opt/stacks/core/ cd /opt/stacks/core && docker compose up -d - # OR from anywhere with full path - docker compose -f /opt/stacks/core/docker-compose.yml up -d - ``` - -6. **Deploy infrastructure stack (includes Dockge):** - ```bash + # Deploy infrastructure stack mkdir -p /opt/stacks/infrastructure cp docker-compose/infrastructure.yml /opt/stacks/infrastructure/docker-compose.yml + cp .env /opt/stacks/infrastructure/ cd /opt/stacks/infrastructure && docker compose up -d + + # Manually open: https://dockge.yourdomain.duckdns.org ``` -7. **Access Dockge:** - Open `https://dockge.yourdomain.duckdns.org` (use Authelia login) +5. **Deploy additional stacks through Dockge:** - Now deploy remaining stacks through Dockge's UI! + Log in to Dockge with your Authelia credentials and deploy additional stacks: + - `dashboards.yml` - Homepage and Homarr dashboards + - `media.yml` - Plex, Jellyfin, Sonarr, Radarr, Prowlarr, qBittorrent + - `media-extended.yml` - Readarr, Lidarr, Lazy Librarian, Mylar3, Calibre-Web + - `homeassistant.yml` - Home Assistant, ESPHome, Node-RED, and accessories + - `productivity.yml` - Nextcloud, Gitea, WordPress, wikis + - `monitoring.yml` - Grafana, Prometheus, Loki + - `utilities.yml` - Backups, code editors, password manager ## Repository Structure diff --git a/scripts/README.md b/scripts/README.md index 59afdf3..87a509b 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,8 +1,13 @@ # AI-Homelab Setup Scripts +This directory contains two scripts for automated AI-Homelab deployment: + +1. **setup-homelab.sh** - System preparation (optional, for fresh installations) +2. **deploy-homelab.sh** - Core infrastructure deployment + ## setup-homelab.sh -Automated first-run setup script for preparing a fresh Debian installation for AI-Homelab deployment. +Automated first-run setup script for preparing a fresh Debian installation for AI-Homelab deployment. **This is optional** - skip if Docker is already installed and configured. ### What It Does @@ -33,9 +38,7 @@ sudo ./scripts/setup-homelab.sh 1. Log out and log back in for group changes to take effect 2. Edit `.env` file with your configuration -3. Deploy the core infrastructure stack -4. Deploy the infrastructure stack (includes Dockge) -5. Access Dockge to manage remaining stacks +3. Run `deploy-homelab.sh` to deploy core infrastructure and Dockge ### NVIDIA GPU Support @@ -72,3 +75,114 @@ This manual approach avoids driver conflicts that often occur with automated ins - Configures Docker networks automatically - SSH is enabled for remote management - NVIDIA driver installation requires manual intervention for reliability + +--- + +## deploy-homelab.sh + +Automated deployment script that deploys the core infrastructure and Dockge. Run this after editing your `.env` file. + +### What It Does + +1. **Validate Prerequisites** - Checks for Docker, .env file, and proper configuration +2. **Create Directories** - Sets up `/opt/stacks/core` and `/opt/stacks/infrastructure` +3. **Create Docker Networks** - Ensures homelab-network, traefik-network, and media-network exist +4. **Deploy Core Stack** - Deploys DuckDNS, Traefik, Authelia, and Gluetun +5. **Deploy Infrastructure Stack** - Deploys Dockge, Portainer, Pi-hole, and monitoring tools +6. **Wait for Dockge** - Waits for Dockge web UI to become accessible +7. **Open Browser** - Automatically opens Dockge in your default browser + +### Usage + +```bash +# From the AI-Homelab directory +cd AI-Homelab + +# Ensure .env is configured +cp .env.example .env +nano .env # Edit with your values + +# Make the script executable (if needed) +chmod +x scripts/deploy-homelab.sh + +# Run WITHOUT sudo (run as your regular user) +./scripts/deploy-homelab.sh +``` + +### After Running + +The script will automatically open `https://dockge.yourdomain.duckdns.org` in your browser when Dockge is ready. + +1. Log in to Dockge using your Authelia credentials (configured in `/opt/stacks/core/authelia/users_database.yml`) +2. Deploy additional stacks through Dockge's web UI: + - `dashboards.yml` - Homepage and Homarr + - `media.yml` - Plex, Jellyfin, Sonarr, Radarr, etc. + - `media-extended.yml` - Readarr, Lidarr, etc. + - `homeassistant.yml` - Home Assistant and accessories + - `productivity.yml` - Nextcloud, Gitea, wikis + - `monitoring.yml` - Grafana, Prometheus + - `utilities.yml` - Backups, password manager + +### Requirements + +- Docker and Docker Compose installed +- `.env` file configured with your domain and credentials +- User must be in docker group (handled by setup-homelab.sh) + +### Browser Detection + +The script will attempt to open Dockge using: +- `xdg-open` (default on most Linux desktops) +- `gnome-open` (GNOME desktop) +- `firefox` or `google-chrome` (direct browser launch) + +If no browser is detected, it will display the URL for manual access. + +### Manual Deployment Alternative + +If you prefer to deploy manually instead of using the script: + +```bash +# Deploy core stack +mkdir -p /opt/stacks/core +cp docker-compose/core.yml /opt/stacks/core/docker-compose.yml +cp -r config-templates/traefik /opt/stacks/core/ +cp -r config-templates/authelia /opt/stacks/core/ +cp .env /opt/stacks/core/ +cd /opt/stacks/core && docker compose up -d + +# Deploy infrastructure stack +mkdir -p /opt/stacks/infrastructure +cp docker-compose/infrastructure.yml /opt/stacks/infrastructure/docker-compose.yml +cp .env /opt/stacks/infrastructure/ +cd /opt/stacks/infrastructure && docker compose up -d + +# Manually open: https://dockge.yourdomain.duckdns.org +``` + +### Troubleshooting + +**Script says "Docker daemon is not running":** +- Run: `sudo systemctl start docker` +- Or log out and back in if you just added yourself to docker group + +**Script says ".env file not found":** +- Run: `cp .env.example .env` and edit with your values + +**Dockge doesn't open automatically:** +- The script will display the URL to open manually +- Wait a minute for services to fully start +- Check logs: `docker compose -f /opt/stacks/infrastructure/docker-compose.yml logs dockge` + +**Traefik SSL certificate errors:** +- Initial certificate generation can take a few minutes +- Check DuckDNS token is correct in .env +- Verify your domain is accessible from the internet + +### Notes + +- Run as regular user (NOT with sudo) +- Validates .env configuration before deployment +- Waits up to 60 seconds for Dockge to become ready +- Automatically copies .env to stack directories +- Safe to run multiple times (idempotent) diff --git a/scripts/deploy-homelab.sh b/scripts/deploy-homelab.sh new file mode 100755 index 0000000..51c9770 --- /dev/null +++ b/scripts/deploy-homelab.sh @@ -0,0 +1,239 @@ +#!/bin/bash +# AI-Homelab Deployment Script +# This script deploys the core infrastructure and Dockge +# Run after: 1) setup-homelab.sh and 2) editing .env file +# Run as: ./deploy-homelab.sh + +set -e # Exit on error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Log functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check if running as root +if [ "$EUID" -eq 0 ]; then + log_error "Please do NOT run this script as root or with sudo" + log_info "Run as: ./deploy-homelab.sh" + exit 1 +fi + +# Get script directory (AI-Homelab/scripts) +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +REPO_DIR="$( cd "$SCRIPT_DIR/.." && pwd )" + +log_info "AI-Homelab Deployment Script" +echo "" + +# Check if .env file exists +if [ ! -f "$REPO_DIR/.env" ]; then + log_error ".env file not found!" + log_info "Please create and configure your .env file first:" + echo " cd $REPO_DIR" + echo " cp .env.example .env" + echo " nano .env" + exit 1 +fi + +# Check if Docker is installed and running +if ! command -v docker &> /dev/null; then + log_error "Docker is not installed. Please run setup-homelab.sh first." + exit 1 +fi + +if ! docker info &> /dev/null; then + log_error "Docker daemon is not running or you don't have permission." + log_info "Try: sudo systemctl start docker" + log_info "Or log out and log back in for group changes to take effect" + exit 1 +fi + +log_success "Docker is available and running" +echo "" + +# Load environment variables for domain check +source "$REPO_DIR/.env" + +if [ -z "$DOMAIN" ]; then + log_error "DOMAIN is not set in .env file" + log_info "Please edit .env and set your DuckDNS domain" + exit 1 +fi + +log_info "Using domain: $DOMAIN" +echo "" + +# Step 1: Create required directories +log_info "Step 1/5: Creating required directories..." +mkdir -p /opt/stacks/core +mkdir -p /opt/stacks/infrastructure +mkdir -p /opt/dockge/data +log_success "Directories created" +echo "" + +# Step 2: Create Docker networks (if they don't exist) +log_info "Step 2/5: Creating Docker networks..." +docker network create homelab-network 2>/dev/null && log_success "Created homelab-network" || log_info "homelab-network already exists" +docker network create traefik-network 2>/dev/null && log_success "Created traefik-network" || log_info "traefik-network already exists" +docker network create media-network 2>/dev/null && log_success "Created media-network" || log_info "media-network already exists" +echo "" + +# Step 3: Deploy core infrastructure (DuckDNS, Traefik, Authelia, Gluetun) +log_info "Step 3/5: Deploying core infrastructure stack..." +log_info " - DuckDNS (Dynamic DNS)" +log_info " - Traefik (Reverse Proxy with SSL)" +log_info " - Authelia (Single Sign-On)" +log_info " - Gluetun (VPN Client)" +echo "" + +# Copy core stack files +cp "$REPO_DIR/docker-compose/core.yml" /opt/stacks/core/docker-compose.yml +cp -r "$REPO_DIR/config-templates/traefik" /opt/stacks/core/ +cp -r "$REPO_DIR/config-templates/authelia" /opt/stacks/core/ +cp "$REPO_DIR/.env" /opt/stacks/core/.env + +# Deploy core stack +cd /opt/stacks/core +docker compose up -d + +log_success "Core infrastructure deployed" +echo "" + +# Wait for Traefik to be ready +log_info "Waiting for Traefik to initialize..." +sleep 10 + +# Check if Traefik is healthy +if docker ps | grep -q "traefik.*Up"; then + log_success "Traefik is running" +else + log_warning "Traefik container check inconclusive, continuing..." +fi +echo "" + +# Step 4: Deploy infrastructure stack (Dockge and monitoring tools) +log_info "Step 4/5: Deploying infrastructure stack..." +log_info " - Dockge (Docker Compose Manager)" +log_info " - Portainer (Alternative Docker UI)" +log_info " - Pi-hole (DNS Ad Blocker)" +log_info " - Watchtower (Container Updates)" +log_info " - Dozzle (Log Viewer)" +log_info " - Glances (System Monitor)" +log_info " - Docker Proxy (Security)" +echo "" + +# Copy infrastructure stack +cp "$REPO_DIR/docker-compose/infrastructure.yml" /opt/stacks/infrastructure/docker-compose.yml +cp "$REPO_DIR/.env" /opt/stacks/infrastructure/.env + +# Deploy infrastructure stack +cd /opt/stacks/infrastructure +docker compose up -d + +log_success "Infrastructure stack deployed" +echo "" + +# Step 5: Wait for Dockge to be ready and open browser +log_info "Step 5/5: Waiting for Dockge web UI to be ready..." + +DOCKGE_URL="https://dockge.${DOMAIN}" +MAX_WAIT=60 # Maximum wait time in seconds +WAITED=0 + +# Function to check if Dockge is accessible +check_dockge() { + # Try to connect to Dockge (ignore SSL cert warnings for self-signed during startup) + curl -k -s -o /dev/null -w "%{http_code}" "$DOCKGE_URL" 2>/dev/null +} + +# Wait for Dockge to respond +while [ $WAITED -lt $MAX_WAIT ]; do + HTTP_CODE=$(check_dockge) + if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "302" ] || [ "$HTTP_CODE" = "401" ]; then + log_success "Dockge web UI is ready!" + break + fi + echo -n "." + sleep 2 + WAITED=$((WAITED + 2)) +done + +echo "" +echo "" + +if [ $WAITED -ge $MAX_WAIT ]; then + log_warning "Dockge did not respond within ${MAX_WAIT} seconds" + log_info "It may still be starting up. Check manually at: $DOCKGE_URL" +else + # Try to open browser + log_info "Opening Dockge in your browser..." + + # Detect and use available browser + if command -v xdg-open &> /dev/null; then + xdg-open "$DOCKGE_URL" &> /dev/null & + log_success "Browser opened" + elif command -v gnome-open &> /dev/null; then + gnome-open "$DOCKGE_URL" &> /dev/null & + log_success "Browser opened" + elif command -v firefox &> /dev/null; then + firefox "$DOCKGE_URL" &> /dev/null & + log_success "Browser opened" + elif command -v google-chrome &> /dev/null; then + google-chrome "$DOCKGE_URL" &> /dev/null & + log_success "Browser opened" + else + log_warning "No browser detected. Please manually open: $DOCKGE_URL" + fi +fi + +echo "" +echo "==========================================" +log_success "Deployment completed successfully!" +echo "==========================================" +echo "" +log_info "Access your services:" +echo "" +echo " 🚀 Dockge: $DOCKGE_URL" +echo " 🔒 Authelia: https://auth.${DOMAIN}" +echo " 🔀 Traefik: https://traefik.${DOMAIN}" +echo "" +log_info "Next steps:" +echo "" +echo " 1. Log in to Dockge using your Authelia credentials" +echo " (configured in /opt/stacks/core/authelia/users_database.yml)" +echo "" +echo " 2. Deploy additional stacks through Dockge's web UI:" +echo " - dashboards.yml (Homepage, Homarr)" +echo " - media.yml (Plex, Jellyfin, Sonarr, Radarr, etc.)" +echo " - media-extended.yml (Readarr, Lidarr, etc.)" +echo " - homeassistant.yml (Home Assistant and accessories)" +echo " - productivity.yml (Nextcloud, Gitea, wikis)" +echo " - monitoring.yml (Grafana, Prometheus, etc.)" +echo " - utilities.yml (Backups, code editors, etc.)" +echo "" +echo " 3. Configure services via the AI assistant in VS Code" +echo "" +echo "==========================================" +echo "" +log_info "For documentation, see: $REPO_DIR/docs/" +log_info "For troubleshooting, see: $REPO_DIR/docs/quick-reference.md" +echo ""