diff --git a/.gitea/scripts/README.md b/.gitea/scripts/README.md new file mode 100644 index 0000000..5ebddaf --- /dev/null +++ b/.gitea/scripts/README.md @@ -0,0 +1,91 @@ +# Wiki Generators for Homelab + +Automated Wiki page generation scripts for documenting homelab infrastructure. + +## 1. Authentik Applications Wiki Generator + +Generates a Wiki page with all applications managed by Authentik from Terraform configuration. + +### Files: +- `generate-apps-wiki.py` - Generates Applications.md from Terraform output +- `process-terraform-output.py` - Processes Terraform JSON output + +### Workflow: +- **Trigger**: Push to `main` branch with Terraform changes +- **Workflow**: `.gitea/workflows/authentik-apps.yaml` +- **Output**: Applications Wiki page + +## 2. Kubernetes Services Wiki Generator + +Analyzes k8s/ directory and generates comprehensive documentation for all Kubernetes services. + +### Files: +- `generate-k8s-wiki.py` - Main script for analyzing k8s services + +### Features: +- **Service Types**: Detects Helm Charts, Kustomize, and YAML manifests +- **ArgoCD Integration**: Shows auto-sync status and project info +- **Service Discovery**: Lists all services, ingresses, and external secrets +- **Categorization**: Groups by apps, core, games categories +- **Detailed Analysis**: Shows deployments, containers, files + +### Workflow: +- **Trigger**: Changes in `k8s/` directory +- **Workflow**: `.gitea/workflows/k8s-wiki.yaml` +- **Output**: Kubernetes-Services Wiki page + +## GitHub Secrets Configuration + +Required secrets in repository settings: + +``` +GT_URL=https://gt.hexor.cy +GT_WIKI_TOKEN=your_gitea_access_token +GT_OWNER=your_username +GT_REPO=homelab +``` + +## Generated Wiki Pages Structure + +### Applications Page +- Table with icons (32x32), external/internal URLs +- Statistics by type (Proxy vs OAuth2) +- Grouping by categories (Core, Tools, Media, etc.) + +### Kubernetes Services Page +- Overview table with service types and status +- Detailed sections by category +- ArgoCD integration status +- Service discovery information + +## Local Testing + +### Authentik Apps: +```bash +cd terraform/authentik +terraform output -json > terraform-output.json +python3 ../../.gitea/scripts/process-terraform-output.py terraform-output.json processed-output.json +python3 ../../.gitea/scripts/generate-apps-wiki.py processed-output.json +``` + +### K8s Services: +```bash +pip install pyyaml +python3 .gitea/scripts/generate-k8s-wiki.py k8s/ Kubernetes-Services.md +``` + +## Troubleshooting + +### Common Issues: + +1. **Terraform output parsing errors** + - Check for [command] prefix in output + - Verify JSON structure with debug mode + +2. **Wiki upload failures** + - Verify Gitea token permissions + - Check network connectivity to Gitea instance + +3. **YAML parsing errors in k8s analysis** + - Ensure valid YAML syntax in k8s files + - Check PyYAML installation \ No newline at end of file diff --git a/.gitea/scripts/generate-k8s-wiki.py b/.gitea/scripts/generate-k8s-wiki.py new file mode 100644 index 0000000..84dd15e --- /dev/null +++ b/.gitea/scripts/generate-k8s-wiki.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +""" +Script for generating Wiki page with Kubernetes services from k8s/ directory +""" + +import os +import yaml +import json +import sys +from datetime import datetime +from collections import defaultdict + +class K8sService: + def __init__(self, name, category, path): + self.name = name + self.category = category + self.path = path + self.namespace = None + self.deployment_type = "Unknown" + self.helm_charts = [] + self.services = [] + self.ingresses = [] + self.external_secrets = [] + self.deployments = [] + self.pvcs = [] + self.argo_app = None + self.files = [] + + def __repr__(self): + return f"K8sService({self.name}, {self.deployment_type})" + +def parse_yaml_file(filepath): + """Parse YAML file and return content""" + try: + with open(filepath, 'r') as f: + # Load all documents in the file + docs = list(yaml.safe_load_all(f)) + return docs if len(docs) > 1 else docs[0] if docs else None + except Exception as e: + print(f" āš ļø Error parsing {filepath}: {e}") + return None + +def analyze_service_directory(service_path, service_name, category): + """Analyze a service directory and extract information""" + service = K8sService(service_name, category, service_path) + + # List all files + for file in os.listdir(service_path): + if file.endswith('.yaml') or file.endswith('.yml'): + service.files.append(file) + filepath = os.path.join(service_path, file) + + # Parse YAML content + content = parse_yaml_file(filepath) + if not content: + continue + + # Handle multiple documents in one file + documents = content if isinstance(content, list) else [content] + + for doc in documents: + if not isinstance(doc, dict) or 'kind' not in doc: + continue + + kind = doc['kind'] + metadata = doc.get('metadata', {}) + + # ArgoCD Application + if kind == 'Application' and doc.get('apiVersion', '').startswith('argoproj.io'): + service.argo_app = { + 'name': metadata.get('name', ''), + 'namespace': doc.get('spec', {}).get('destination', {}).get('namespace', ''), + 'project': doc.get('spec', {}).get('project', ''), + 'auto_sync': doc.get('spec', {}).get('syncPolicy', {}).get('automated') is not None + } + service.namespace = service.argo_app['namespace'] + + # Kustomization + elif kind == 'Kustomization': + if 'helmCharts' in doc: + service.deployment_type = "Helm Chart" + for chart in doc.get('helmCharts', []): + service.helm_charts.append({ + 'name': chart.get('name', ''), + 'repo': chart.get('repo', ''), + 'version': chart.get('version', ''), + 'namespace': chart.get('namespace', service.namespace) + }) + else: + service.deployment_type = "Kustomize" + + # Deployment + elif kind == 'Deployment': + service.deployments.append({ + 'name': metadata.get('name', ''), + 'namespace': metadata.get('namespace', service.namespace), + 'replicas': doc.get('spec', {}).get('replicas', 1), + 'containers': [c.get('name', '') for c in doc.get('spec', {}).get('template', {}).get('spec', {}).get('containers', [])] + }) + if service.deployment_type == "Unknown": + service.deployment_type = "YAML Manifests" + + # Service + elif kind == 'Service': + svc_spec = doc.get('spec', {}) + service.services.append({ + 'name': metadata.get('name', ''), + 'type': svc_spec.get('type', 'ClusterIP'), + 'ports': svc_spec.get('ports', []) + }) + + # Ingress + elif kind == 'Ingress': + rules = doc.get('spec', {}).get('rules', []) + hosts = [] + for rule in rules: + if 'host' in rule: + hosts.append(rule['host']) + service.ingresses.append({ + 'name': metadata.get('name', ''), + 'hosts': hosts + }) + + # ExternalSecret + elif kind == 'ExternalSecret': + service.external_secrets.append({ + 'name': metadata.get('name', ''), + 'store': doc.get('spec', {}).get('secretStoreRef', {}).get('name', '') + }) + + # PersistentVolumeClaim + elif kind == 'PersistentVolumeClaim': + service.pvcs.append({ + 'name': metadata.get('name', ''), + 'size': doc.get('spec', {}).get('resources', {}).get('requests', {}).get('storage', '') + }) + + # If no specific deployment type found but has YAML files + if service.deployment_type == "Unknown" and service.files: + service.deployment_type = "YAML Manifests" + + return service + +def generate_markdown_table(services): + """Generate markdown table for services""" + markdown = [] + markdown.append("# Kubernetes Services") + markdown.append("") + markdown.append(f"*Automatically generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S UTC')}*") + markdown.append("") + + # Group by category + categories = defaultdict(list) + for service in services: + categories[service.category].append(service) + + # Statistics + markdown.append("## Statistics") + markdown.append("") + markdown.append(f"- **Total Services**: {len(services)}") + markdown.append(f"- **Categories**: {len(categories)}") + helm_count = sum(1 for s in services if s.deployment_type == "Helm Chart") + kustomize_count = sum(1 for s in services if s.deployment_type == "Kustomize") + yaml_count = sum(1 for s in services if s.deployment_type == "YAML Manifests") + markdown.append(f"- **Helm Charts**: {helm_count}") + markdown.append(f"- **Kustomize**: {kustomize_count}") + markdown.append(f"- **YAML Manifests**: {yaml_count}") + markdown.append("") + + # Main table + markdown.append("## All Services") + markdown.append("") + markdown.append("| Service | Category | Type | Namespace | Ingresses | Services | Secrets | Auto-Sync |") + markdown.append("|---------|----------|------|-----------|-----------|----------|---------|-----------|") + + for category in sorted(categories.keys()): + for service in sorted(categories[category], key=lambda x: x.name): + # Service name with link to directory + name_link = f"[{service.name}](k8s/{service.category}/{service.name}/)" + + # Deployment type with emoji + type_emoji = { + "Helm Chart": "šŸŽ©", + "Kustomize": "šŸ”§", + "YAML Manifests": "šŸ“„", + "Unknown": "ā“" + } + type_str = f"{type_emoji.get(service.deployment_type, '')} {service.deployment_type}" + + # Ingresses + ingresses = [] + for ing in service.ingresses: + for host in ing['hosts']: + ingresses.append(f"[{host}](https://{host})") + ingress_str = "
".join(ingresses) if ingresses else "-" + + # Services + svc_list = [] + for svc in service.services: + ports = [f"{p.get('port', '?')}" for p in svc['ports']] + svc_list.append(f"`{svc['name']}:{','.join(ports)}`") + svc_str = "
".join(svc_list) if svc_list else "-" + + # External Secrets + secrets_str = f"{len(service.external_secrets)} secrets" if service.external_secrets else "-" + + # Auto-sync + auto_sync = "āœ…" if service.argo_app and service.argo_app.get('auto_sync') else "āŒ" + + markdown.append(f"| **{name_link}** | {category} | {type_str} | {service.namespace or '-'} | {ingress_str} | {svc_str} | {secrets_str} | {auto_sync} |") + + markdown.append("") + + # Detailed sections by category + for category in sorted(categories.keys()): + markdown.append(f"## {category.title()} Services") + markdown.append("") + + for service in sorted(categories[category], key=lambda x: x.name): + markdown.append(f"### {service.name}") + markdown.append("") + + # Basic info + markdown.append(f"- **Type**: {service.deployment_type}") + markdown.append(f"- **Namespace**: {service.namespace or 'Not specified'}") + markdown.append(f"- **Path**: `{service.path}`") + + # Helm charts + if service.helm_charts: + markdown.append("- **Helm Charts**:") + for chart in service.helm_charts: + markdown.append(f" - {chart['name']} v{chart['version']} from {chart['repo']}") + + # Deployments + if service.deployments: + markdown.append("- **Deployments**:") + for dep in service.deployments: + containers = ', '.join(dep['containers']) + markdown.append(f" - {dep['name']} ({dep['replicas']} replicas) - Containers: {containers}") + + # Files + if service.files: + markdown.append(f"- **Files**: {', '.join(sorted(service.files))}") + + markdown.append("") + + markdown.append("---") + markdown.append("*This page is automatically generated from k8s/ directory via CI/CD*") + + return "\n".join(markdown) + +def main(): + if len(sys.argv) < 2: + print("Usage: generate-k8s-wiki.py [output-file]") + sys.exit(1) + + k8s_dir = sys.argv[1] + output_file = sys.argv[2] if len(sys.argv) > 2 else "Kubernetes-Services.md" + + if not os.path.exists(k8s_dir): + print(f"āŒ Directory {k8s_dir} not found") + sys.exit(1) + + print(f"šŸ“‚ Scanning {k8s_dir}...") + + services = [] + + # Scan each category directory + for category in ['apps', 'core', 'games']: + category_path = os.path.join(k8s_dir, category) + if not os.path.exists(category_path): + print(f" āš ļø Category {category} not found") + continue + + print(f"šŸ“ Processing {category}/...") + + # Scan each service in category + for service_name in os.listdir(category_path): + service_path = os.path.join(category_path, service_name) + if os.path.isdir(service_path): + print(f" šŸ” Analyzing {service_name}...") + service = analyze_service_directory(service_path, service_name, category) + services.append(service) + + print(f"\nāœ… Found {len(services)} services") + + # Generate markdown + markdown = generate_markdown_table(services) + + # Write output + with open(output_file, 'w', encoding='utf-8') as f: + f.write(markdown) + + print(f"šŸ“„ Wiki page generated: {output_file}") + print(f"šŸ“Š Total services: {len(services)}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.gitea/workflows/k8s-wiki.yaml b/.gitea/workflows/k8s-wiki.yaml new file mode 100644 index 0000000..22d44c9 --- /dev/null +++ b/.gitea/workflows/k8s-wiki.yaml @@ -0,0 +1,111 @@ +name: 'Update Kubernetes Services Wiki' + +on: + push: + branches: [ "main" ] +# paths: +# - 'k8s/**' +# - '.gitea/scripts/generate-k8s-wiki.py' + workflow_dispatch: + +permissions: + contents: read + +jobs: + update-k8s-wiki: + name: 'Generate and Update K8s Wiki' + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install Python dependencies + run: | + pip install pyyaml + + - name: Generate K8s Services Wiki + run: | + echo "šŸ“‹ Starting K8s wiki generation..." + python3 .gitea/scripts/generate-k8s-wiki.py k8s/ Kubernetes-Services.md + + if [ -f "Kubernetes-Services.md" ]; then + echo "āœ… Wiki content generated successfully" + echo "šŸ“„ File size: $(wc -c < Kubernetes-Services.md) bytes" + echo "šŸ“„ Lines: $(wc -l < Kubernetes-Services.md)" + else + echo "āŒ Wiki content not generated" + exit 1 + fi + + - name: Upload Wiki to Gitea + continue-on-error: true + run: | + # Set variables + GITEA_URL="${{ secrets.GT_URL }}" + GITEA_TOKEN="${{ secrets.GT_WIKI_TOKEN }}" + GITEA_OWNER="${{ secrets.GT_OWNER }}" + GITEA_REPO="${{ secrets.GT_REPO }}" + + # Debug variables (without exposing token) + echo "šŸ” Checking variables..." + echo "GITEA_URL: ${GITEA_URL:-NOT SET}" + echo "GITEA_OWNER: ${GITEA_OWNER:-NOT SET}" + echo "GITEA_REPO: ${GITEA_REPO:-NOT SET}" + echo "GITEA_TOKEN: $(if [ -n "$GITEA_TOKEN" ]; then echo "SET"; else echo "NOT SET"; fi)" + + if [ ! -f "Kubernetes-Services.md" ]; then + echo "āŒ Kubernetes-Services.md not found" + exit 1 + fi + + echo "šŸ“¤ Uploading to Gitea Wiki..." + + # Encode content to base64 + CONTENT=$(base64 -w 0 Kubernetes-Services.md) + + # Check if wiki page exists + WIKI_PAGE_EXISTS=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: token $GITEA_TOKEN" \ + "$GITEA_URL/api/v1/repos/$GITEA_OWNER/$GITEA_REPO/wiki/page/Kubernetes-Services" || echo "000") + + if [ "$WIKI_PAGE_EXISTS" = "200" ]; then + echo "šŸ“ Updating existing wiki page..." + curl -X PATCH \ + -H "Authorization: token $GITEA_TOKEN" \ + -H "Content-Type: application/json" \ + -d "{ + \"title\": \"Kubernetes-Services\", + \"content_base64\": \"$CONTENT\", + \"message\": \"Update K8s services list from CI/CD [$(date)]\" + }" \ + "$GITEA_URL/api/v1/repos/$GITEA_OWNER/$GITEA_REPO/wiki/page/Kubernetes-Services" || echo "āš ļø Wiki update failed" + else + echo "šŸ“„ Creating new wiki page..." + curl -X POST \ + -H "Authorization: token $GITEA_TOKEN" \ + -H "Content-Type: application/json" \ + -d "{ + \"title\": \"Kubernetes-Services\", + \"content_base64\": \"$CONTENT\", + \"message\": \"Create K8s services list from CI/CD [$(date)]\" + }" \ + "$GITEA_URL/api/v1/repos/$GITEA_OWNER/$GITEA_REPO/wiki/new" || echo "āš ļø Wiki creation failed" + fi + + echo "āœ… Wiki update process completed" + echo "šŸ”— Wiki URL: $GITEA_URL/$GITEA_OWNER/$GITEA_REPO/wiki/Kubernetes-Services" + + - name: Summary + if: always() + run: | + echo "## šŸ“Š K8s Wiki Update Summary" >> $GITHUB_STEP_SUMMARY + if [ -f "Kubernetes-Services.md" ]; then + echo "- āœ… K8s services analyzed" >> $GITHUB_STEP_SUMMARY + echo "- āœ… Wiki page generated" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Services found:** $(grep -c '^|' Kubernetes-Services.md || echo 0)" >> $GITHUB_STEP_SUMMARY + else + echo "- āŒ Wiki generation failed" >> $GITHUB_STEP_SUMMARY + fi + echo "**Generated at:** $(date)" >> $GITHUB_STEP_SUMMARY