# Version control
.git
.gitignore

# Python
__pycache__
*.pyc
*.pyo
*.pyd
.Python
venv/
.env

# Node
node_modules/
npm-debug.log
yarn-debug.log
yarn-error.log

# Build
dist/
build/
*.egg-info/

# IDE
.idea/
.vscode/
*.swp
*.swo

# OS
.DS_Store
Thumbs.db

# EditorConfig is awesome: https://EditorConfig.org

# top-most EditorConfig file
root = true

# Unix-style newlines with a newline ending every file
[*]
end_of_line = lf
insert_final_newline = true
charset = utf-8
trim_trailing_whitespace = true

# Python files
[*.py]
indent_style = space
indent_size = 4
max_line_length = 100

# JavaScript/TypeScript files
[*.{js,jsx,ts,tsx}]
indent_style = space
indent_size = 2
max_line_length = 100

# YAML files
[*.{yml,yaml}]
indent_style = space
indent_size = 2

# Markdown files
[*.md]
trim_trailing_whitespace = false
max_line_length = off

# JSON files
[*.json]
indent_style = space
indent_size = 2

# Shell scripts
[*.sh]
indent_style = space
indent_size = 2

SUPABASE_URL=your_supabase_project_url
SUPABASE_KEY=your_supabase_anon_key

# Supabase Configuration
SUPABASE_URL=your-project-url-here
SUPABASE_KEY=your-anon-key-here

# API Configuration
API_URL=http://localhost:5000/api/gpu-stats
COLLECTION_INTERVAL=0.25  # in seconds

version: 2
updates:
  # Frontend dependencies
  - package-ecosystem: "npm"
    directory: "/frontend"
    schedule:
      interval: "weekly"
    open-pull-requests-limit: 10
    labels:
      - "npm"
      - "dependencies"
    commit-message:
      prefix: "chore"
      prefix-development: "chore"
      include: "scope"

  # Backend dependencies
  - package-ecosystem: "pip"
    directory: "/backend"
    schedule:
      interval: "weekly"
    open-pull-requests-limit: 10
    labels:
      - "pip"
      - "dependencies"
    commit-message:
      prefix: "chore"
      prefix-development: "chore"
      include: "scope"

  # GitHub Actions
  - package-ecosystem: "github-actions"
    directory: "/"
    schedule:
      interval: "weekly"
    labels:
      - "github-actions"
      - "dependencies"
    commit-message:
      prefix: "chore"
      prefix-development: "chore"
      include: "scope"

name: CI

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  backend-tests:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10", "3.11"]

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r backend/requirements.txt
        pip install pytest pytest-cov pylint black

    - name: Check formatting
      run: |
        black --check backend/

    - name: Lint with pylint
      run: |
        pylint backend/src/

    - name: Run tests with coverage
      run: |
        pytest backend/tests/ --cov=backend/src/ --cov-report=xml

  frontend-tests:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        node-version: [18.x, 20.x]

    steps:
    - uses: actions/checkout@v4
    - name: Use Node.js ${{ matrix.node-version }}
      uses: actions/setup-node@v3
      with:
        node-version: ${{ matrix.node-version }}
        
    - name: Install dependencies
      run: |
        cd frontend
        npm ci

    - name: Check formatting
      run: |
        cd frontend
        npm run format:check

    - name: Lint
      run: |
        cd frontend
        npm run lint

    - name: Type check
      run: |
        cd frontend
        npm run typecheck

    - name: Build
      run: |
        cd frontend
        npm run build

name: "CodeQL"

on:
  push:
    branches: [ "main" ]
  pull_request:
    branches: [ "main" ]
  schedule:
    - cron: '30 1 * * 0'  # Run at 1:30 UTC every Sunday

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python', 'javascript', 'typescript' ]

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Initialize CodeQL
      uses: github/codeql-action/init@v2
      with:
        languages: ${{ matrix.language }}

    - name: Autobuild
      uses: github/codeql-action/autobuild@v2

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v2
      with:
        category: "/language:${{matrix.language}}"

name: Snyk Security Scan

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]
  schedule:
    - cron: '0 2 * * 0'  # Run at 2:00 UTC every Sunday

jobs:
  security:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'

      - name: Set up Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '20'

      - name: Install dependencies
        run: |
          python -m pip install -r backend/requirements.txt
          cd frontend && npm install

      - name: Run Snyk to check for vulnerabilities
        uses: snyk/actions/python@master
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        with:
          command: monitor

      - name: Run Snyk on frontend
        uses: snyk/actions/node@master
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        with:
          command: monitor
          args: --all-projects

name: SonarCloud Analysis

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  sonarcloud:
    name: SonarCloud
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'

      - name: Set up Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '20'

      - name: Install dependencies
        run: |
          python -m pip install -r backend/requirements.txt
          python -m pip install coverage pytest
          cd frontend && npm install

      - name: Run backend tests with coverage
        run: |
          cd backend
          coverage run -m pytest
          coverage xml -o coverage-reports/coverage.xml

      - name: Run frontend tests with coverage
        run: |
          cd frontend
          npm run test:coverage

      - name: SonarCloud Scan
        uses: SonarSource/sonarcloud-github-action@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
        with:
          args: >
            -Dsonar.organization=jackccrawford
            -Dsonar.projectKey=jackccrawford_gpu-sentinel-pro
            -Dsonar.python.coverage.reportPaths=backend/coverage-reports/coverage.xml
            -Dsonar.javascript.lcov.reportPaths=frontend/coverage/lcov.info
            -Dsonar.sources=backend/src,frontend/src
            -Dsonar.tests=backend/tests,frontend/src/**/*.test.tsx

# Dependencies
node_modules/
**/node_modules/**
venv/
__pycache__/

# Environment files
.env
*.env

# Logs
*.log
frontend/frontend.log
backend/src/service/gpu_service.log

# Service PIDs
*.pid
frontend/frontend.pid
backend/src/service/service.pid

# Build files
dist/
build/
*.pyc

# IDE files
.vscode/
.idea/
*.swp
*.swo

# OS files
.DS_Store
Thumbs.db

# Supabase local dev
supabase/volumes/

# TypeScript
*.tsbuildinfo
frontend/.vite/
.vite/
dist-ssr/

# Package files
package-lock.json
**/package-lock.json

FROM python:3.10-slim

WORKDIR /app

# Copy requirements first for better caching
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

# Copy application code
COPY backend/ backend/
COPY src/ src/

# Set environment variables
ENV PYTHONPATH=/app

# Run both collector and API server
CMD ["sh", "-c", "python -m src.collector.collector & python -m backend.src.service.app"]

MIT License

Copyright (c) 2025 Jack Crawford

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

-- Create an extension for generating UUIDs if not exists
create extension if not exists "uuid-ossp";

-- Create the gpu_metrics table
create table if not exists gpu_metrics (
    id uuid primary key default uuid_generate_v4(),
    timestamp timestamptz not null default now(),
    
    -- GPU Burn Metrics
    duration integer not null,
    errors integer not null,
    running boolean not null,
    
    -- Nvidia Info
    cuda_version text not null,
    driver_version text not null,
    
    -- GPU Metrics Array (stored as JSONB)
    gpus jsonb not null,
    
    -- Additional fields
    processes jsonb default '[]'::jsonb,
    success boolean not null,
    
    -- Indexes for common queries
    created_at timestamptz not null default now()
);

-- Create indexes for better query performance
create index if not exists idx_gpu_metrics_timestamp on gpu_metrics(timestamp);
create index if not exists idx_gpu_metrics_created_at on gpu_metrics(created_at);

-- Add a comment to the table
comment on table gpu_metrics is 'Stores GPU metrics data collected from NVIDIA GPUs';

-- Create alerts table
CREATE TABLE IF NOT EXISTS alert_thresholds (
    id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
    metric_name TEXT NOT NULL,
    warning_threshold FLOAT NOT NULL,
    critical_threshold FLOAT NOT NULL,
    duration_seconds INTEGER NOT NULL DEFAULT 60,
    enabled BOOLEAN NOT NULL DEFAULT true,
    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);

-- Create alerts history table
CREATE TABLE IF NOT EXISTS alert_history (
    id uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
    alert_threshold_id uuid REFERENCES alert_thresholds(id),
    gpu_index INTEGER NOT NULL,
    metric_value FLOAT NOT NULL,
    threshold_value FLOAT NOT NULL,
    severity TEXT NOT NULL,  -- 'warning' or 'critical'
    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);

-- Create index for querying recent alerts
CREATE INDEX IF NOT EXISTS idx_alert_history_created_at 
ON alert_history(created_at);

-- Create function to cleanup old alert history
CREATE OR REPLACE FUNCTION cleanup_old_alerts(days_to_keep INTEGER)
RETURNS void AS $$
BEGIN
    DELETE FROM alert_history 
    WHERE created_at < NOW() - (days_to_keep || ' days')::INTERVAL;
END;
$$ LANGUAGE plpgsql;

-- Create function to update alert thresholds from config
CREATE OR REPLACE FUNCTION update_alert_thresholds_from_config(
    config jsonb
) RETURNS void AS $$
BEGIN
    -- Temperature alerts
    INSERT INTO alert_thresholds (
        metric_name, 
        warning_threshold, 
        critical_threshold, 
        duration_seconds
    ) VALUES (
        'temperature',
        (config->'alerts'->>'temperature_warning')::float,
        (config->'alerts'->>'temperature_critical')::float,
        (config->'alerts'->>'temperature_duration')::integer
    ) ON CONFLICT (metric_name) DO UPDATE SET
        warning_threshold = EXCLUDED.warning_threshold,
        critical_threshold = EXCLUDED.critical_threshold,
        duration_seconds = EXCLUDED.duration_seconds,
        updated_at = NOW();

    -- GPU utilization alerts
    INSERT INTO alert_thresholds (
        metric_name, 
        warning_threshold, 
        critical_threshold, 
        duration_seconds
    ) VALUES (
        'gpu_utilization',
        (config->'alerts'->>'gpu_utilization_warning')::float,
        (config->'alerts'->>'gpu_utilization_critical')::float,
        (config->'alerts'->>'gpu_utilization_duration')::integer
    ) ON CONFLICT (metric_name) DO UPDATE SET
        warning_threshold = EXCLUDED.warning_threshold,
        critical_threshold = EXCLUDED.critical_threshold,
        duration_seconds = EXCLUDED.duration_seconds,
        updated_at = NOW();

    -- Memory usage alerts
    INSERT INTO alert_thresholds (
        metric_name, 
        warning_threshold, 
        critical_threshold, 
        duration_seconds
    ) VALUES (
        'memory_usage',
        (config->'alerts'->>'memory_usage_warning')::float,
        (config->'alerts'->>'memory_usage_critical')::float,
        (config->'alerts'->>'memory_usage_duration')::integer
    ) ON CONFLICT (metric_name) DO UPDATE SET
        warning_threshold = EXCLUDED.warning_threshold,
        critical_threshold = EXCLUDED.critical_threshold,
        duration_seconds = EXCLUDED.duration_seconds,
        updated_at = NOW();
END;
$$ LANGUAGE plpgsql;

-- Add unique constraint for metric_name
ALTER TABLE alert_thresholds 
ADD CONSTRAINT unique_metric_name UNIQUE (metric_name);

COMMENT ON TABLE alert_thresholds IS 'Stores configurable alert thresholds for GPU metrics';
COMMENT ON TABLE alert_history IS 'Stores history of triggered alerts';

fastapi==0.109.2
uvicorn==0.27.1
psycopg2-binary==2.9.9
pyyaml==6.0.1
python-dotenv==1.0.1
pynvml==11.5.0
pandas==2.2.0
numpy==1.26.3
sqlalchemy==1.4.51
alembic==1.13.1
python-jose==3.3.0
fastapi-utils==0.2.1
prometheus-client==0.19.0
aiohttp==3.9.3
asyncpg==0.29.0
psutil==5.9.8

import psycopg2
from psycopg2.extras import Json
from datetime import datetime
from ..models.gpu_metrics import GpuMetricsRecord

class DatabaseClient:
    def __init__(self):
        self.conn_params = {
            'dbname': 'postgres',
            'user': 'postgres',
            'password': 'postgres',
            'host': 'localhost',
            'port': 54432
        }

    def get_connection(self):
        return psycopg2.connect(**self.conn_params)

    def insert_gpu_metrics(self, metrics: GpuMetricsRecord) -> dict:
        """
        Insert GPU metrics into PostgreSQL
        Returns the inserted record
        """
        if not metrics.timestamp:
            metrics.timestamp = datetime.utcnow().isoformat()

        data = metrics.model_dump()
        
        with self.get_connection() as conn:
            with conn.cursor() as cur:
                cur.execute("""
                    INSERT INTO gpu_metrics (
                        timestamp,
                        duration,
                        errors,
                        running,
                        cuda_version,
                        driver_version,
                        gpus,
                        processes,
                        success,
                        created_at
                    ) VALUES (
                        %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW()
                    ) RETURNING id
                """, (
                    data['timestamp'],
                    data['gpu_burn_metrics']['duration'],
                    data['gpu_burn_metrics']['errors'],
                    data['gpu_burn_metrics']['running'],
                    data['nvidia_info']['cuda_version'],
                    data['nvidia_info']['driver_version'],
                    Json(data['gpus']),
                    Json(data['processes']),
                    data['success']
                ))
                record_id = cur.fetchone()[0]
                return {"id": record_id}

    def get_metrics_in_timerange(self, start_time: str, end_time: str):
        """
        Retrieve metrics within a specific time range
        """
        with self.get_connection() as conn:
            with conn.cursor() as cur:
                cur.execute("""
                    SELECT *
                    FROM gpu_metrics
                    WHERE timestamp >= %s AND timestamp <= %s
                    ORDER BY timestamp DESC
                """, (start_time, end_time))
                
                columns = [desc[0] for desc in cur.description]
                results = []
                
                for row in cur.fetchall():
                    result = dict(zip(columns, row))
                    results.append(result)
                
                return results

# Create a singleton instance
db = DatabaseClient()

import os
from dotenv import load_dotenv

load_dotenv()

SUPABASE_URL = os.getenv('SUPABASE_URL')
SUPABASE_KEY = os.getenv('SUPABASE_KEY')

if not SUPABASE_URL or not SUPABASE_KEY:
    raise ValueError("Missing required Supabase credentials in environment variables")

import os
from dotenv import load_dotenv
from supabase import create_client, Client
from datetime import datetime

# Load environment variables
load_dotenv()

# Get Supabase credentials
SUPABASE_URL = os.getenv('SUPABASE_URL')
SUPABASE_KEY = os.getenv('SUPABASE_KEY')

def test_supabase_connection():
    try:
        if not SUPABASE_URL or not SUPABASE_KEY:
            print("Error: Missing Supabase credentials in .env file")
            return False

        # Initialize Supabase client
        supabase: Client = create_client(SUPABASE_URL, SUPABASE_KEY)
        
        # Try a simple query to test connection
        test_data = {
            "timestamp": datetime.utcnow().isoformat(),
            "test": True,
            "message": "Connection test"
        }
        
        print("Attempting to connect to Supabase...")
        result = supabase.table('gpu_metrics').insert(test_data).execute()
        
        print("Successfully connected to Supabase!")
        print("\nTest record inserted:")
        print(result)
        return True

    except Exception as e:
        print(f"Error connecting to Supabase: {e}")
        return False

if __name__ == "__main__":
    test_supabase_connection()

from typing import List, Optional
from pydantic import BaseModel


class GpuBurnMetrics(BaseModel):
    duration: int
    errors: int
    running: bool


class NvidiaInfo(BaseModel):
    cuda_version: str
    driver_version: str


class GpuMetrics(BaseModel):
    compute_mode: str
    fan_speed: int
    gpu_utilization: int
    index: int
    memory_total: int
    memory_used: int
    name: str
    peak_temperature: int
    power_draw: float
    power_limit: int
    temp_change_rate: int
    temperature: int


class GpuMetricsRecord(BaseModel):
    gpu_burn_metrics: GpuBurnMetrics
    gpus: List[GpuMetrics]
    nvidia_info: NvidiaInfo
    processes: List[dict] = []
    success: bool
    timestamp: Optional[str] = None  # We'll add this for tracking when the record was created

from datetime import datetime, timedelta
import logging
from .config import config
from src.database.client import db

logger = logging.getLogger(__name__)

class AlertManager:
    def __init__(self):
        self.alert_cache = {}  # {metric_name_gpu_index: last_alert_time}
        self.load_config()

    def load_config(self):
        """Load alert configuration"""
        self.thresholds = config.get('alerts')
        logger.info("Alert thresholds loaded from config")

    def check_metrics(self, gpu_metrics):
        """Check GPU metrics against thresholds"""
        alerts = []
        current_time = datetime.utcnow()

        for gpu in gpu_metrics.gpus:
            # Temperature checks
            alerts.extend(self._check_metric(
                metric_name='temperature',
                metric_value=gpu.temperature,
                gpu_index=gpu.index,
                warning=self.thresholds['temperature']['warning'],
                critical=self.thresholds['temperature']['critical'],
                duration=self.thresholds['temperature']['duration'],
                current_time=current_time
            ))

            # GPU utilization checks
            alerts.extend(self._check_metric(
                metric_name='gpu_utilization',
                metric_value=gpu.gpu_utilization,
                gpu_index=gpu.index,
                warning=self.thresholds['gpu_utilization']['warning'],
                critical=self.thresholds['gpu_utilization']['critical'],
                duration=self.thresholds['gpu_utilization']['duration'],
                current_time=current_time
            ))

            # Memory usage checks
            memory_usage_percent = (gpu.memory_used / gpu.memory_total) * 100
            alerts.extend(self._check_metric(
                metric_name='memory_usage',
                metric_value=memory_usage_percent,
                gpu_index=gpu.index,
                warning=self.thresholds['memory_usage']['warning'],
                critical=self.thresholds['memory_usage']['critical'],
                duration=self.thresholds['memory_usage']['duration'],
                current_time=current_time
            ))

            # Power usage checks
            power_usage_percent = (gpu.power_draw / gpu.power_limit) * 100
            alerts.extend(self._check_metric(
                metric_name='power_draw',
                metric_value=power_usage_percent,
                gpu_index=gpu.index,
                warning=self.thresholds['power_draw']['warning'],
                critical=self.thresholds['power_draw']['critical'],
                duration=self.thresholds['power_draw']['duration'],
                current_time=current_time
            ))

        if alerts:
            self._store_alerts(alerts)
            logger.warning(f"Generated {len(alerts)} alerts")

        return alerts

    def _check_metric(self, metric_name, metric_value, gpu_index, warning, critical, 
                     duration, current_time):
        """Check a single metric against its thresholds"""
        alerts = []
        cache_key = f"{metric_name}_gpu{gpu_index}"
        
        # Check if we should alert based on duration
        last_alert = self.alert_cache.get(cache_key)
        should_alert = (
            last_alert is None or 
            (current_time - last_alert).total_seconds() >= duration
        )

        if not should_alert:
            return alerts

        if metric_value >= critical:
            alerts.append({
                'metric_name': metric_name,
                'gpu_index': gpu_index,
                'value': metric_value,
                'threshold': critical,
                'severity': 'critical',
                'timestamp': current_time
            })
            self.alert_cache[cache_key] = current_time
        elif metric_value >= warning:
            alerts.append({
                'metric_name': metric_name,
                'gpu_index': gpu_index,
                'value': metric_value,
                'threshold': warning,
                'severity': 'warning',
                'timestamp': current_time
            })
            self.alert_cache[cache_key] = current_time

        return alerts

    def _store_alerts(self, alerts):
        """Store alerts in the database"""
        try:
            with db.get_connection() as conn:
                with conn.cursor() as cur:
                    for alert in alerts:
                        cur.execute("""
                            INSERT INTO alert_history (
                                gpu_index, metric_value, threshold_value, 
                                severity, created_at
                            ) VALUES (%s, %s, %s, %s, %s)
                        """, (
                            alert['gpu_index'],
                            alert['value'],
                            alert['threshold'],
                            alert['severity'],
                            alert['timestamp']
                        ))
        except Exception as e:
            logger.error(f"Failed to store alerts: {e}")

    def cleanup_old_alerts(self):
        """Clean up old alerts based on retention config"""
        try:
            retention_days = config.get('retention', 'days_to_keep')
            with db.get_connection() as conn:
                with conn.cursor() as cur:
                    cur.execute("SELECT cleanup_old_alerts(%s)", (retention_days,))
            logger.info(f"Cleaned up alerts older than {retention_days} days")
        except Exception as e:
            logger.error(f"Failed to cleanup old alerts: {e}")

    def get_recent_alerts(self, hours=24):
        """Get recent alerts"""
        try:
            with db.get_connection() as conn:
                with conn.cursor() as cur:
                    cur.execute("""
                        SELECT * FROM alert_history
                        WHERE created_at > NOW() - interval '%s hours'
                        ORDER BY created_at DESC
                    """, (hours,))
                    columns = [desc[0] for desc in cur.description]
                    return [dict(zip(columns, row)) for row in cur.fetchall()]
        except Exception as e:
            logger.error(f"Failed to get recent alerts: {e}")
            return []

# Create singleton instance
alert_manager = AlertManager()

from datetime import datetime, timedelta
import logging
from typing import List, Dict, Any
from src.service.settings import settings
from src.database.client import db
from src.models.gpu_metrics import GpuMetricsRecord

logger = logging.getLogger(__name__)

class AlertLevel:
    CRITICAL = "critical"
    WARNING = "warning"
    CAUTION = "caution"
    GOOD = "good"
    IDEAL = "ideal"

class AlertSystem:
    def __init__(self):
        # Cache structure: {f"{gpu_index}:{metric}:{severity}": timestamp}
        self.alert_cache = {}
        # Minimum time between similar alerts (5 minutes)
        self.alert_cooldown = timedelta(minutes=5)

    def should_trigger_alert(self, gpu_index: int, metric: str, 
                           severity: str, value: float) -> bool:
        """Determine if an alert should be triggered based on cache and cooldown"""
        cache_key = f"{gpu_index}:{metric}:{severity}"
        current_time = datetime.utcnow()

        # If no previous alert, always trigger
        if cache_key not in self.alert_cache:
            self.alert_cache[cache_key] = current_time
            return True

        # Check if enough time has passed since last alert
        last_alert_time = self.alert_cache[cache_key]
        if current_time - last_alert_time >= self.alert_cooldown:
            self.alert_cache[cache_key] = current_time
            return True

        return False

    def get_metric_level(self, metric: str, value: float) -> str:
        """Determine alert level for any metric based on thresholds"""
        thresholds = settings.get('alerts', metric)
        if value >= thresholds['critical']:
            return AlertLevel.CRITICAL
        elif value >= thresholds['warning']:
            return AlertLevel.WARNING
        elif value >= thresholds['caution']:
            return AlertLevel.CAUTION
        elif value >= thresholds['good']:
            return AlertLevel.GOOD
        return AlertLevel.IDEAL

    def check_metrics(self, metrics: GpuMetricsRecord) -> List[Dict[str, Any]]:
        """Check GPU metrics against all threshold levels"""
        alerts = []
        current_time = datetime.utcnow()

        for gpu in metrics.gpus:
            # Temperature check
            temp_level = self.get_metric_level('temperature', gpu.temperature)
            if temp_level in [AlertLevel.CRITICAL, AlertLevel.WARNING]:
                if self.should_trigger_alert(gpu.index, 'temperature', temp_level, gpu.temperature):
                    alerts.append(self._create_alert(
                        'temperature', gpu.index, gpu.temperature,
                        settings.get('alerts', 'temperature', temp_level),
                        temp_level, current_time
                    ))

            # GPU utilization check
            util_level = self.get_metric_level('gpu_utilization', gpu.gpu_utilization)
            if util_level in [AlertLevel.CRITICAL, AlertLevel.WARNING]:
                if self.should_trigger_alert(gpu.index, 'gpu_utilization', util_level, gpu.gpu_utilization):
                    alerts.append(self._create_alert(
                        'gpu_utilization', gpu.index, gpu.gpu_utilization,
                        settings.get('alerts', 'gpu_utilization', util_level),
                        util_level, current_time
                    ))

            # Fan speed check
            fan_level = self.get_metric_level('fan_speed', gpu.fan_speed)
            if fan_level in [AlertLevel.CRITICAL, AlertLevel.WARNING]:
                if self.should_trigger_alert(gpu.index, 'fan_speed', fan_level, gpu.fan_speed):
                    alerts.append(self._create_alert(
                        'fan_speed', gpu.index, gpu.fan_speed,
                        settings.get('alerts', 'fan_speed', fan_level),
                        fan_level, current_time
                    ))

            # Memory usage check
            memory_percent = (gpu.memory_used / gpu.memory_total) * 100
            mem_level = self.get_metric_level('memory_usage', memory_percent)
            if mem_level in [AlertLevel.CRITICAL, AlertLevel.WARNING]:
                if self.should_trigger_alert(gpu.index, 'memory_usage', mem_level, memory_percent):
                    alerts.append(self._create_alert(
                        'memory_usage', gpu.index, memory_percent,
                        settings.get('alerts', 'memory_usage', mem_level),
                        mem_level, current_time
                    ))

        if alerts:
            self._store_alerts(alerts)
            logger.warning(f"Generated {len(alerts)} alerts")

        return alerts

    def _create_alert(self, metric: str, gpu_index: int, value: float, 
                     threshold: float, severity: str, timestamp: datetime) -> Dict[str, Any]:
        """Create alert dictionary"""
        return {
            'metric': metric,
            'gpu_index': gpu_index,
            'value': value,
            'threshold': threshold,
            'severity': severity,
            'timestamp': timestamp
        }

    def _store_alerts(self, alerts: List[Dict[str, Any]]):
        """Store alerts in database"""
        try:
            with db.get_connection() as conn:
                with conn.cursor() as cur:
                    for alert in alerts:
                        cur.execute("""
                            INSERT INTO alert_history (
                                gpu_index, metric_value, threshold_value, 
                                severity, created_at
                            ) VALUES (%s, %s, %s, %s, %s)
                        """, (
                            alert['gpu_index'],
                            alert['value'],
                            alert['threshold'],
                            alert['severity'],
                            alert['timestamp']
                        ))
        except Exception as e:
            logger.error(f"Failed to store alerts: {e}")

    def get_recent_alerts(self, hours: int = 24) -> List[Dict[str, Any]]:
        """Get recent alerts from database"""
        try:
            with db.get_connection() as conn:
                with conn.cursor() as cur:
                    cur.execute("""
                        SELECT * FROM alert_history
                        WHERE created_at > NOW() - interval '%s hours'
                        ORDER BY created_at DESC
                    """, (hours,))
                    columns = [desc[0] for desc in cur.description]
                    return [dict(zip(columns, row)) for row in cur.fetchall()]
        except Exception as e:
            logger.error(f"Failed to get recent alerts: {e}")
            return []

# Create singleton instance
alert_system = AlertSystem()

import logging
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Tuple
import pandas as pd
import numpy as np
from scipy import stats
from .config import config
from src.database.client import db

logger = logging.getLogger(__name__)

class AnalyticsService:
    def __init__(self):
        self.anomaly_threshold = 2.0  # Standard deviations for anomaly detection

    def get_historical_metrics(self, start_time: datetime, 
                             end_time: datetime) -> pd.DataFrame:
        """Fetch historical GPU metrics within the specified time range."""
        try:
            with db.get_connection() as conn:
                query = """
                    SELECT 
                        timestamp,
                        jsonb_array_elements(gpus) as gpu_data
                    FROM gpu_metrics
                    WHERE timestamp BETWEEN %s AND %s
                    ORDER BY timestamp
                """
                df = pd.read_sql_query(query, conn, params=(start_time, end_time))
                
                # Parse GPU data from JSONB
                df['gpu_data'] = df['gpu_data'].apply(eval)
                metrics_df = pd.json_normalize(df['gpu_data'])
                
                # Combine with timestamp
                metrics_df['timestamp'] = df['timestamp']
                
                return metrics_df
        except Exception as e:
            logger.error(f"Error fetching historical metrics: {e}")
            return pd.DataFrame()

    def calculate_usage_patterns(self, days: int = 7) -> Dict:
        """Calculate GPU usage patterns over time."""
        end_time = datetime.now()
        start_time = end_time - timedelta(days=days)
        
        df = self.get_historical_metrics(start_time, end_time)
        if df.empty:
            return {}

        patterns = {
            'hourly_avg': self._calculate_hourly_averages(df),
            'daily_avg': self._calculate_daily_averages(df),
            'peak_usage_times': self._find_peak_usage_times(df),
            'utilization_distribution': self._calculate_utilization_distribution(df)
        }
        
        return patterns

    def detect_anomalies(self, hours: int = 24) -> List[Dict]:
        """Detect anomalies in GPU metrics."""
        end_time = datetime.now()
        start_time = end_time - timedelta(hours=hours)
        
        df = self.get_historical_metrics(start_time, end_time)
        if df.empty:
            return []

        anomalies = []
        
        # Check for anomalies in different metrics
        metrics = ['utilization', 'temperature', 'memory_used', 'power_draw']
        for metric in metrics:
            if metric in df.columns:
                anomalies.extend(
                    self._detect_metric_anomalies(df, metric)
                )
        
        return anomalies

    def analyze_performance_trends(self, days: int = 30) -> Dict:
        """Analyze long-term performance trends."""
        end_time = datetime.now()
        start_time = end_time - timedelta(days=days)
        
        df = self.get_historical_metrics(start_time, end_time)
        if df.empty:
            return {}

        trends = {
            'utilization_trend': self._calculate_trend(df, 'utilization'),
            'temperature_trend': self._calculate_trend(df, 'temperature'),
            'memory_trend': self._calculate_trend(df, 'memory_used'),
            'power_trend': self._calculate_trend(df, 'power_draw')
        }
        
        return trends

    def calculate_efficiency_metrics(self, days: int = 7) -> Dict:
        """Calculate GPU efficiency metrics."""
        end_time = datetime.now()
        start_time = end_time - timedelta(days=days)
        
        df = self.get_historical_metrics(start_time, end_time)
        if df.empty:
            return {}

        metrics = {}
        
        # Calculate power efficiency (GFLOPS/Watt if available)
        if all(col in df.columns for col in ['power_draw', 'utilization']):
            metrics['power_efficiency'] = self._calculate_power_efficiency(df)
        
        # Calculate memory efficiency
        if all(col in df.columns for col in ['memory_used', 'memory_total']):
            metrics['memory_efficiency'] = self._calculate_memory_efficiency(df)
        
        return metrics

    def _calculate_hourly_averages(self, df: pd.DataFrame) -> Dict:
        """Calculate average metrics by hour of day."""
        df['hour'] = df['timestamp'].dt.hour
        hourly_avg = df.groupby('hour').agg({
            'utilization': 'mean',
            'temperature': 'mean',
            'memory_used': 'mean',
            'power_draw': 'mean'
        }).to_dict()
        
        return hourly_avg

    def _calculate_daily_averages(self, df: pd.DataFrame) -> Dict:
        """Calculate average metrics by day of week."""
        df['day'] = df['timestamp'].dt.dayofweek
        daily_avg = df.groupby('day').agg({
            'utilization': 'mean',
            'temperature': 'mean',
            'memory_used': 'mean',
            'power_draw': 'mean'
        }).to_dict()
        
        return daily_avg

    def _find_peak_usage_times(self, df: pd.DataFrame) -> List[Dict]:
        """Find times of peak GPU usage."""
        peaks = []
        metrics = ['utilization', 'temperature', 'memory_used', 'power_draw']
        
        for metric in metrics:
            if metric in df.columns:
                peak_idx = df[metric].idxmax()
                peaks.append({
                    'metric': metric,
                    'value': df.loc[peak_idx, metric],
                    'timestamp': df.loc[peak_idx, 'timestamp']
                })
        
        return peaks

    def _calculate_utilization_distribution(self, df: pd.DataFrame) -> Dict:
        """Calculate distribution of GPU utilization."""
        if 'utilization' not in df.columns:
            return {}

        bins = [0, 20, 40, 60, 80, 100]
        labels = ['0-20%', '21-40%', '41-60%', '61-80%', '81-100%']
        df['util_bin'] = pd.cut(df['utilization'], bins=bins, labels=labels)
        
        distribution = df['util_bin'].value_counts().to_dict()
        return {str(k): v for k, v in distribution.items()}

    def _detect_metric_anomalies(self, df: pd.DataFrame, 
                               metric: str) -> List[Dict]:
        """Detect anomalies in a specific metric."""
        if metric not in df.columns:
            return []

        mean = df[metric].mean()
        std = df[metric].std()
        threshold = self.anomaly_threshold * std
        
        anomalies = []
        anomaly_points = df[abs(df[metric] - mean) > threshold]
        
        for idx, row in anomaly_points.iterrows():
            anomalies.append({
                'metric': metric,
                'value': row[metric],
                'timestamp': row['timestamp'],
                'deviation': abs(row[metric] - mean) / std
            })
        
        return anomalies

    def _calculate_trend(self, df: pd.DataFrame, metric: str) -> Dict:
        """Calculate trend for a specific metric."""
        if metric not in df.columns:
            return {}

        # Calculate linear regression
        x = np.arange(len(df))
        y = df[metric].values
        slope, intercept, r_value, p_value, std_err = stats.linregress(x, y)
        
        return {
            'slope': slope,
            'r_squared': r_value**2,
            'p_value': p_value,
            'trend_direction': 'increasing' if slope > 0 else 'decreasing',
            'significance': p_value < 0.05
        }

    def _calculate_power_efficiency(self, df: pd.DataFrame) -> Dict:
        """Calculate power efficiency metrics."""
        efficiency = (df['utilization'] / df['power_draw']).mean()
        return {
            'avg_efficiency': efficiency,
            'peak_efficiency': (df['utilization'] / df['power_draw']).max()
        }

    def _calculate_memory_efficiency(self, df: pd.DataFrame) -> Dict:
        """Calculate memory efficiency metrics."""
        memory_util = (df['memory_used'] / df['memory_total']).mean() * 100
        return {
            'avg_memory_utilization': memory_util,
            'peak_memory_utilization': (df['memory_used'] / df['memory_total']).max() * 100
        }

# Create singleton instance
analytics_service = AnalyticsService()

import sys
from pathlib import Path
backend_dir = str(Path(__file__).resolve().parent.parent.parent)
if backend_dir not in sys.path:
    sys.path.insert(0, backend_dir)

from fastapi import FastAPI, HTTPException, Query
from fastapi.middleware.cors import CORSMiddleware
from fastapi.openapi.docs import get_swagger_ui_html, get_redoc_html
from fastapi.responses import JSONResponse
import subprocess
import json
import re
from collections import deque
from datetime import datetime, timedelta
from typing import Optional, List, Dict
import logging
import os
import shutil

# Import our components
from src.database.client import db
from src.models.gpu_metrics import GpuMetricsRecord, GpuBurnMetrics, NvidiaInfo, GpuMetrics
from src.service.alerts import alert_system
from src.service.system_health import SystemHealthCheck

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

# Initialize system health checker for nvidia-smi operations
system_health = SystemHealthCheck()

app = FastAPI(
    title="GPU Sentinel Pro API",
    description="""
    Enterprise-grade NVIDIA GPU monitoring API with real-time analytics and alerts.
    
    ## Features
    * Real-time GPU metrics monitoring
    * Historical data analysis
    * Alert system with configurable thresholds
    * System health diagnostics
    
    ## Authentication
    All endpoints are currently open. For enterprise deployments, configure authentication 
    as needed.
    
    ## Rate Limiting
    Default rate limit: 100 requests per minute per IP
    """,
    version="1.0.0",
    docs_url=None,  # Disable default docs to use custom endpoint
    redoc_url=None  # Disable default redoc to use custom endpoint
)

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# In-memory state
temperature_history = {}
peak_temperatures = {}
logging_enabled = True

# Custom documentation endpoints
@app.get("/docs", include_in_schema=False)
async def custom_swagger_ui_html():
    return get_swagger_ui_html(
        openapi_url="/openapi.json",
        title="GPU Sentinel Pro - API Documentation",
        swagger_js_url="https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui-bundle.js",
        swagger_css_url="https://cdn.jsdelivr.net/npm/swagger-ui-dist@5/swagger-ui.css",
    )

@app.get("/redoc", include_in_schema=False)
async def redoc_html():
    return get_redoc_html(
        openapi_url="/openapi.json",
        title="GPU Sentinel Pro - API Documentation",
        redoc_js_url="https://cdn.jsdelivr.net/npm/redoc@next/bundles/redoc.standalone.js",
    )

def get_nvidia_info() -> NvidiaInfo:
    try:
        result = system_health._run_nvidia_command([system_health.nvidia_smi_path])
        cuda_version = "Unknown"
        driver_version = "Unknown"
        
        if result.stdout:
            cuda_match = re.search(r'CUDA Version: ([\d\.]+)', result.stdout)
            if cuda_match:
                cuda_version = cuda_match.group(1)
            
            driver_match = re.search(r'Driver Version: ([\d\.]+)', result.stdout)
            if driver_match:
                driver_version = driver_match.group(1)

        return NvidiaInfo(
            driver_version=driver_version,
            cuda_version=cuda_version
        )
    except Exception as e:
        logger.error(f"Error getting NVIDIA info: {str(e)}")
        return NvidiaInfo(
            driver_version="Unknown",
            cuda_version="Unknown"
        )

def get_gpu_metrics() -> GpuMetricsRecord:
    try:
        nvidia_info = get_nvidia_info()
        
        gpu_info = system_health._run_nvidia_command([
            system_health.nvidia_smi_path,
            "--query-gpu=index,name,fan.speed,power.draw,memory.total,memory.used,utilization.gpu,temperature.gpu,compute_mode,power.limit",
            "--format=csv,noheader,nounits"
        ])

        gpus = []
        current_time = datetime.now().timestamp()
        
        if gpu_info.stdout.strip():
            for line in gpu_info.stdout.strip().split('\n'):
                values = [v.strip() for v in line.split(',')]
                if len(values) >= 10:
                    gpu_index = int(values[0])
                    temperature = float(values[7])
                    
                    if gpu_index not in temperature_history:
                        temperature_history[gpu_index] = deque(maxlen=40)
                    temperature_history[gpu_index].append((current_time, temperature))
                    
                    if gpu_index not in peak_temperatures or temperature > peak_temperatures[gpu_index]:
                        peak_temperatures[gpu_index] = temperature

                    gpu = GpuMetrics(
                        index=gpu_index,
                        name=values[1],
                        fan_speed=int(float(values[2])),
                        power_draw=float(values[3]),
                        power_limit=int(float(values[9])),
                        memory_total=int(float(values[4])),
                        memory_used=int(float(values[5])),
                        gpu_utilization=int(float(values[6])),
                        temperature=int(temperature),
                        peak_temperature=int(peak_temperatures[gpu_index]),
                        temp_change_rate=0,
                        compute_mode=values[8]
                    )
                    gpus.append(gpu)

        metrics = GpuMetricsRecord(
            nvidia_info=nvidia_info,
            gpus=gpus,
            processes=[],
            gpu_burn_metrics=GpuBurnMetrics(
                running=False,
                duration=0,
                errors=0
            ),
            success=True,
            timestamp=datetime.utcnow().isoformat()
        )

        # Check for alerts
        alert_system.check_metrics(metrics)

        # Store in database only if logging is enabled
        if logging_enabled:
            try:
                db.insert_gpu_metrics(metrics)
                logger.info("Metrics stored in database")
            except Exception as e:
                logger.error(f"Failed to store metrics: {e}")
        else:
            logger.debug("Metrics logging is disabled, skipping database insert")

        return metrics
    except Exception as e:
        logger.error(f"Error getting GPU metrics: {str(e)}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/gpu-stats", 
    response_model=List[GpuMetrics],
    tags=["Metrics"],
    summary="Get current GPU statistics",
    description="Returns real-time metrics for all available NVIDIA GPUs including temperature, utilization, memory usage, and power consumption."
)
async def get_gpu_stats():
    """Service information and status"""
    try:
        metrics = get_gpu_metrics()
        return metrics.gpus
    except Exception as e:
        logger.error(f"Error getting GPU stats: {str(e)}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/gpu-stats/history",
    response_model=List[GpuMetricsRecord],
    tags=["Metrics"],
    summary="Get historical GPU metrics",
    description="""
    Retrieve historical GPU metrics within a specified time range.
    
    - Use ISO format for dates (e.g., 2025-02-08T20:00:00Z)
    - Default lookback period is 24 hours
    - Maximum lookback period is 168 hours (1 week)
    """
)
async def get_gpu_history(
    start_time: Optional[str] = Query(
        None,
        description="Start time in ISO format (default: 24 hours ago)"
    ),
    end_time: Optional[str] = Query(
        None,
        description="End time in ISO format (default: current time)"
    ),
    hours: Optional[int] = Query(
        24,
        description="Number of hours to look back (used if start_time not provided)",
        ge=1,
        le=168  # 1 week max
    )
):
    """Get historical GPU metrics"""
    try:
        # If no start_time provided, use hours parameter
        if not start_time:
            start_time = (datetime.utcnow() - timedelta(hours=hours)).isoformat()
        
        # If no end_time provided, use current time
        if not end_time:
            end_time = datetime.utcnow().isoformat()

        # Validate and parse timestamps
        try:
            datetime.fromisoformat(start_time.replace('Z', '+00:00'))
            datetime.fromisoformat(end_time.replace('Z', '+00:00'))
        except ValueError:
            raise HTTPException(
                status_code=400,
                detail="Invalid timestamp format. Use ISO format (e.g., 2024-01-01T00:00:00Z)"
            )

        return db.get_metrics_in_timerange(start_time, end_time)
    except Exception as e:
        logger.error(f"Error getting historical data: {e}")
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/alerts",
    response_model=List[Dict],
    tags=["Alerts"],
    summary="Get recent alerts",
    description="Retrieve alerts generated within the specified time period. Includes temperature spikes, resource constraints, and system health issues."
)
async def get_alerts(
    hours: int = Query(
        24,
        description="Number of hours to look back",
        ge=1,
        le=168
    )
):
    """Get recent alerts"""
    return alert_system.get_recent_alerts(hours)

@app.post("/api/logging/toggle",
    response_model=Dict[str, bool],
    tags=["System"],
    summary="Toggle metrics logging",
    description="Enable or disable metrics logging to the database. Useful for maintenance or debugging."
)
async def toggle_logging():
    """Toggle metrics logging"""
    global logging_enabled
    logging_enabled = not logging_enabled
    logger.info(f"Metrics logging {'enabled' if logging_enabled else 'disabled'}")
    return {"logging_enabled": logging_enabled}

@app.get("/api/logging/status",
    response_model=Dict[str, bool],
    tags=["System"],
    summary="Get logging status",
    description="Check if metrics logging is currently enabled or disabled."
)
async def get_logging_status():
    """Get logging status"""
    return {"logging_enabled": logging_enabled}

@app.get("/")
async def root():
    """Service information and status"""
    return {
        "name": "GPU Metrics Service",
        "version": "1.0.0",
        "status": "running",
        "endpoints": {
            "GET /api/gpu-stats": "Current GPU metrics",
            "GET /api/gpu-stats/history": "Historical GPU metrics (optional: start_time, end_time, hours=24)",
            "GET /api/alerts": "Recent alerts",
            "GET /api/logging/status": "Get current logging status",
            "POST /api/logging/toggle": "Toggle metrics logging"
        }
    }

if __name__ == "__main__":
    import uvicorn
    logger.info("Starting GPU Metrics Service")
    uvicorn.run(
        "app:app",
        host="0.0.0.0",
        port=5183,
        reload=True
    )