boc/SIL/experiment-report.mjs

#!/usr/bin/env node
// ═══════════════════════════════════════════════════════════════════════════
// SIL Experiment Report — Veckovis rapport för experiment-fasen
// Erik-krav: "Börja samla data nu — men behandla de kommande veckorna
//             som ett experiment, inte som en utrullning"
// ═══════════════════════════════════════════════════════════════════════════

import { readFileSync, existsSync, writeFileSync } from 'fs';
import { execSync } from 'child_process';

const SHADOW_LOG = '/home/bernt/.openclaw/workspace/SIL/shadow-log.jsonl';
const GOLD_SET = '/home/bernt/.openclaw/workspace/SIL/gold-set.jsonl';
const REPORT_DIR = '/home/bernt/.openclaw/workspace/SIL/reports';

class ExperimentReport {
  constructor() {
    this.entries = this.loadLog(SHADOW_LOG);
    this.goldEntries = this.loadLog(GOLD_SET);
  }

  loadLog(path) {
    if (!existsSync(path)) return [];
    return readFileSync(path, 'utf8')
      .split('\n')
      .filter(line => line.trim())
      .map(line => {
        try { return JSON.parse(line); } catch { return null; }
      })
      .filter(Boolean);
  }

  generateWeeklyReport(weekStart) {
    const weekEntries = this.entries.filter(e => {
      const entryDate = new Date(e.timestamp);
      const start = new Date(weekStart);
      const end = new Date(start);
      end.setDate(end.getDate() + 7);
      return entryDate >= start && entryDate < end;
    });

    const weekGold = this.goldEntries.filter(e => {
      const entryDate = new Date(e.date);
      const start = new Date(weekStart);
      const end = new Date(start);
      end.setDate(end.getDate() + 7);
      return entryDate >= start && entryDate < end;
    });

    // Beräkna metriker
    const metrics = this.calculateMetrics(weekGold);

    const report = {
      week: weekStart,
      generatedAt: new Date().toISOString(),
      summary: {
        totalAnalyses: weekEntries.length,
        goldSetEvaluations: weekGold.length,
        experimentDay: this.calculateExperimentDay()
      },
      metrics,
      learnings: this.extractLearnings(weekGold),
      recommendations: this.generateRecommendations(metrics),
      nextSteps: this.planNextSteps(metrics)
    };

    return report;
  }

  calculateMetrics(goldEntries) {
    if (goldEntries.length === 0) {
      return { error: 'Inga Gold Set-entries för denna vecka' };
    }

    let totalTP = 0, totalFP = 0, totalFN = 0;
    const byComponent = {};

    for (const entry of goldEntries) {
      const predicted = new Set(entry.predicted || []);
      const actual = new Set(entry.actual || []);

      for (const comp of actual) {
        if (!byComponent[comp]) byComponent[comp] = { tp: 0, fp: 0, fn: 0 };
        if (predicted.has(comp)) {
          byComponent[comp].tp++;
          totalTP++;
        } else {
          byComponent[comp].fn++;
          totalFN++;
        }
      }

      for (const comp of predicted) {
        if (!byComponent[comp]) byComponent[comp] = { tp: 0, fp: 0, fn: 0 };
        if (!actual.has(comp)) {
          byComponent[comp].fp++;
          totalFP++;
        }
      }
    }

    const recall = totalTP + totalFN > 0 ? totalTP / (totalTP + totalFN) : 0;
    const precision = totalTP + totalFP > 0 ? totalTP / (totalTP + totalFP) : 0;
    const f1 = recall + precision > 0 ? 2 * (recall * precision) / (recall + precision) : 0;

    return {
      recall: Math.round(recall * 100) + '%',
      precision: Math.round(precision * 100) + '%',
      f1: Math.round(f1 * 100) + '%',
      falseNegativeRate: totalTP + totalFN > 0 ? Math.round(totalFN / (totalTP + totalFN) * 100) + '%' : 'N/A',
      falsePositiveRate: totalTP + totalFP > 0 ? Math.round(totalFP / (totalTP + totalFP) * 100) + '%' : 'N/A',
      totalPRs: goldEntries.length,
      byComponent: Object.entries(byComponent).map(([comp, data]) => ({
        component: comp,
        recall: data.tp + data.fn > 0 ? Math.round(data.tp / (data.tp + data.fn) * 100) + '%' : 'N/A',
        precision: data.tp + data.fp > 0 ? Math.round(data.tp / (data.tp + data.fp) * 100) + '%' : 'N/A',
        truePositives: data.tp,
        falsePositives: data.fp,
        falseNegatives: data.fn
      }))
    };
  }

  extractLearnings(goldEntries) {
    const learnings = [];

    // Vilka typer av ändringar fungerar bäst?
    const byFileCount = {};
    for (const entry of goldEntries) {
      const bucket = entry.fileCount <= 5 ? 'small' : entry.fileCount <= 20 ? 'medium' : 'large';
      if (!byFileCount[bucket]) byFileCount[bucket] = { total: 0, correct: 0 };
      byFileCount[bucket].total++;

      const correct = entry.predictions?.filter(p => p.verified).length || 0;
      const total = entry.predictions?.length || 0;
      if (total > 0 && correct / total > 0.8) {
        byFileCount[bucket].correct++;
      }
    }

    for (const [bucket, data] of Object.entries(byFileCount)) {
      const accuracy = data.total > 0 ? Math.round(data.correct / data.total * 100) : 0;
      learnings.push({
        type: 'change_size',
        category: bucket,
        finding: `SIL presterar ${accuracy}% på ${bucket} ändringar (${data.correct}/${data.total})`,
        implication: accuracy > 80
          ? 'Bra för denna storlek'
          : 'Behöver förbättras för denna storlek'
      });
    }

    // Vilka komponenter missas oftast?
    const missedComponents = {};
    for (const entry of goldEntries) {
      const predicted = new Set(entry.predicted || []);
      for (const actual of entry.actual || []) {
        if (!predicted.has(actual)) {
          missedComponents[actual] = (missedComponents[actual] || 0) + 1;
        }
      }
    }

    const topMissed = Object.entries(missedComponents)
      .sort((a, b) => b[1] - a[1])
      .slice(0, 5);

    if (topMissed.length > 0) {
      learnings.push({
        type: 'missed_components',
        finding: `Vanligast missade komponenter: ${topMissed.map(([c, n]) => `${c}(${n})`).join(', ')}`,
        implication: 'Grafen behöver kompletteras eller fil-mappningen förbättras'
      });
    }

    // Hur ofta förändras grafen efter en PR?
    const graphChanges = goldEntries.filter(e =>
      e.predicted?.length !== e.actual?.length
    ).length;

    learnings.push({
      type: 'graph_stability',
      finding: `${graphChanges}/${goldEntries.length} PR:er krävde graf-uppdatering`,
      implication: graphChanges / goldEntries.length > 0.3
        ? 'Grafen är för instabil — behöver bättre auto-uppdatering'
        : 'Grafen är relativt stabil'
    });

    return learnings;
  }

  generateRecommendations(metrics) {
    const recs = [];

    if (metrics.error) return recs;

    const recall = parseFloat(metrics.recall);
    const precision = parseFloat(metrics.precision);
    const fnr = parseFloat(metrics.falseNegativeRate);

    if (fnr > 15) {
      recs.push({
        priority: 'HIGH',
        issue: 'För hög false negative rate',
        action: 'Förbättra komponent-detektion. SIL missar kritiska påverkade komponenter.',
        owner: 'sil-team'
      });
    }

    if (recall < 80) {
      recs.push({
        priority: 'HIGH',
        issue: 'Låg recall',
        action: 'Utöka graf-coverage eller förbättra fil-till-komponent-mappning',
        owner: 'sil-team'
      });
    }

    if (precision < 70) {
      recs.push({
        priority: 'MEDIUM',
        issue: 'Låg precision — för många falska larm',
        action: 'Justera confidence-trösklar eller förbättra heuristik',
        owner: 'sil-team'
      });
    }

    // Komponentspecifika rekommendationer
    for (const comp of metrics.byComponent || []) {
      const compRecall = parseFloat(comp.recall);
      if (compRecall < 70) {
        recs.push({
          priority: 'MEDIUM',
          issue: `Låg recall för ${comp.component}`,
          action: `Granska fil-mappning och grafnoder för ${comp.component}`,
          owner: 'sil-team'
        });
      }
    }

    return recs;
  }

  planNextSteps(metrics) {
    const steps = [];

    if (metrics.error) {
      steps.push('Bygg Gold Set: node SIL/gold-set.mjs --build');
      return steps;
    }

    const recall = parseFloat(metrics.recall);
    const precision = parseFloat(metrics.precision);
    const fnr = parseFloat(metrics.falseNegativeRate);

    if (fnr > 10 || recall < 85) {
      steps.push('Vecka 1-2: Fokus på att minska false negatives');
      steps.push('  - Granska alla missade komponenter i Gold Set');
      steps.push('  - Uppdatera fil-till-komponent-mappning');
      steps.push('  - Lägg till saknade grafnoder');
    }

    if (precision < 80) {
      steps.push('Vecka 2-3: Fokus på precision');
      steps.push('  - Justera confidence för indirekta påverkan');
      steps.push('  - Granska falska positiver i Gold Set');
    }

    if (fnr < 10 && recall > 85 && precision > 80) {
      steps.push('✅ Kriterier uppfyllda — överväg aktivering för merge-reviews');
      steps.push('  - Sätt upp webhook-integrering');
      steps.push('  - Börja med opt-in för utvecklare');
    }

    steps.push('Fortsätt samla Gold Set-data varje vecka');
    steps.push('Kör veckorapport: node SIL/experiment-report.mjs');

    return steps;
  }

  calculateExperimentDay() {
    const start = new Date('2026-06-30'); // Experimentstart
    const now = new Date();
    return Math.floor((now - start) / (1000 * 60 * 60 * 24));
  }

  printReport(report) {
    console.log('═══════════════════════════════════════════════════════════════');
    console.log('           SIL EXPERIMENT REPORT');
    console.log(`           Vecka: ${report.week}`);
    console.log(`           Dag: ${report.summary.experimentDay} av experimentet`);
    console.log('═══════════════════════════════════════════════════════════════\n');

    console.log('📊 SAMMANFATTNING\n');
    console.log(`   Analyser denna vecka: ${report.summary.totalAnalyses}`);
    console.log(`   Gold Set-evalueringar: ${report.summary.goldSetEvaluations}`);
    console.log();

    if (report.metrics.error) {
      console.log(`❌ ${report.metrics.error}\n`);
      console.log('   Kör: node SIL/gold-set.mjs --build\n');
    } else {
      console.log('📈 METRIKER\n');
      console.log(`   Recall:              ${report.metrics.recall}`);
      console.log(`   Precision:           ${report.metrics.precision}`);
      console.log(`   F1:                  ${report.metrics.f1}`);
      console.log(`   False Negative Rate: ${report.metrics.falseNegativeRate}`);
      console.log(`   False Positive Rate: ${report.metrics.falsePositiveRate}`);
      console.log();

      console.log('📋 PER KOMPONENT\n');
      for (const comp of report.metrics.byComponent) {
        console.log(`   ${comp.component}:`);
        console.log(`     recall=${comp.recall} precision=${comp.precision}`);
        console.log(`     (TP=${comp.truePositives} FP=${comp.falsePositives} FN=${comp.falseNegatives})`);
      }
      console.log();
    }

    console.log('🧠 LÄRDOMAR\n');
    for (const learning of report.learnings) {
      const icon = learning.implication.includes('Bra') || learning.implication.includes('stabil')
        ? '✅'
        : '⚠️';
      console.log(`   ${icon} ${learning.finding}`);
      console.log(`      → ${learning.implication}`);
      console.log();
    }

    console.log('💡 REKOMMENDATIONER\n');
    for (const rec of report.recommendations) {
      const icon = rec.priority === 'HIGH' ? '🔴' : '🟡';
      console.log(`   ${icon} [${rec.priority}] ${rec.issue}`);
      console.log(`      Åtgärd: ${rec.action}`);
      console.log();
    }

    console.log('🎯 NÄSTA STEG\n');
    for (const step of report.nextSteps) {
      console.log(`   • ${step}`);
    }
    console.log();

    console.log('═══════════════════════════════════════════════════════════════\n');
  }

  saveReport(report) {
    const filename = `${REPORT_DIR}/experiment-${report.week}.json`;
    try {
      writeFileSync(filename, JSON.stringify(report, null, 2));
      console.log(`💾 Rapport sparad: ${filename}\n`);
    } catch (e) {
      console.error(`❌ Kunde inte spara rapport: ${e.message}`);
    }
  }
}

// ── Main ──────────────────────────────────────────────────────────────────

const reporter = new ExperimentReport();

// Bestäm vecka (senaste måndag)
const now = new Date();
const dayOfWeek = now.getDay();
const daysSinceMonday = dayOfWeek === 0 ? 6 : dayOfWeek - 1;
const monday = new Date(now);
monday.setDate(monday.getDate() - daysSinceMonday);
const weekStart = monday.toISOString().split('T')[0];

const report = reporter.generateWeeklyReport(weekStart);
reporter.printReport(report);
reporter.saveReport(report);