boc/SIL/validation/provenance-audit.mjs

#!/usr/bin/env node
// ═══════════════════════════════════════════════════════════════════════════
// SIL Validation — Fråga 2: Kan SIL motivera sina slutsatser?
// Granskar 50 slumpmässiga analyser för provenance completeness
// ═══════════════════════════════════════════════════════════════════════════

import { readFileSync, existsSync, writeFileSync } from 'fs';

const SHADOW_LOG = '/home/bernt/.openclaw/workspace/SIL/shadow-log.jsonl';
const AUDIT_RESULTS = '/home/bernt/.openclaw/workspace/SIL/validation/provenance-audit-results.jsonl';

/**
 * Checklista för provenance completeness
 */
const PROVENANCE_CHECKLIST = [
  {
    id: 'rule',
    question: 'Vilken regel användes?',
    check: (analysis) => analysis.provenance?.trace?.steps?.some(s => s.type === 'rule')
  },
  {
    id: 'observation',
    question: 'Vilka observationer användes?',
    check: (analysis) => analysis.provenance?.trace?.steps?.some(s => s.type === 'observation')
  },
  {
    id: 'artifacts',
    question: 'Vilka artefakter användes?',
    check: (analysis) => analysis.evidence?.static?.length > 0
  },
  {
    id: 'nodes',
    question: 'Vilka noder traverserades?',
    check: (analysis) => analysis.provenance?.trace?.steps?.some(s => s.type === 'traversal')
  },
  {
    id: 'verified',
    question: 'Hur stor del var verifierad?',
    check: (analysis) => {
      const conclusions = analysis.provenance?.trace?.conclusions || [];
      if (conclusions.length === 0) return false;
      const verified = conclusions.filter(c => c.verified).length;
      return verified / conclusions.length >= 0.5; // Minst hälften verifierad
    }
  },
  {
    id: 'inferred',
    question: 'Hur stor del var inferens?',
    check: (analysis) => {
      const conclusions = analysis.provenance?.trace?.conclusions || [];
      if (conclusions.length === 0) return false;
      const inferred = conclusions.filter(c => c.inferred).length;
      return inferred / conclusions.length <= 0.5; // Max hälften inferens
    }
  }
];

class ProvenanceAuditor {
  constructor() {
    this.analyses = this.loadAnalyses();
  }

  loadAnalyses() {
    if (!existsSync(SHADOW_LOG)) return [];
    return readFileSync(SHADOW_LOG, 'utf8')
      .split('\n')
      .filter(line => line.trim())
      .map(line => {
        try { return JSON.parse(line); } catch { return null; }
      })
      .filter(Boolean);
  }

  /**
   * Granska N slumpmässiga analyser
   */
  audit(sampleSize = 50) {
    // Välj slumpmässiga analyser
    const shuffled = [...this.analyses].sort(() => Math.random() - 0.5);
    const sample = shuffled.slice(0, Math.min(sampleSize, shuffled.length));

    console.log(`🔍 Granskar provenance för ${sample.length} analyser...\n`);

    const results = [];

    for (const analysis of sample) {
      const audit = this.auditAnalysis(analysis);
      results.push(audit);
    }

    const report = this.generateReport(results);
    this.saveResults(results);
    this.printReport(report);

    return report;
  }

  /**
   * Granska en enskild analys
   */
  auditAnalysis(analysis) {
    const checks = [];
    let passed = 0;
    let total = PROVENANCE_CHECKLIST.length;

    for (const check of PROVENANCE_CHECKLIST) {
      const result = check.check(analysis);
      checks.push({
        id: check.id,
        question: check.question,
        passed: result
      });
      if (result) passed++;
    }

    const completeness = Math.round(passed / total * 100);

    return {
      analysisId: analysis.analysisId || analysis.timestamp,
      timestamp: analysis.timestamp,
      checks,
      completeness,
      passed,
      total,
      // Kategorisera
      quality: completeness >= 90 ? 'EXCELLENT' :
               completeness >= 70 ? 'GOOD' :
               completeness >= 50 ? 'FAIR' : 'POOR'
    };
  }

  /**
   * Generera rapport
   */
  generateReport(results) {
    const total = results.length;
    if (total === 0) {
      return { error: 'Inga analyser att granska' };
    }

    const avgCompleteness = results.reduce((a, r) => a + r.completeness, 0) / total;

    const byQuality = {
      EXCELLENT: results.filter(r => r.quality === 'EXCELLENT').length,
      GOOD: results.filter(r => r.quality === 'GOOD').length,
      FAIR: results.filter(r => r.quality === 'FAIR').length,
      POOR: results.filter(r => r.quality === 'POOR').length
    };

    // Per-check resultat
    const byCheck = {};
    for (const check of PROVENANCE_CHECKLIST) {
      const passed = results.filter(r =>
        r.checks.find(c => c.id === check.id)?.passed
      ).length;
      byCheck[check.id] = {
        question: check.question,
        passed,
        total,
        rate: Math.round(passed / total * 100) + '%'
      };
    }

    return {
      timestamp: new Date().toISOString(),
      sampleSize: total,
      avgCompleteness: Math.round(avgCompleteness) + '%',
      byQuality,
      byCheck,
      // Success-kriterium
      meetsCriteria: avgCompleteness >= 90
    };
  }

  printReport(report) {
    console.log('╔═══════════════════════════════════════════════════════════════╗');
    console.log('║           VALIDATION: FRÅGA 2                                 ║');
    console.log('║           Kan SIL motivera sina slutsatser?                   ║');
    console.log('╚═══════════════════════════════════════════════════════════════╝');
    console.log();

    if (report.error) {
      console.log(`❌ ${report.error}\n`);
      return;
    }

    console.log('📊 PROVENANCE COMPLETENESS\n');
    console.log(`   Genomsnitt: ${report.avgCompleteness}`);
    console.log(`   Analyser granskade: ${report.sampleSize}`);
    console.log();

    console.log('📈 KVALITETSFÖRDELNING\n');
    console.log(`   EXCELLENT (≥90%): ${report.byQuality.EXCELLENT}`);
    console.log(`   GOOD (70-89%):    ${report.byQuality.GOOD}`);
    console.log(`   FAIR (50-69%):    ${report.byQuality.FAIR}`);
    console.log(`   POOR (<50%):      ${report.byQuality.POOR}`);
    console.log();

    console.log('✅ PER CHECK\n');
    for (const [id, data] of Object.entries(report.byCheck)) {
      const icon = parseFloat(data.rate) >= 90 ? '✅' : '⚠️';
      console.log(`   ${icon} ${data.question}`);
      console.log(`      ${data.passed}/${data.total} (${data.rate})`);
    }
    console.log();

    console.log('🎯 SUCCESS-KRITERIUM\n');
    console.log(`   Krav: ≥90% completeness`);
    console.log(`   Resultat: ${report.avgCompleteness}`);
    console.log(`   ${report.meetsCriteria ? '✅ UPPFYLLT' : '❌ EJ UPPFYLLT'}`);
    console.log();

    console.log('═══════════════════════════════════════════════════════════════\n');
  }

  saveResults(results) {
    for (const result of results) {
      writeFileSync(AUDIT_RESULTS, JSON.stringify(result) + '\n', { flag: 'a' });
    }
  }
}

// ── Main ──────────────────────────────────────────────────────────────────

const auditor = new ProvenanceAuditor();
const command = process.argv[2] || '--audit';

if (command === '--audit') {
  const sampleSize = parseInt(process.argv[3]) || 50;
  auditor.audit(sampleSize);
} else if (command === '--checklist') {
  console.log('📋 PROVENANCE CHECKLIST\n');
  for (const check of PROVENANCE_CHECKLIST) {
    console.log(`   • ${check.question}`);
  }
} else {
  console.log('Användning:');
  console.log('  node provenance-audit.mjs --audit [n]    # Granska N analyser');
  console.log('  node provenance-audit.mjs --checklist    # Visa checklista');
}