#!/usr/bin/env node // ═══════════════════════════════════════════════════════════════════════════ // SIL Stability — Mät stabilitet och determinism // Erik-krav: "Samma PR analyseras tio gånger. Blir analysen identisk?" // ═══════════════════════════════════════════════════════════════════════════ import { execSync } from 'child_process'; import { readFileSync, writeFileSync, existsSync, appendFileSync } from 'fs'; import { createHash } from 'crypto'; const STABILITY_LOG = '/home/bernt/.openclaw/workspace/SIL/stability-log.jsonl'; class StabilityTest { constructor(repoPath) { this.repoPath = repoPath; } /** * Kör samma analys N gånger och jämför resultat */ runStabilityTest(prBase = 'HEAD~1', prHead = 'HEAD', runs = 10) { console.log(`🔄 Kör stabilitetstest: ${runs} analyser av samma PR\n`); const results = []; const hashes = []; for (let i = 0; i < runs; i++) { const start = Date.now(); try { const output = execSync( `node /home/bernt/.openclaw/workspace/SIL/pr-analyzer.mjs ${this.repoPath} ${prBase} ${prHead} 2>&1`, { encoding: 'utf8', maxBuffer: 10 * 1024 * 1024 } ); // Extrahera nyckeldata från output const analysis = this.parseAnalysis(output); const hash = this.hashAnalysis(analysis); results.push({ run: i + 1, duration: Date.now() - start, analysis, hash }); hashes.push(hash); console.log(` Körning ${i + 1}/${runs}: ${analysis.components.length} komponenter, hash=${hash.substring(0, 8)}`); } catch (e) { results.push({ run: i + 1, duration: Date.now() - start, error: e.message }); console.log(` Körning ${i + 1}/${runs}: FEL — ${e.message}`); } } // Beräkna stabilitet const uniqueHashes = new Set(hashes); const stabilityScore = hashes.length > 0 ? (hashes.length - uniqueHashes.size + 1) / hashes.length : 0; const determinismIndex = uniqueHashes.size === 1 ? 1.0 : 1.0 / uniqueHashes.size; // Jämför grafnoder mellan körningar const graphChurn = this.calculateGraphChurn(results); const report = { timestamp: new Date().toISOString(), pr: { base: prBase, head: prHead }, runs, stability: { stabilityScore: Math.round(stabilityScore * 100) + '%', determinismIndex: Math.round(determinismIndex * 100) + '%', uniqueResults: uniqueHashes.size, totalRuns: hashes.length, isDeterministic: uniqueHashes.size === 1 }, graphChurn, perRun: results.map(r => ({ run: r.run, duration: r.duration, components: r.analysis?.components?.length || 0, hash: r.hash?.substring(0, 8) || 'error' })), // Erik: "Om analysen varierar kraftigt trots samma indata finns ett determinismproblem" verdict: uniqueHashes.size === 1 ? '✅ DETERMINISTISK — Samma indata ger samma utdata' : uniqueHashes.size <= 2 ? '⚠️ MARGINELL VARIATION — Acceptabelt men undersök' : '🔴 ICKE-DETERMINISTISK — Kritiskt problem' }; this.logResult(report); this.printReport(report); return report; } /** * Extrahera strukturerad data från analys-output */ parseAnalysis(output) { const components = []; const lines = output.split('\n'); for (const line of lines) { // Extrahera komponenter const match = line.match(/✅\s+(.+)/); if (match && !line.includes('Körning')) { components.push(match[1].trim()); } } return { components }; } /** * Hasha analysresultat för jämförelse */ hashAnalysis(analysis) { return createHash('sha256') .update(JSON.stringify(analysis.components.sort())) .digest('hex'); } /** * Beräkna graf-churn mellan körningar */ calculateGraphChurn(results) { const validResults = results.filter(r => r.analysis && !r.error); if (validResults.length < 2) return null; // Jämför första och sista körningen const first = validResults[0].analysis.components; const last = validResults[validResults.length - 1].analysis.components; const firstSet = new Set(first); const lastSet = new Set(last); const added = [...lastSet].filter(c => !firstSet.has(c)); const removed = [...firstSet].filter(c => !lastSet.has(c)); return { nodeChurn: added.length + removed.length, added, removed, stable: added.length === 0 && removed.length === 0 }; } logResult(report) { appendFileSync(STABILITY_LOG, JSON.stringify(report) + '\n', 'utf8'); } printReport(report) { console.log('\n═══════════════════════════════════════════════════════════════'); console.log(' SIL STABILITY REPORT'); console.log('═══════════════════════════════════════════════════════════════\n'); console.log(`📊 STABILITET\n`); console.log(` Stability Score: ${report.stability.stabilityScore}`); console.log(` Determinism Index: ${report.stability.determinismIndex}`); console.log(` Unika resultat: ${report.stability.uniqueResults}/${report.stability.totalRuns}`); console.log(); console.log(`🔍 VERDIKT\n`); console.log(` ${report.verdict}\n`); if (report.graphChurn) { console.log(`📈 GRAPH CHURN\n`); console.log(` Noder ändrade: ${report.graphChurn.nodeChurn}`); if (report.graphChurn.added.length > 0) { console.log(` Tillagda: ${report.graphChurn.added.join(', ')}`); } if (report.graphChurn.removed.length > 0) { console.log(` Borttagna: ${report.graphChurn.removed.join(', ')}`); } console.log(` Stabil: ${report.graphChurn.stable ? 'Ja' : 'Nej'}\n`); } console.log(`⏱️ PER KÖRNING\n`); for (const run of report.perRun) { const icon = run.hash === report.perRun[0].hash ? '✅' : '⚠️'; console.log(` ${icon} Körning ${run.run}: ${run.duration}ms, ${run.components} komponenter, hash=${run.hash}`); } console.log(); console.log('═══════════════════════════════════════════════════════════════\n'); } /** * Kör stabilitetstest på historiska PR:er */ runHistoricalStabilityTest(count = 10) { console.log(`📚 Kör stabilitetstest på ${count} historiska PR:er...\n`); try { const merges = execSync( `git log --merges --pretty=format:"%H %P" -n ${count}`, { cwd: this.repoPath, encoding: 'utf8' } ).trim().split('\n'); for (const merge of merges) { const [hash, ...parents] = merge.split(' '); if (parents.length >= 2) { console.log(`\n🔍 Testar PR: ${hash.substring(0, 8)}`); this.runStabilityTest(parents[0], parents[1], 3); // 3 körningar per PR } } } catch (e) { console.error('Fel:', e.message); } } } // ── Main ────────────────────────────────────────────────────────────────── const repoPath = process.argv[2] || '/home/bernt/repos/quixzoom.com'; const command = process.argv[3] || '--test'; const test = new StabilityTest(repoPath); if (command === '--test') { const runs = parseInt(process.argv[4]) || 10; test.runStabilityTest('HEAD~1', 'HEAD', runs); } else if (command === '--historical') { const count = parseInt(process.argv[4]) || 10; test.runHistoricalStabilityTest(count); } else { console.log('Användning:'); console.log(' node stability.mjs [repo] --test [runs] # Kör stabilitetstest'); console.log(' node stability.mjs [repo] --historical [n] # Testa historiska PR:er'); }