Bernt
bae705aa97
- Add NFC ePassport roadmap (ICAO 9303, eIDAS) - Add TensorFlow.js edge face detection (BlazeFace) - Add structured audit logger (GDPR-compliant) - Risk scoring support Part of KYC Apple Native UX v1.1.0
564 lines
16 KiB
JavaScript
564 lines
16 KiB
JavaScript
/**
|
|
* QUIXZOOM Active Learning — Information Gain Optimization
|
|
*
|
|
* Systemet identifierar vilka observationer som ger störst
|
|
* informationsvärde och prioriterar dem.
|
|
*
|
|
* Tre strategier:
|
|
* 1. Uncertainty Sampling — Insamla där modellen är osäker
|
|
* 2. Diversity Sampling — Insamla varierande exempel
|
|
* 3. Density-Weighted Sampling — Insamla representativa exempel
|
|
*/
|
|
|
|
const tf = require('@tensorflow/tfjs-node');
|
|
|
|
class ActiveLearning {
|
|
constructor(options = {}) {
|
|
this.strategy = options.strategy || 'uncertainty'; // uncertainty, diversity, density, combined
|
|
this.batchSize = options.batchSize || 10;
|
|
this.uncertaintyThreshold = options.uncertaintyThreshold || 0.3;
|
|
this.diversityThreshold = options.diversityThreshold || 0.5;
|
|
|
|
// Modell för att beräkna osäkerhet
|
|
this.uncertaintyModel = null;
|
|
|
|
// Feature space för diversitet
|
|
this.featureSpace = new Map();
|
|
|
|
// Historik över insamlade prover
|
|
this.collectedSamples = [];
|
|
|
|
// Information gain per område
|
|
this.informationGain = new Map();
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* 1. UNCERTAINTY SAMPLING
|
|
* ============================================================
|
|
* Insamla data där modellen är mest osäker.
|
|
* Metoder: Least Confidence, Margin Sampling, Entropy
|
|
*/
|
|
|
|
async uncertaintySampling(unlabeledPool, model) {
|
|
console.log('[ACTIVE] Running uncertainty sampling...');
|
|
|
|
const uncertainties = [];
|
|
|
|
for (const sample of unlabeledPool) {
|
|
// Förutsäg med modellen
|
|
const predictions = await model.predict(sample.features);
|
|
|
|
// Beräkna osäkerhet
|
|
const uncertainty = this.calculateUncertainty(predictions, this.strategy);
|
|
|
|
uncertainties.push({
|
|
sample,
|
|
uncertainty,
|
|
predictions,
|
|
});
|
|
}
|
|
|
|
// Sortera efter osäkerhet (högst först)
|
|
uncertainties.sort((a, b) => b.uncertainty - a.uncertainty);
|
|
|
|
// Välj toppen
|
|
const selected = uncertainties.slice(0, this.batchSize);
|
|
|
|
console.log(`[ACTIVE] Selected ${selected.length} samples by uncertainty`);
|
|
|
|
return selected.map(s => ({
|
|
...s.sample,
|
|
selectionReason: 'uncertainty',
|
|
uncertaintyScore: s.uncertainty,
|
|
}));
|
|
}
|
|
|
|
calculateUncertainty(predictions, method = 'entropy') {
|
|
const probs = predictions.arraySync()[0];
|
|
|
|
switch (method) {
|
|
case 'least_confidence':
|
|
// 1 - max(P(y|x))
|
|
return 1 - Math.max(...probs);
|
|
|
|
case 'margin':
|
|
// P(y1|x) - P(y2|x) där y1 och y2 är topp-2
|
|
const sorted = [...probs].sort((a, b) => b - a);
|
|
return 1 - (sorted[0] - sorted[1]);
|
|
|
|
case 'entropy':
|
|
// -sum(P(y|x) * log(P(y|x)))
|
|
return -probs.reduce((sum, p) => {
|
|
if (p > 0) {
|
|
return sum + p * Math.log2(p);
|
|
}
|
|
return sum;
|
|
}, 0);
|
|
|
|
default:
|
|
return 1 - Math.max(...probs);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* 2. DIVERSITY SAMPLING
|
|
* ============================================================
|
|
* Insamla varierande exempel för att täcka hela feature space.
|
|
* Metoder: Core-set, Clustering, Adversarial
|
|
*/
|
|
|
|
async diversitySampling(unlabeledPool, labeledPool) {
|
|
console.log('[ACTIVE] Running diversity sampling...');
|
|
|
|
// Extrahera features för alla prover
|
|
const unlabeledFeatures = await this.extractFeatures(unlabeledPool);
|
|
const labeledFeatures = await this.extractFeatures(labeledPool);
|
|
|
|
// Beräkna diversitet för varje opmärkt prov
|
|
const diversities = [];
|
|
|
|
for (let i = 0; i < unlabeledPool.length; i++) {
|
|
const sample = unlabeledPool[i];
|
|
const features = unlabeledFeatures[i];
|
|
|
|
// Beräkna avstånd till närmaste märkta prov
|
|
const minDistance = this.minDistanceToLabeled(features, labeledFeatures);
|
|
|
|
// Beräkna avstånd till andra opmärkta prover (för att undvika kluster)
|
|
const avgDistance = this.avgDistanceToUnlabeled(features, unlabeledFeatures, i);
|
|
|
|
diversities.push({
|
|
sample,
|
|
diversity: minDistance * avgDistance, // Kombinera
|
|
minDistance,
|
|
avgDistance,
|
|
});
|
|
}
|
|
|
|
// Sortera efter diversitet (högst först)
|
|
diversities.sort((a, b) => b.diversity - a.diversity);
|
|
|
|
// Välj toppen
|
|
const selected = diversities.slice(0, this.batchSize);
|
|
|
|
console.log(`[ACTIVE] Selected ${selected.length} samples by diversity`);
|
|
|
|
return selected.map(s => ({
|
|
...s.sample,
|
|
selectionReason: 'diversity',
|
|
diversityScore: s.diversity,
|
|
}));
|
|
}
|
|
|
|
minDistanceToLabeled(features, labeledFeatures) {
|
|
let minDist = Infinity;
|
|
|
|
for (const labeled of labeledFeatures) {
|
|
const dist = this.euclideanDistance(features, labeled);
|
|
if (dist < minDist) {
|
|
minDist = dist;
|
|
}
|
|
}
|
|
|
|
return minDist === Infinity ? 0 : minDist;
|
|
}
|
|
|
|
avgDistanceToUnlabeled(features, unlabeledFeatures, excludeIndex) {
|
|
let totalDist = 0;
|
|
let count = 0;
|
|
|
|
for (let i = 0; i < unlabeledFeatures.length; i++) {
|
|
if (i !== excludeIndex) {
|
|
totalDist += this.euclideanDistance(features, unlabeledFeatures[i]);
|
|
count++;
|
|
}
|
|
}
|
|
|
|
return count > 0 ? totalDist / count : 0;
|
|
}
|
|
|
|
euclideanDistance(a, b) {
|
|
let sum = 0;
|
|
for (let i = 0; i < a.length; i++) {
|
|
sum += (a[i] - b[i]) ** 2;
|
|
}
|
|
return Math.sqrt(sum);
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* 3. DENSITY-WEIGHTED SAMPLING
|
|
* ============================================================
|
|
* Insamla representativa exempel från täta områden i feature space.
|
|
*/
|
|
|
|
async densityWeightedSampling(unlabeledPool) {
|
|
console.log('[ACTIVE] Running density-weighted sampling...');
|
|
|
|
// Extrahera features
|
|
const features = await this.extractFeatures(unlabeledPool);
|
|
|
|
// Beräkna densitet för varje prov
|
|
const densities = [];
|
|
|
|
for (let i = 0; i < unlabeledPool.length; i++) {
|
|
const sample = unlabeledPool[i];
|
|
const feature = features[i];
|
|
|
|
// Beräkna lokal densitet (antal närliggande prover)
|
|
const density = this.calculateLocalDensity(feature, features, i);
|
|
|
|
densities.push({
|
|
sample,
|
|
density,
|
|
});
|
|
}
|
|
|
|
// Sortera efter densitet (högst först)
|
|
densities.sort((a, b) => b.density - a.density);
|
|
|
|
// Välj toppen
|
|
const selected = densities.slice(0, this.batchSize);
|
|
|
|
console.log(`[ACTIVE] Selected ${selected.length} samples by density`);
|
|
|
|
return selected.map(s => ({
|
|
...s.sample,
|
|
selectionReason: 'density',
|
|
densityScore: s.density,
|
|
}));
|
|
}
|
|
|
|
calculateLocalDensity(feature, allFeatures, excludeIndex, radius = 0.5) {
|
|
let count = 0;
|
|
|
|
for (let i = 0; i < allFeatures.length; i++) {
|
|
if (i !== excludeIndex) {
|
|
const dist = this.euclideanDistance(feature, allFeatures[i]);
|
|
if (dist < radius) {
|
|
count++;
|
|
}
|
|
}
|
|
}
|
|
|
|
return count;
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* 4. COMBINED STRATEGY
|
|
* ============================================================
|
|
* Kombinera osäkerhet, diversitet och densitet.
|
|
*/
|
|
|
|
async combinedSampling(unlabeledPool, model, labeledPool) {
|
|
console.log('[ACTIVE] Running combined sampling...');
|
|
|
|
// Kör varje strategi
|
|
const uncertaintySamples = await this.uncertaintySampling(unlabeledPool, model);
|
|
const diversitySamples = await this.diversitySampling(unlabeledPool, labeledPool);
|
|
const densitySamples = await this.densityWeightedSampling(unlabeledPool);
|
|
|
|
// Kombinera och deduplicera
|
|
const combined = new Map();
|
|
|
|
// Vikta varje strategi
|
|
for (const sample of uncertaintySamples) {
|
|
const key = sample.id;
|
|
if (!combined.has(key)) {
|
|
combined.set(key, { ...sample, score: 0, reasons: [] });
|
|
}
|
|
combined.get(key).score += sample.uncertaintyScore * 0.4;
|
|
combined.get(key).reasons.push('uncertainty');
|
|
}
|
|
|
|
for (const sample of diversitySamples) {
|
|
const key = sample.id;
|
|
if (!combined.has(key)) {
|
|
combined.set(key, { ...sample, score: 0, reasons: [] });
|
|
}
|
|
combined.get(key).score += sample.diversityScore * 0.35;
|
|
combined.get(key).reasons.push('diversity');
|
|
}
|
|
|
|
for (const sample of densitySamples) {
|
|
const key = sample.id;
|
|
if (!combined.has(key)) {
|
|
combined.set(key, { ...sample, score: 0, reasons: [] });
|
|
}
|
|
combined.get(key).score += sample.densityScore * 0.25;
|
|
combined.get(key).reasons.push('density');
|
|
}
|
|
|
|
// Sortera efter kombinerad score
|
|
const sorted = Array.from(combined.values()).sort((a, b) => b.score - a.score);
|
|
|
|
// Välj toppen
|
|
const selected = sorted.slice(0, this.batchSize);
|
|
|
|
console.log(`[ACTIVE] Selected ${selected.length} samples by combined strategy`);
|
|
|
|
return selected.map(s => ({
|
|
...s,
|
|
selectionReason: s.reasons.join('+'),
|
|
combinedScore: s.score,
|
|
}));
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* INFORMATION GAIN BERÄKNING
|
|
* ============================================================
|
|
*/
|
|
|
|
calculateInformationGain(newSample, currentModel) {
|
|
// Beräkna hur mycket modellen skulle förbättras
|
|
// om detta prov lades till träningsdata
|
|
|
|
const beforeAccuracy = this.estimateModelAccuracy(currentModel);
|
|
|
|
// Simulera träning med nytt prov
|
|
const simulatedModel = this.simulateTraining(currentModel, newSample);
|
|
const afterAccuracy = this.estimateModelAccuracy(simulatedModel);
|
|
|
|
return afterAccuracy - beforeAccuracy;
|
|
}
|
|
|
|
estimateModelAccuracy(model) {
|
|
// Förenklad uppskattning — i praktiken korsvalidering
|
|
return model.accuracy || 0.7;
|
|
}
|
|
|
|
simulateTraining(model, sample) {
|
|
// Simulera träning — i praktiken faktisk träning
|
|
return {
|
|
...model,
|
|
accuracy: Math.min(0.99, (model.accuracy || 0.7) + 0.01),
|
|
};
|
|
}
|
|
|
|
async extractFeatures(samples) {
|
|
// Extrahera features från prover
|
|
// I praktiken: kör genom en feature extractor (t.ex. ResNet)
|
|
|
|
return samples.map(s => s.features || [Math.random(), Math.random(), Math.random()]);
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* KUNSKAPSGAP-IDENTIFIERING
|
|
* ============================================================
|
|
*/
|
|
|
|
identifyKnowledgeGaps(area, existingData) {
|
|
const gaps = [];
|
|
|
|
// 1. Spatiala gap — områden utan data
|
|
const spatialGaps = this.findSpatialGaps(area, existingData);
|
|
gaps.push(...spatialGaps);
|
|
|
|
// 2. Temporala gap — tider utan data
|
|
const temporalGaps = this.findTemporalGaps(existingData);
|
|
gaps.push(...temporalGaps);
|
|
|
|
// 3. Kategoriska gap — objekttyper med få exempel
|
|
const categoricalGaps = this.findCategoricalGaps(existingData);
|
|
gaps.push(...categoricalGaps);
|
|
|
|
// 4. Kvalitetsgap — områden med lågkvalitativ data
|
|
const qualityGaps = this.findQualityGaps(existingData);
|
|
gaps.push(...qualityGaps);
|
|
|
|
return gaps;
|
|
}
|
|
|
|
findSpatialGaps(area, existingData) {
|
|
// Dela upp området i grid-celler
|
|
const gridSize = 50; // meter
|
|
const cells = new Map();
|
|
|
|
// Markera celler med data
|
|
for (const data of existingData) {
|
|
const cellX = Math.floor(data.gps.lng / gridSize);
|
|
const cellY = Math.floor(data.gps.lat / gridSize);
|
|
const key = `${cellX},${cellY}`;
|
|
cells.set(key, true);
|
|
}
|
|
|
|
// Hitta tomma celler
|
|
const gaps = [];
|
|
const bounds = area.bounds;
|
|
|
|
for (let x = bounds.minLng; x < bounds.maxLng; x += gridSize) {
|
|
for (let y = bounds.minLat; y < bounds.maxLat; y += gridSize) {
|
|
const key = `${Math.floor(x / gridSize)},${Math.floor(y / gridSize)}`;
|
|
if (!cells.has(key)) {
|
|
gaps.push({
|
|
type: 'spatial',
|
|
location: { lat: y, lng: x },
|
|
description: `No data in grid cell (${key})`,
|
|
criticality: 0.6,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
return gaps;
|
|
}
|
|
|
|
findTemporalGaps(existingData) {
|
|
// Hitta tidsperioder med lite data
|
|
const hourCounts = new Array(24).fill(0);
|
|
|
|
for (const data of existingData) {
|
|
const hour = new Date(data.timestamp).getHours();
|
|
hourCounts[hour]++;
|
|
}
|
|
|
|
const gaps = [];
|
|
const avgCount = existingData.length / 24;
|
|
|
|
for (let hour = 0; hour < 24; hour++) {
|
|
if (hourCounts[hour] < avgCount * 0.3) {
|
|
gaps.push({
|
|
type: 'temporal',
|
|
timeOfDay: hour,
|
|
description: `Few samples at ${hour}:00-${hour + 1}:00`,
|
|
criticality: 0.5,
|
|
});
|
|
}
|
|
}
|
|
|
|
return gaps;
|
|
}
|
|
|
|
findCategoricalGaps(existingData) {
|
|
// Hitta objekttyper med få exempel
|
|
const typeCounts = new Map();
|
|
|
|
for (const data of existingData) {
|
|
for (const obj of data.objects || []) {
|
|
const count = typeCounts.get(obj.class) || 0;
|
|
typeCounts.set(obj.class, count + 1);
|
|
}
|
|
}
|
|
|
|
const gaps = [];
|
|
const avgCount = existingData.length / typeCounts.size;
|
|
|
|
for (const [type, count] of typeCounts) {
|
|
if (count < avgCount * 0.2) {
|
|
gaps.push({
|
|
type: 'categorical',
|
|
objectType: type,
|
|
description: `Few examples of ${type} (${count} samples)`,
|
|
criticality: 0.7,
|
|
});
|
|
}
|
|
}
|
|
|
|
return gaps;
|
|
}
|
|
|
|
findQualityGaps(existingData) {
|
|
// Hitta områden med lågkvalitativ data
|
|
const gaps = [];
|
|
|
|
for (const data of existingData) {
|
|
if (data.quality && data.quality.score < 0.5) {
|
|
gaps.push({
|
|
type: 'quality',
|
|
location: data.gps,
|
|
description: `Low quality data at this location`,
|
|
criticality: 0.4,
|
|
});
|
|
}
|
|
}
|
|
|
|
return gaps;
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* UPPDATERING OCH STATISTIK
|
|
* ============================================================
|
|
*/
|
|
|
|
updateCollectedSamples(samples) {
|
|
this.collectedSamples.push(...samples);
|
|
|
|
// Uppdatera feature space
|
|
for (const sample of samples) {
|
|
this.featureSpace.set(sample.id, sample.features);
|
|
}
|
|
}
|
|
|
|
getStatistics() {
|
|
return {
|
|
totalCollected: this.collectedSamples.length,
|
|
byStrategy: this.collectedSamples.reduce((acc, s) => {
|
|
const strategy = s.selectionReason || 'unknown';
|
|
acc[strategy] = (acc[strategy] || 0) + 1;
|
|
return acc;
|
|
}, {}),
|
|
averageUncertainty: this.collectedSamples
|
|
.filter(s => s.uncertaintyScore)
|
|
.reduce((sum, s) => sum + s.uncertaintyScore, 0) / this.collectedSamples.length,
|
|
coverage: this.calculateKnowledgeCoverage(),
|
|
};
|
|
}
|
|
|
|
calculateKnowledgeCoverage() {
|
|
// Beräkna spatial täckning
|
|
const uniqueLocations = new Set(
|
|
this.collectedSamples.map(s => `${s.gps?.lat?.toFixed(3)},${s.gps?.lng?.toFixed(3)}`)
|
|
);
|
|
|
|
return uniqueLocations.size / 100; // Förenklad
|
|
}
|
|
}
|
|
|
|
// Exportera
|
|
module.exports = ActiveLearning;
|
|
|
|
// Demo
|
|
if (require.main === module) {
|
|
const al = new ActiveLearning({ strategy: 'combined', batchSize: 5 });
|
|
|
|
// Simulerad pool
|
|
const unlabeledPool = Array.from({ length: 100 }, (_, i) => ({
|
|
id: `sample_${i}`,
|
|
features: [Math.random(), Math.random(), Math.random()],
|
|
gps: { lat: 13.7 + Math.random() * 0.1, lng: 100.4 + Math.random() * 0.2 },
|
|
}));
|
|
|
|
const labeledPool = Array.from({ length: 20 }, (_, i) => ({
|
|
id: `labeled_${i}`,
|
|
features: [Math.random(), Math.random(), Math.random()],
|
|
gps: { lat: 13.7 + Math.random() * 0.1, lng: 100.4 + Math.random() * 0.2 },
|
|
}));
|
|
|
|
const model = {
|
|
predict: async (features) => {
|
|
// Simulerad förutsägelse
|
|
return tf.tensor2d([[Math.random(), Math.random(), Math.random()]]);
|
|
},
|
|
accuracy: 0.75,
|
|
};
|
|
|
|
console.log('=== ACTIVE LEARNING DEMO ===\n');
|
|
|
|
al.combinedSampling(unlabeledPool, model, labeledPool).then(selected => {
|
|
console.log('\nSelected samples:');
|
|
selected.forEach((s, i) => {
|
|
console.log(` ${i + 1}. ${s.id} — ${s.selectionReason} (score: ${s.combinedScore?.toFixed(3) || 'N/A'})`);
|
|
});
|
|
|
|
al.updateCollectedSamples(selected);
|
|
|
|
console.log('\nStatistics:', al.getStatistics());
|
|
});
|
|
}
|