Bernt
bae705aa97
- Add NFC ePassport roadmap (ICAO 9303, eIDAS) - Add TensorFlow.js edge face detection (BlazeFace) - Add structured audit logger (GDPR-compliant) - Risk scoring support Part of KYC Apple Native UX v1.1.0
524 lines
13 KiB
JavaScript
524 lines
13 KiB
JavaScript
/**
|
|
* QUIXZOOM Video Pipeline — Source Reliability System
|
|
*
|
|
* Varje datapunkt får metadata:
|
|
* - Källa
|
|
* - Datum
|
|
* - Plats
|
|
* - Licensstatus
|
|
* - Kamerakvalitet
|
|
* - Objektionsgrad (hur mycket som är skymt)
|
|
* - AI:s konfidens
|
|
* - Verifieringsstatus
|
|
*
|
|
* AI:n lär sig ENDAST av observerbara fenomen, inte av åsikter eller narration.
|
|
*/
|
|
|
|
const crypto = require('crypto');
|
|
|
|
/**
|
|
* Källa-typer och deras tillförlitlighet
|
|
*/
|
|
const SOURCE_TYPES = {
|
|
// QUIXZOOM-egen data (högst tillförlitlighet)
|
|
'quixzoom_capture': {
|
|
baseReliability: 0.95,
|
|
licenseCertainty: 1.0,
|
|
metadataCompleteness: 0.95,
|
|
qualityControl: 'active_guided',
|
|
description: 'QUIXZOOM guided capture with active learning',
|
|
},
|
|
|
|
// Creative Commons-videor
|
|
'youtube_cc': {
|
|
baseReliability: 0.75,
|
|
licenseCertainty: 0.9,
|
|
metadataCompleteness: 0.6,
|
|
qualityControl: 'post_hoc',
|
|
description: 'YouTube Creative Commons video',
|
|
},
|
|
|
|
// Öppna dataset
|
|
'open_dataset': {
|
|
baseReliability: 0.85,
|
|
licenseCertainty: 0.95,
|
|
metadataCompleteness: 0.8,
|
|
qualityControl: 'curated',
|
|
description: 'Open research dataset (Cityscapes, etc.)',
|
|
},
|
|
|
|
// Öppna kartdata
|
|
'openstreetmap': {
|
|
baseReliability: 0.8,
|
|
licenseCertainty: 0.95,
|
|
metadataCompleteness: 0.7,
|
|
qualityControl: 'community',
|
|
description: 'OpenStreetMap data',
|
|
},
|
|
|
|
// Offentliga geodata
|
|
'public_geodata': {
|
|
baseReliability: 0.85,
|
|
licenseCertainty: 0.9,
|
|
metadataCompleteness: 0.75,
|
|
qualityControl: 'government',
|
|
description: 'Public geodata with appropriate license',
|
|
},
|
|
|
|
// Dashcam (med tillstånd)
|
|
'dashcam_licensed': {
|
|
baseReliability: 0.7,
|
|
licenseCertainty: 0.8,
|
|
metadataCompleteness: 0.5,
|
|
qualityControl: 'passive',
|
|
description: 'Dashcam footage with licensing permission',
|
|
},
|
|
|
|
// Walking tours (med tillstånd)
|
|
'walking_tour_licensed': {
|
|
baseReliability: 0.7,
|
|
licenseCertainty: 0.8,
|
|
metadataCompleteness: 0.5,
|
|
qualityControl: 'passive',
|
|
description: 'Walking tour video with licensing permission',
|
|
},
|
|
};
|
|
|
|
/**
|
|
* Kvalitetsdimensioner för varje datapunkt
|
|
*/
|
|
const QUALITY_DIMENSIONS = {
|
|
// Kamerakvalitet
|
|
camera: {
|
|
resolution: { weight: 0.3, min: 720, max: 4320 }, // p (720p to 8K)
|
|
fps: { weight: 0.2, min: 24, max: 60 },
|
|
stabilization: { weight: 0.2, boolean: true },
|
|
lensQuality: { weight: 0.3, min: 0, max: 1 },
|
|
},
|
|
|
|
// Bildkvalitet
|
|
image: {
|
|
sharpness: { weight: 0.25, min: 0, max: 1 },
|
|
exposure: { weight: 0.2, min: 0, max: 1 },
|
|
colorAccuracy: { weight: 0.15, min: 0, max: 1 },
|
|
noise: { weight: 0.2, min: 0, max: 1 }, // lower is better
|
|
compression: { weight: 0.2, min: 0, max: 1 }, // lower is better
|
|
},
|
|
|
|
// Objektionsgrad (hur mycket som är skymt)
|
|
occlusion: {
|
|
foreground: { weight: 0.3, min: 0, max: 1 }, // lower is better
|
|
weather: { weight: 0.25, min: 0, max: 1 }, // lower is better
|
|
lighting: { weight: 0.25, min: 0, max: 1 }, // lower is better
|
|
motion: { weight: 0.2, min: 0, max: 1 }, // lower is better
|
|
},
|
|
|
|
// AI-konfidens
|
|
aiConfidence: {
|
|
objectDetection: { weight: 0.3, min: 0, max: 1 },
|
|
sceneClassification: { weight: 0.25, min: 0, max: 1 },
|
|
ocr: { weight: 0.2, min: 0, max: 1 },
|
|
segmentation: { weight: 0.25, min: 0, max: 1 },
|
|
},
|
|
};
|
|
|
|
/**
|
|
* Beräkna tillförlitlighet för en datapunkt
|
|
*/
|
|
function calculateReliability(dataPoint) {
|
|
const sourceConfig = SOURCE_TYPES[dataPoint.sourceType];
|
|
|
|
if (!sourceConfig) {
|
|
console.warn(`[RELIABILITY] Unknown source type: ${dataPoint.sourceType}`);
|
|
return { overall: 0, breakdown: {} };
|
|
}
|
|
|
|
// 1. Bas-tillförlitlighet från källa
|
|
let reliability = sourceConfig.baseReliability;
|
|
|
|
// 2. Justera för licens-säkerhet
|
|
reliability *= sourceConfig.licenseCertainty;
|
|
|
|
// 3. Justera för metadata-kompletthet
|
|
const metadataScore = calculateMetadataScore(dataPoint);
|
|
reliability *= (0.5 + 0.5 * metadataScore); // 0.5-1.0 range
|
|
|
|
// 4. Justera för kamerakvalitet
|
|
const cameraScore = calculateCameraScore(dataPoint);
|
|
reliability *= (0.7 + 0.3 * cameraScore); // 0.7-1.0 range
|
|
|
|
// 5. Justera för bildkvalitet
|
|
const imageScore = calculateImageScore(dataPoint);
|
|
reliability *= (0.6 + 0.4 * imageScore); // 0.6-1.0 range
|
|
|
|
// 6. Justera för objektionsgrad
|
|
const occlusionScore = calculateOcclusionScore(dataPoint);
|
|
reliability *= (0.5 + 0.5 * occlusionScore); // 0.5-1.0 range
|
|
|
|
// 7. Justera för AI-konfidens
|
|
const aiScore = calculateAIConfidenceScore(dataPoint);
|
|
reliability *= (0.6 + 0.4 * aiScore); // 0.6-1.0 range
|
|
|
|
// 8. Justera för verifieringsstatus
|
|
const verificationMultiplier = getVerificationMultiplier(dataPoint.verificationStatus);
|
|
reliability *= verificationMultiplier;
|
|
|
|
return {
|
|
overall: Math.min(1.0, reliability),
|
|
breakdown: {
|
|
sourceBase: sourceConfig.baseReliability,
|
|
license: sourceConfig.licenseCertainty,
|
|
metadata: metadataScore,
|
|
camera: cameraScore,
|
|
image: imageScore,
|
|
occlusion: occlusionScore,
|
|
aiConfidence: aiScore,
|
|
verification: verificationMultiplier,
|
|
},
|
|
sourceType: dataPoint.sourceType,
|
|
qualityControl: sourceConfig.qualityControl,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Beräkna metadata-score
|
|
*/
|
|
function calculateMetadataScore(dataPoint) {
|
|
const required = ['timestamp', 'gps', 'source', 'license'];
|
|
const optional = ['device', 'camera', 'weather', 'operator'];
|
|
|
|
let score = 0;
|
|
|
|
// Required fields
|
|
for (const field of required) {
|
|
if (dataPoint.metadata?.[field]) score += 0.15;
|
|
}
|
|
|
|
// Optional fields
|
|
for (const field of optional) {
|
|
if (dataPoint.metadata?.[field]) score += 0.05;
|
|
}
|
|
|
|
return Math.min(1.0, score);
|
|
}
|
|
|
|
/**
|
|
* Beräkna kamerakvalitet
|
|
*/
|
|
function calculateCameraScore(dataPoint) {
|
|
const camera = dataPoint.camera || {};
|
|
|
|
let score = 0;
|
|
|
|
// Resolution
|
|
const resolution = camera.resolution || 1080;
|
|
score += Math.min(1.0, (resolution - 720) / (4320 - 720)) * 0.3;
|
|
|
|
// FPS
|
|
const fps = camera.fps || 30;
|
|
score += Math.min(1.0, (fps - 24) / (60 - 24)) * 0.2;
|
|
|
|
// Stabilization
|
|
if (camera.stabilization) score += 0.2;
|
|
|
|
// Lens quality
|
|
score += (camera.lensQuality || 0.5) * 0.3;
|
|
|
|
return score;
|
|
}
|
|
|
|
/**
|
|
* Beräkna bildkvalitet
|
|
*/
|
|
function calculateImageScore(dataPoint) {
|
|
const image = dataPoint.imageQuality || {};
|
|
|
|
let score = 0;
|
|
|
|
score += (image.sharpness || 0.5) * 0.25;
|
|
score += (image.exposure || 0.5) * 0.2;
|
|
score += (image.colorAccuracy || 0.5) * 0.15;
|
|
score += (1 - (image.noise || 0.5)) * 0.2; // lower noise is better
|
|
score += (1 - (image.compression || 0.5)) * 0.2; // lower compression is better
|
|
|
|
return score;
|
|
}
|
|
|
|
/**
|
|
* Beräkna objektionsgrad (hur mycket som är skymt)
|
|
* Lower is better (0 = inget skymt, 1 = helt skymt)
|
|
*/
|
|
function calculateOcclusionScore(dataPoint) {
|
|
const occlusion = dataPoint.occlusion || {};
|
|
|
|
let score = 0;
|
|
|
|
score += (1 - (occlusion.foreground || 0)) * 0.3;
|
|
score += (1 - (occlusion.weather || 0)) * 0.25;
|
|
score += (1 - (occlusion.lighting || 0)) * 0.25;
|
|
score += (1 - (occlusion.motion || 0)) * 0.2;
|
|
|
|
return score;
|
|
}
|
|
|
|
/**
|
|
* Beräkna AI-konfidens
|
|
*/
|
|
function calculateAIConfidenceScore(dataPoint) {
|
|
const ai = dataPoint.aiAnalysis || {};
|
|
|
|
let score = 0;
|
|
|
|
score += (ai.objectDetection || 0.5) * 0.3;
|
|
score += (ai.sceneClassification || 0.5) * 0.25;
|
|
score += (ai.ocr || 0.5) * 0.2;
|
|
score += (ai.segmentation || 0.5) * 0.25;
|
|
|
|
return score;
|
|
}
|
|
|
|
/**
|
|
* Verifierings-multiplier
|
|
*/
|
|
function getVerificationMultiplier(status) {
|
|
const multipliers = {
|
|
'unverified': 0.7,
|
|
'ai_verified': 0.85,
|
|
'human_verified': 0.95,
|
|
'expert_verified': 1.0,
|
|
'disputed': 0.5,
|
|
'rejected': 0.0,
|
|
};
|
|
|
|
return multipliers[status] || 0.7;
|
|
}
|
|
|
|
/**
|
|
* Filtrera bort opålitliga datapunkter
|
|
*/
|
|
function filterReliableData(dataPoints, minReliability = 0.6) {
|
|
return dataPoints.filter(dp => {
|
|
const reliability = calculateReliability(dp);
|
|
return reliability.overall >= minReliability;
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Gruppera datapunkter efter tillförlitlighet
|
|
*/
|
|
function groupByReliability(dataPoints) {
|
|
const groups = {
|
|
high: [], // >= 0.8
|
|
medium: [], // 0.6-0.8
|
|
low: [], // 0.4-0.6
|
|
unreliable: [], // < 0.4
|
|
};
|
|
|
|
for (const dp of dataPoints) {
|
|
const reliability = calculateReliability(dp);
|
|
|
|
if (reliability.overall >= 0.8) groups.high.push({ ...dp, reliability });
|
|
else if (reliability.overall >= 0.6) groups.medium.push({ ...dp, reliability });
|
|
else if (reliability.overall >= 0.4) groups.low.push({ ...dp, reliability });
|
|
else groups.unreliable.push({ ...dp, reliability });
|
|
}
|
|
|
|
return groups;
|
|
}
|
|
|
|
/**
|
|
* Generera fullständig metadata för en datapunkt
|
|
*/
|
|
function generateDataPointMetadata(options) {
|
|
return {
|
|
// Identitet
|
|
id: options.id || crypto.randomUUID(),
|
|
timestamp: options.timestamp || new Date().toISOString(),
|
|
|
|
// Källa
|
|
sourceType: options.sourceType || 'unknown',
|
|
sourceUrl: options.sourceUrl,
|
|
sourceChannel: options.sourceChannel,
|
|
sourceUploader: options.sourceUploader,
|
|
|
|
// Licens
|
|
license: options.license || 'unknown',
|
|
licenseUrl: options.licenseUrl,
|
|
attribution: options.attribution,
|
|
|
|
// Plats
|
|
gps: {
|
|
latitude: options.latitude,
|
|
longitude: options.longitude,
|
|
accuracy: options.gpsAccuracy,
|
|
altitude: options.altitude,
|
|
},
|
|
|
|
// Kamera
|
|
camera: {
|
|
model: options.cameraModel,
|
|
resolution: options.resolution,
|
|
fps: options.fps,
|
|
stabilization: options.stabilization,
|
|
lensQuality: options.lensQuality,
|
|
},
|
|
|
|
// Bildkvalitet (beräknas av AI)
|
|
imageQuality: {
|
|
sharpness: options.sharpness,
|
|
exposure: options.exposure,
|
|
colorAccuracy: options.colorAccuracy,
|
|
noise: options.noise,
|
|
compression: options.compression,
|
|
},
|
|
|
|
// Objektionsgrad (beräknas av AI)
|
|
occlusion: {
|
|
foreground: options.foregroundOcclusion,
|
|
weather: options.weatherOcclusion,
|
|
lighting: options.lightingOcclusion,
|
|
motion: options.motionOcclusion,
|
|
},
|
|
|
|
// AI-analys
|
|
aiAnalysis: {
|
|
objectDetection: options.objectDetectionConfidence,
|
|
sceneClassification: options.sceneClassificationConfidence,
|
|
ocr: options.ocrConfidence,
|
|
segmentation: options.segmentationConfidence,
|
|
},
|
|
|
|
// Verifiering
|
|
verificationStatus: options.verificationStatus || 'unverified',
|
|
verifiedBy: options.verifiedBy,
|
|
verifiedAt: options.verifiedAt,
|
|
|
|
// Narration-filter
|
|
narrationFiltered: options.narrationFiltered || false,
|
|
narrationText: options.narrationText,
|
|
|
|
// Observerbara fenomen (detta är vad AI:n lär sig av)
|
|
observablePhenomena: options.observablePhenomena || [],
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Extrahera observerbara fenomen från video (inte åsikter)
|
|
*/
|
|
function extractObservablePhenomena(analysis) {
|
|
const phenomena = [];
|
|
|
|
// Fysiska objekt
|
|
for (const obj of analysis.objects || []) {
|
|
phenomena.push({
|
|
type: 'physical_object',
|
|
class: obj.class,
|
|
confidence: obj.confidence,
|
|
bbox: obj.bbox,
|
|
observable: true,
|
|
});
|
|
}
|
|
|
|
// Infrastruktur-tillstånd
|
|
if (analysis.infrastructure) {
|
|
phenomena.push({
|
|
type: 'infrastructure_condition',
|
|
category: analysis.infrastructure.category,
|
|
condition: analysis.infrastructure.condition,
|
|
observable: true,
|
|
});
|
|
}
|
|
|
|
// Miljöförhållanden
|
|
if (analysis.environment) {
|
|
phenomena.push({
|
|
type: 'environmental_condition',
|
|
weather: analysis.environment.weather,
|
|
lighting: analysis.environment.lighting,
|
|
observable: true,
|
|
});
|
|
}
|
|
|
|
// Människor och aktivitet
|
|
if (analysis.people) {
|
|
phenomena.push({
|
|
type: 'human_activity',
|
|
count: analysis.people.count,
|
|
density: analysis.people.density,
|
|
observable: true,
|
|
});
|
|
}
|
|
|
|
// Filtrera bort åsikter och narration
|
|
return phenomena.filter(p => p.observable);
|
|
}
|
|
|
|
/**
|
|
* Huvudfunktion — processa en datapunkt
|
|
*/
|
|
function processDataPoint(rawData) {
|
|
// 1. Generera metadata
|
|
const metadata = generateDataPointMetadata(rawData);
|
|
|
|
// 2. Extrahera observerbara fenomen
|
|
const phenomena = extractObservablePhenomena(rawData.analysis || {});
|
|
|
|
// 3. Beräkna tillförlitlighet
|
|
const reliability = calculateReliability(metadata);
|
|
|
|
// 4. Filtrera narration
|
|
const narrationFiltered = filterNarration(rawData.narration);
|
|
|
|
return {
|
|
metadata,
|
|
phenomena,
|
|
reliability,
|
|
narrationFiltered,
|
|
usableForTraining: reliability.overall >= 0.6,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Filtrera bort narration och åsikter
|
|
*/
|
|
function filterNarration(narrationText) {
|
|
if (!narrationText) return null;
|
|
|
|
// Lista över indikatorer på åsikter/narration
|
|
const opinionIndicators = [
|
|
'think', 'believe', 'feel', 'opinion', 'should', 'must',
|
|
'terrible', 'amazing', 'worst', 'best', 'dangerous', 'safe',
|
|
];
|
|
|
|
const lowerText = narrationText.toLowerCase();
|
|
|
|
for (const indicator of opinionIndicators) {
|
|
if (lowerText.includes(indicator)) {
|
|
return {
|
|
original: narrationText,
|
|
filtered: true,
|
|
reason: `Contains opinion indicator: ${indicator}`,
|
|
usable: false,
|
|
};
|
|
}
|
|
}
|
|
|
|
return {
|
|
original: narrationText,
|
|
filtered: false,
|
|
usable: true,
|
|
};
|
|
}
|
|
|
|
module.exports = {
|
|
calculateReliability,
|
|
filterReliableData,
|
|
groupByReliability,
|
|
generateDataPointMetadata,
|
|
extractObservablePhenomena,
|
|
processDataPoint,
|
|
filterNarration,
|
|
SOURCE_TYPES,
|
|
QUALITY_DIMENSIONS,
|
|
};
|