boc/quixzoom-video-pipeline/processing/ai-annotator.js

/**
 * QUIXZOOM Video Pipeline — AI Annotator
 *
 * Kör AI-analys på extraherade frames:
 * - Scene Classification
 * - Semantic Segmentation
 * - OCR
 * - Object Detection
 * - Traffic Sign Detection
 * - Building Detection
 * - Road Surface Analysis
 * - Sidewalk Analysis
 * - Pole Detection
 * - Utility Box Detection
 * - Pavement Crack Detection
 * - Vegetation Detection
 * - Lighting Detection
 * - Storefront Detection
 * - Accessibility Detection
 *
 * Teknik: Node.js, TensorFlow.js, Sharp
 */

const tf = require('@tensorflow/tfjs-node');
const sharp = require('sharp');
const { createWorker } = require('tesseract.js');
const fs = require('fs').promises;
const path = require('path');

// Modellkonfiguration
const MODELS = {
  sceneClassification: './models/scene-classification/model.json',
  objectDetection: './models/coco-ssd/model.json',
  segmentation: './models/deeplab/model.json',
  trafficSign: './models/traffic-sign/model.json',
  building: './models/building-detection/model.json',
  roadSurface: './models/road-surface/model.json',
  sidewalk: './models/sidewalk/model.json',
  pole: './models/pole-detection/model.json',
  utilityBox: './models/utility-box/model.json',
  crack: './models/crack-detection/model.json',
  vegetation: './models/vegetation/model.json',
  lighting: './models/lighting/model.json',
  storefront: './models/storefront/model.json',
  accessibility: './models/accessibility/model.json',
};

// Laddade modeller
let loadedModels = {};

/**
 * Initiera alla AI-modeller
 */
async function initializeModels() {
  console.log('[AI] Initializing models...');

  for (const [name, modelPath] of Object.entries(MODELS)) {
    try {
      loadedModels[name] = await tf.loadGraphModel(`file://${modelPath}`);
      console.log(`[AI] Loaded: ${name}`);
    } catch (error) {
      console.warn(`[AI] Failed to load ${name}:`, error.message);
    }
  }

  console.log('[AI] All models initialized');
}

/**
 * Konvertera bild till tensor
 */
async function imageToTensor(imagePath, size = [512, 512]) {
  const { data, info } = await sharp(imagePath)
    .resize(size[0], size[1], { fit: 'fill' })
    .raw()
    .toBuffer({ resolveWithObject: true });

  return tf.tidy(() => {
    const image = tf.tensor3d(new Uint8Array(data), [info.height, info.width, 3]);
    return image.expandDims(0).toFloat().div(255.0);
  });
}

/**
 * 1. Scene Classification
 */
async function classifyScene(imagePath) {
  if (!loadedModels.sceneClassification) return null;

  const tensor = await imageToTensor(imagePath, [224, 224]);
  const predictions = await loadedModels.sceneClassification.predict(tensor).data();
  tensor.dispose();

  const labels = [
    'road', 'highway', 'intersection', 'building', 'residential',
    'commercial', 'industrial', 'park', 'water', 'bridge',
    'tunnel', 'construction', 'parking', 'sidewalk', 'alley',
    'plaza', 'market', 'residential_area', 'downtown', 'suburb',
  ];

  const results = predictions
    .map((score, idx) => ({ label: labels[idx] || 'unknown', score }))
    .sort((a, b) => b.score - a.score)
    .slice(0, 5);

  return {
    topScene: results[0],
    allScenes: results,
  };
}

/**
 * 2. Semantic Segmentation
 */
async function segmentImage(imagePath) {
  if (!loadedModels.segmentation) return null;

  const tensor = await imageToTensor(imagePath, [513, 513]);
  const predictions = await loadedModels.segmentation.predict(tensor);
  tensor.dispose();

  // Klasser för DeepLab
  const classes = [
    'background', 'road', 'sidewalk', 'building', 'wall', 'fence',
    'pole', 'traffic_light', 'traffic_sign', 'vegetation', 'terrain',
    'sky', 'person', 'rider', 'car', 'truck', 'bus', 'train',
    'motorcycle', 'bicycle',
  ];

  // Extrahera segmenteringsmask
  const mask = await predictions.argMax(-1).data();
  predictions.dispose();

  // Beräkna pixel-fördelning
  const distribution = {};
  for (let i = 0; i < mask.length; i++) {
    const classIdx = mask[i];
    const className = classes[classIdx] || 'unknown';
    distribution[className] = (distribution[className] || 0) + 1;
  }

  // Normalisera till procent
  const total = mask.length;
  for (const key in distribution) {
    distribution[key] = (distribution[key] / total * 100).toFixed(2);
  }

  return {
    mask: Array.from(mask),
    distribution,
    classes,
  };
}

/**
 * 3. OCR (Optical Character Recognition)
 */
async function performOCR(imagePath) {
  const worker = await createWorker('eng+tha+deu+fra+spa');

  const { data: { text, confidence, words } } = await worker.recognize(imagePath);
  await worker.terminate();

  // Extrahera specifika typer av text
  const signs = extractSigns(text);
  const storefronts = extractStorefronts(text);

  return {
    text,
    confidence,
    words: words.map(w => ({
      text: w.text,
      confidence: w.confidence,
      bbox: w.bbox,
    })),
    signs,
    storefronts,
  };
}

/**
 * Extrahera skyltar från OCR-text
 */
function extractSigns(text) {
  const signPatterns = [
    /STOP/i,
    /YIELD/i,
    /NO PARKING/i,
    /SPEED LIMIT\s*(\d+)/i,
    /ONE WAY/i,
    /DO NOT ENTER/i,
    /PEDESTRIAN CROSSING/i,
    /NO ENTRY/i,
    /EXIT/i,
    /ENTRANCE/i,
  ];

  const signs = [];
  for (const pattern of signPatterns) {
    const match = text.match(pattern);
    if (match) {
      signs.push({
        type: match[0],
        value: match[1] || null,
      });
    }
  }

  return signs;
}

/**
 * Extrahera butiksfasader från OCR-text
 */
function extractStorefronts(text) {
  const businessPatterns = [
    /HOTEL/i,
    /RESTAURANT/i,
    /CAFE/i,
    /SHOP/i,
    /STORE/i,
    /BANK/i,
    /PHARMACY/i,
    /CLINIC/i,
    /MASSAGE/i,
    /TOUR/i,
    /MART/i,
    /7-ELEVEN/i,
    /FAMILY MART/i,
  ];

  const storefronts = [];
  for (const pattern of businessPatterns) {
    const match = text.match(pattern);
    if (match) {
      storefronts.push(match[0]);
    }
  }

  return storefronts;
}

/**
 * 4. Object Detection
 */
async function detectObjects(imagePath) {
  if (!loadedModels.objectDetection) return null;

  const tensor = await imageToTensor(imagePath, [640, 640]);
  const predictions = await loadedModels.objectDetection.predict(tensor);
  tensor.dispose();

  // COCO-SSD klasser
  const classes = [
    'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train',
    'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign',
    'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep',
    'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella',
    'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard',
    'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard',
    'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork',
    'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange',
    'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair',
    'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv',
    'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave',
    'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase',
    'scissors', 'teddy bear', 'hair drier', 'toothbrush',
  ];

  // Formatera resultat
  const objects = predictions.map(p => ({
    class: classes[p.class] || 'unknown',
    score: p.score,
    bbox: p.bbox,
  })).filter(p => p.score > 0.5);

  return {
    objects,
    totalCount: objects.length,
    grouped: groupByClass(objects),
  };
}

/**
 * Gruppera objekt efter klass
 */
function groupByClass(objects) {
  const grouped = {};
  for (const obj of objects) {
    if (!grouped[obj.class]) grouped[obj.class] = [];
    grouped[obj.class].push(obj);
  }
  return grouped;
}

/**
 * 5. Traffic Sign Detection
 */
async function detectTrafficSigns(imagePath) {
  if (!loadedModels.trafficSign) return null;

  const tensor = await imageToTensor(imagePath, [416, 416]);
  const predictions = await loadedModels.trafficSign.predict(tensor);
  tensor.dispose();

  const signTypes = [
    'stop', 'yield', 'no_parking', 'speed_limit', 'one_way',
    'no_entry', 'pedestrian_crossing', 'school_zone', 'construction',
    'railroad_crossing', 'roundabout', 'merge', 'lane_ends',
  ];

  return predictions.map(p => ({
    type: signTypes[p.class] || 'unknown',
    confidence: p.confidence,
    bbox: p.bbox,
  })).filter(p => p.confidence > 0.6);
}

/**
 * 6. Building Detection
 */
async function detectBuildings(imagePath) {
  if (!loadedModels.building) return null;

  const tensor = await imageToTensor(imagePath, [512, 512]);
  const predictions = await loadedModels.building.predict(tensor);
  tensor.dispose();

  return {
    buildings: predictions.map(p => ({
      type: p.type, // 'residential', 'commercial', 'industrial'
      height: p.height,
      confidence: p.confidence,
      bbox: p.bbox,
    })),
    count: predictions.length,
  };
}

/**
 * 7. Road Surface Analysis
 */
async function analyzeRoadSurface(imagePath) {
  if (!loadedModels.roadSurface) return null;

  const tensor = await imageToTensor(imagePath, [512, 512]);
  const predictions = await loadedModels.roadSurface.predict(tensor);
  tensor.dispose();

  const surfaceTypes = [
    'asphalt_good', 'asphalt_fair', 'asphalt_poor',
    'concrete_good', 'concrete_fair', 'concrete_poor',
    'cobblestone', 'gravel', 'dirt', 'mud',
  ];

  const results = predictions.map((score, idx) => ({
    type: surfaceTypes[idx],
    confidence: score,
  })).sort((a, b) => b.confidence - a.confidence);

  return {
    topSurface: results[0],
    allSurfaces: results.slice(0, 5),
    hasCracks: results.some(r => r.type.includes('poor') && r.confidence > 0.5),
  };
}

/**
 * 8. Sidewalk Analysis
 */
async function analyzeSidewalk(imagePath) {
  if (!loadedModels.sidewalk) return null;

  const tensor = await imageToTensor(imagePath, [512, 512]);
  const predictions = await loadedModels.sidewalk.predict(tensor);
  tensor.dispose();

  return {
    present: predictions.present > 0.5,
    width: predictions.width,
    condition: predictions.condition, // 'good', 'fair', 'poor'
    obstacles: predictions.obstacles || [],
    accessibility: {
      wheelchair: predictions.wheelchair_accessible > 0.5,
      tactile_paving: predictions.tactile_paving > 0.5,
      curb_ramps: predictions.curb_ramps > 0.5,
    },
  };
}

/**
 * 9. Pole Detection
 */
async function detectPoles(imagePath) {
  if (!loadedModels.pole) return null;

  const tensor = await imageToTensor(imagePath, [512, 512]);
  const predictions = await loadedModels.pole.predict(tensor);
  tensor.dispose();

  const poleTypes = [
    'utility_pole', 'light_pole', 'traffic_pole', 'sign_pole',
    'telephone_pole', 'flag_pole',
  ];

  return predictions.map(p => ({
    type: poleTypes[p.class] || 'unknown',
    confidence: p.confidence,
    bbox: p.bbox,
  })).filter(p => p.confidence > 0.5);
}

/**
 * 10. Utility Box Detection
 */
async function detectUtilityBoxes(imagePath) {
  if (!loadedModels.utilityBox) return null;

  const tensor = await imageToTensor(imagePath, [512, 512]);
  const predictions = await loadedModels.utilityBox.predict(tensor);
  tensor.dispose();

  return predictions.map(p => ({
    type: p.type, // 'electrical', 'telecom', 'traffic_control'
    confidence: p.confidence,
    bbox: p.bbox,
  })).filter(p => p.confidence > 0.5);
}

/**
 * 11. Pavement Crack Detection
 */
async function detectCracks(imagePath) {
  if (!loadedModels.crack) return null;

  const tensor = await imageToTensor(imagePath, [512, 512]);
  const predictions = await loadedModels.crack.predict(tensor);
  tensor.dispose();

  return {
    hasCracks: predictions.has_cracks > 0.5,
    crackCount: predictions.crack_count,
    severity: predictions.severity, // 'low', 'medium', 'high'
    totalLength: predictions.total_length,
    locations: predictions.locations || [],
  };
}

/**
 * 12. Vegetation Detection
 */
async function detectVegetation(imagePath) {
  if (!loadedModels.vegetation) return null;

  const tensor = await imageToTensor(imagePath, [512, 512]);
  const predictions = await loadedModels.vegetation.predict(tensor);
  tensor.dispose();

  return {
    treeCount: predictions.tree_count,
    coverage: predictions.coverage,
    health: predictions.health, // 'good', 'fair', 'poor'
    species: predictions.species || [],
  };
}

/**
 * 13. Lighting Detection
 */
async function detectLighting(imagePath) {
  if (!loadedModels.lighting) return null;

  const tensor = await imageToTensor(imagePath, [512, 512]);
  const predictions = await loadedModels.lighting.predict(tensor);
  tensor.dispose();

  return {
    streetLights: predictions.street_lights || [],
    buildingLights: predictions.building_lights || [],
    naturalLight: predictions.natural_light,
    shadows: predictions.shadows,
    timeOfDay: predictions.time_of_day, // 'day', 'dusk', 'night'
  };
}

/**
 * 14. Storefront Detection
 */
async function detectStorefronts(imagePath) {
  if (!loadedModels.storefront) return null;

  const tensor = await imageToTensor(imagePath, [512, 512]);
  const predictions = await loadedModels.storefront.predict(tensor);
  tensor.dispose();

  const businessTypes = [
    'restaurant', 'cafe', 'retail', 'hotel', 'bank',
    'pharmacy', 'clinic', 'massage', 'tour_agency', 'convenience_store',
  ];

  return predictions.map(p => ({
    type: businessTypes[p.class] || 'unknown',
    name: p.name,
    confidence: p.confidence,
    bbox: p.bbox,
    open: p.open > 0.5,
  })).filter(p => p.confidence > 0.5);
}

/**
 * 15. Accessibility Detection
 */
async function detectAccessibility(imagePath) {
  if (!loadedModels.accessibility) return null;

  const tensor = await imageToTensor(imagePath, [512, 512]);
  const predictions = await loadedModels.accessibility.predict(tensor);
  tensor.dispose();

  return {
    wheelchairRamp: predictions.wheelchair_ramp > 0.5,
    tactilePaving: predictions.tactile_paving > 0.5,
    audibleSignals: predictions.audible_signals > 0.5,
    brailleSignage: predictions.braille_signage > 0.5,
    accessibleParking: predictions.accessible_parking > 0.5,
    obstacles: predictions.obstacles || [],
    overallScore: predictions.overall_score,
  };
}

/**
 * Huvudfunktion — kör all AI-analys på en frame
 */
async function analyzeFrame(framePath) {
  console.log(`[AI] Analyzing: ${path.basename(framePath)}`);

  const startTime = Date.now();

  const results = {
    framePath,
    analyzedAt: new Date().toISOString(),
    scene: await classifyScene(framePath),
    segmentation: await segmentImage(framePath),
    ocr: await performOCR(framePath),
    objects: await detectObjects(framePath),
    trafficSigns: await detectTrafficSigns(framePath),
    buildings: await detectBuildings(framePath),
    roadSurface: await analyzeRoadSurface(framePath),
    sidewalk: await analyzeSidewalk(framePath),
    poles: await detectPoles(framePath),
    utilityBoxes: await detectUtilityBoxes(framePath),
    cracks: await detectCracks(framePath),
    vegetation: await detectVegetation(framePath),
    lighting: await detectLighting(framePath),
    storefronts: await detectStorefronts(framePath),
    accessibility: await detectAccessibility(framePath),
  };

  const duration = Date.now() - startTime;
  results.processingTime = duration;

  console.log(`[AI] Completed in ${duration}ms`);

  return results;
}

/**
 * Batch-analys av flera frames
 */
async function analyzeFrames(framePaths, options = {}) {
  const results = [];
  const concurrency = options.concurrency || 2;

  for (let i = 0; i < framePaths.length; i += concurrency) {
    const batch = framePaths.slice(i, i + concurrency);
    const batchResults = await Promise.all(
      batch.map(path => analyzeFrame(path))
    );
    results.push(...batchResults);
  }

  return results;
}

module.exports = {
  initializeModels,
  analyzeFrame,
  analyzeFrames,
  classifyScene,
  segmentImage,
  performOCR,
  detectObjects,
  detectTrafficSigns,
  detectBuildings,
  analyzeRoadSurface,
  analyzeSidewalk,
  detectPoles,
  detectUtilityBoxes,
  detectCracks,
  detectVegetation,
  detectLighting,
  detectStorefronts,
  detectAccessibility,
};