diff --git a/docs/design/adr/ADR-011-Four-Layer-Data-Architecture.md b/docs/design/adr/ADR-011-Four-Layer-Data-Architecture.md new file mode 100644 index 000000000..7e603f7cd --- /dev/null +++ b/docs/design/adr/ADR-011-Four-Layer-Data-Architecture.md @@ -0,0 +1,67 @@ +# ADR-011: Four-Layer Data Architecture + +## Status +Accepted + +## Context +We need a clear separation between raw data, extracted knowledge, and decision intelligence to avoid building expensive, tightly-coupled AI systems. + +## Decision +Separate into four layers: + +### Layer 1: Raw Archive (cheap, immutable) +- Original files never changed +- Hash guarantees integrity +- Storage policy: active (30 days) → archive (long-term, cheaper) + +### Layer 2: Knowledge Extraction +- Objects, geometry, GPS, classifications, bounding boxes, embeddings +- Observations and Evidence created +- Structured knowledge used for most queries, not video files + +### Layer 3: Knowledge Graph / Ontology +- Road A12 → has Observation → Crack → belongs to Area → Stockholm +- Answers: "Show all main roads where cracking increased last 12 months" +- Comes from knowledge graph, not by reading video files + +### Layer 4: Decision Intelligence +- Verified Decision Cases +- Curated datasets for AI training +- Business impact tracking + +## Key Principles + +1. **Archive Artifact** = Original file, never modified +2. **Knowledge Artifact** = Extracted knowledge, can be re-generated +3. **Ontology before model** — taxonomy answers "what does it mean in our domain?" +4. **AI models trained on curated datasets**, not whole archive + +## Data Lifecycle + +``` +Upload → Immutable Archive → Metadata Extraction → Knowledge Extraction +→ Ontology Mapping → Decision Pipeline → Learning → Archive Retention +``` + +Each step produces a new Artifact with version history. + +## Long-term Goal + +"Every real observation is converted once to structured knowledge and can then be reused infinitely for analysis, decision support, history, training, and future AI models." + +## Consequences + +### Positive +- Raw data = revisionable archive +- Real product = growing knowledge base + verified Decision Cases +- Can swap AI models without losing originals +- Cheaper storage (archive vs active) + +### Negative +- More complex pipeline +- Need to manage extraction models +- Knowledge graph maintenance + +## Related +- ADR-002: Artifact is common base contract +- ADR-003: Event Sourcing for traceability diff --git a/packages/domain/src/artifacts/archive-artifact.ts b/packages/domain/src/artifacts/archive-artifact.ts new file mode 100644 index 000000000..5c4e7e8ce --- /dev/null +++ b/packages/domain/src/artifacts/archive-artifact.ts @@ -0,0 +1,68 @@ +/** + * Archive Artifact — Immutable Original + * + * Principle: Original files never change. + * Hash guarantees integrity. + * Storage policy: active → archive (cheaper long-term). + * + * ADR-011: Four-Layer Data Architecture + * Layer 1: Raw Archive (this file) + */ + +import { ArtifactId } from '../common/ids'; +import { Hash, StorageUri } from '../common/value-objects'; + +export interface ArchiveArtifact { + readonly id: ArtifactId; + readonly originalName: string; + readonly mimeType: string; + readonly sizeBytes: number; + readonly hash: Hash; + readonly storageUri: StorageUri; + readonly exif?: { + readonly device?: string; + readonly gpsLat?: number; + readonly gpsLng?: number; + readonly timestamp?: Date; + readonly iso?: number; + readonly exposure?: string; + readonly focalLength?: string; + }; + readonly uploadedAt: Date; + readonly uploadedBy: string; + readonly retentionPolicy: 'active' | 'archive'; + readonly archiveAfterDate?: Date; +} + +export interface CreateArchiveArtifactParams { + readonly id: ArtifactId; + readonly originalName: string; + readonly mimeType: string; + readonly sizeBytes: number; + readonly hash: Hash; + readonly storageUri: StorageUri; + readonly exif?: ArchiveArtifact['exif']; + readonly uploadedBy: string; + readonly retentionPolicy?: 'active' | 'archive'; +} + +export class ArchiveArtifactFactory { + static create(params: CreateArchiveArtifactParams): ArchiveArtifact { + return { + ...params, + retentionPolicy: params.retentionPolicy ?? 'active', + archiveAfterDate: params.retentionPolicy === 'archive' + ? undefined + : new Date(Date.now() + 30 * 24 * 60 * 60 * 1000), // 30 days default + uploadedAt: new Date(), + }; + } + + static archive(artifact: ArchiveArtifact): ArchiveArtifact { + return { + ...artifact, + retentionPolicy: 'archive', + archiveAfterDate: undefined, + }; + } +} diff --git a/packages/domain/src/artifacts/data-lifecycle.ts b/packages/domain/src/artifacts/data-lifecycle.ts new file mode 100644 index 000000000..d49803448 --- /dev/null +++ b/packages/domain/src/artifacts/data-lifecycle.ts @@ -0,0 +1,52 @@ +/** + * Data Lifecycle — From Reality to Decision + * + * ADR-011: Four-Layer Data Architecture + * + * Flow: + * Reality (phone, video, images) + * ↓ + * Raw Archive (immutable) + * ↓ + * Knowledge Extraction + * ↓ + * Knowledge Graph / Ontology + * ↓ + * Decision Intelligence + * + * Each step produces a new Artifact with its own version history. + */ + +import { ArchiveArtifact } from './archive-artifact'; +import { KnowledgeArtifact } from './knowledge-artifact'; + +export interface DataLifecycleStep { + readonly step: 'upload' | 'archive' | 'extract_metadata' | 'extract_knowledge' | 'ontology_map' | 'decision' | 'learn' | 'retain'; + readonly inputArtifactIds: string[]; + readonly outputArtifactIds: string[]; + readonly timestamp: Date; + readonly processor: string; // Model, human, or system +} + +export interface DataLifecycle { + readonly archiveArtifact: ArchiveArtifact; + readonly knowledgeArtifacts: KnowledgeArtifact[]; + readonly steps: DataLifecycleStep[]; +} + +export const DataLifecycleRules = { + // Original never changes + immutableArchive: (archive: ArchiveArtifact): boolean => { + return archive.hash !== undefined && archive.hash.value.length > 0; + }, + + // Knowledge can be re-extracted without touching archive + reextractable: (lifecycle: DataLifecycle): boolean => { + return lifecycle.archiveArtifact !== undefined; + }, + + // Every step produces artifacts + artifactAtEveryStep: (lifecycle: DataLifecycle): boolean => { + return lifecycle.steps.every(step => step.outputArtifactIds.length > 0); + }, +} as const; diff --git a/packages/domain/src/artifacts/knowledge-artifact.ts b/packages/domain/src/artifacts/knowledge-artifact.ts new file mode 100644 index 000000000..280ce138a --- /dev/null +++ b/packages/domain/src/artifacts/knowledge-artifact.ts @@ -0,0 +1,60 @@ +/** + * Knowledge Artifact — Extracted Knowledge + * + * Principle: Structured knowledge is the real product. + * Raw data becomes knowledge once, then reused infinitely. + * + * ADR-011: Four-Layer Data Architecture + * Layer 2: Knowledge Extraction (this file) + */ + +import { ArtifactId, ObservationId } from '../common/ids'; +import { Hash, StorageUri, GeoLocation } from '../common/value-objects'; + +export interface KnowledgeArtifact { + readonly id: ArtifactId; + readonly sourceArchiveId: ArtifactId; // Links to original + readonly type: 'observation' | 'segmentation' | 'classification' | 'feature_vector' | 'relation'; + readonly extractedAt: Date; + readonly extractedBy: string; // Model or human + readonly confidence: number; // 0.0 to 1.0 + readonly data: unknown; // Type-specific data + readonly hash: Hash; + readonly storageUri: StorageUri; + readonly lineage: ArtifactId[]; +} + +// Specific knowledge types +export interface ObservationKnowledge { + readonly observationId: ObservationId; + readonly description: string; + readonly location: GeoLocation; + readonly boundingBox?: { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; + }; + readonly classifications: string[]; // Taxonomy references + readonly qualityScore: number; +} + +export interface SegmentationKnowledge { + readonly maskUri: string; + readonly objectClass: string; + readonly pixelCount: number; + readonly areaMeters: number; +} + +export interface FeatureVectorKnowledge { + readonly dimensions: number; + readonly vector: number[]; + readonly modelVersion: string; +} + +export interface RelationKnowledge { + readonly subjectId: string; + readonly predicate: string; + readonly objectId: string; + readonly confidence: number; +} diff --git a/packages/domain/src/index.ts b/packages/domain/src/index.ts index 4c1a48f2d..9c3ef12d3 100644 --- a/packages/domain/src/index.ts +++ b/packages/domain/src/index.ts @@ -44,6 +44,27 @@ export { Severity, PriorityValue, } from './common/value-objects'; + +// Four-Layer Data Architecture +export { + ArchiveArtifact, + CreateArchiveArtifactParams, + ArchiveArtifactFactory, +} from './artifacts/archive-artifact'; + +export { + KnowledgeArtifact, + ObservationKnowledge, + SegmentationKnowledge, + FeatureVectorKnowledge, + RelationKnowledge, +} from './artifacts/knowledge-artifact'; + +export { + DataLifecycle, + DataLifecycleStep, + DataLifecycleRules, +} from './artifacts/data-lifecycle'; export * from './common/enums'; export * from './common/errors';